1use std::env;
2use std::ffi::OsString;
3use std::fs::{self, File};
4use std::io::{BufRead, BufReader, BufWriter, ErrorKind, Write};
5use std::path::{Path, PathBuf};
6use std::sync::OnceLock;
7
8use xz2::bufread::XzDecoder;
9
10use crate::core::config::{BUILDER_CONFIG_FILENAME, TargetSelection};
11use crate::utils::build_stamp::BuildStamp;
12use crate::utils::exec::{ExecutionContext, command};
13use crate::utils::helpers::{exe, hex_encode, move_file};
14use crate::{Config, t};
15
16static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock<bool> = OnceLock::new();
17
18fn extract_curl_version(out: String) -> semver::Version {
19 out.lines()
21 .next()
22 .and_then(|line| line.split(" ").nth(1))
23 .and_then(|version| semver::Version::parse(version).ok())
24 .unwrap_or(semver::Version::new(1, 0, 0))
25}
26
27impl Config {
29 pub fn is_verbose(&self) -> bool {
30 self.exec_ctx.is_verbose()
31 }
32
33 pub(crate) fn create<P: AsRef<Path>>(&self, path: P, s: &str) {
34 if self.dry_run() {
35 return;
36 }
37 t!(fs::write(path, s));
38 }
39
40 pub(crate) fn remove(&self, f: &Path) {
41 remove(&self.exec_ctx, f);
42 }
43
44 pub(crate) fn tempdir(&self) -> PathBuf {
49 let tmp = self.out.join("tmp");
50 t!(fs::create_dir_all(&tmp));
51 tmp
52 }
53
54 fn should_fix_bins_and_dylibs(&self) -> bool {
57 should_fix_bins_and_dylibs(self.patch_binaries_for_nix, &self.exec_ctx)
58 }
59
60 fn fix_bin_or_dylib(&self, fname: &Path) {
68 fix_bin_or_dylib(&self.out, fname, &self.exec_ctx);
69 }
70
71 fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) {
72 let dwn_ctx: DownloadContext<'_> = self.into();
73 download_file(dwn_ctx, url, dest_path, help_on_error);
74 }
75
76 fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
77 unpack(&self.exec_ctx, tarball, dst, pattern);
78 }
79
80 #[cfg(test)]
82 pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
83 verify(&self.exec_ctx, path, expected)
84 }
85}
86
87fn recorded_entries(dst: &Path, pattern: &str) -> Option<BufWriter<File>> {
88 let name = if pattern == "rustc-dev" {
89 ".rustc-dev-contents"
90 } else if pattern.starts_with("rust-std") {
91 ".rust-std-contents"
92 } else {
93 return None;
94 };
95 Some(BufWriter::new(t!(File::create(dst.join(name)))))
96}
97
98#[derive(Clone)]
99enum DownloadSource {
100 CI,
101 Dist,
102}
103
104impl Config {
106 pub(crate) fn download_clippy(&self) -> PathBuf {
107 self.verbose(|| println!("downloading stage0 clippy artifacts"));
108
109 let date = &self.stage0_metadata.compiler.date;
110 let version = &self.stage0_metadata.compiler.version;
111 let host = self.host_target;
112
113 let clippy_stamp =
114 BuildStamp::new(&self.initial_sysroot).with_prefix("clippy").add_stamp(date);
115 let cargo_clippy = self.initial_sysroot.join("bin").join(exe("cargo-clippy", host));
116 if cargo_clippy.exists() && clippy_stamp.is_up_to_date() {
117 return cargo_clippy;
118 }
119
120 let filename = format!("clippy-{version}-{host}.tar.xz");
121 self.download_component(DownloadSource::Dist, filename, "clippy-preview", date, "stage0");
122 if self.should_fix_bins_and_dylibs() {
123 self.fix_bin_or_dylib(&cargo_clippy);
124 self.fix_bin_or_dylib(&cargo_clippy.with_file_name(exe("clippy-driver", host)));
125 }
126
127 t!(clippy_stamp.write());
128 cargo_clippy
129 }
130
131 pub(crate) fn ci_rust_std_contents(&self) -> Vec<String> {
132 self.ci_component_contents(".rust-std-contents")
133 }
134
135 pub(crate) fn ci_rustc_dev_contents(&self) -> Vec<String> {
136 self.ci_component_contents(".rustc-dev-contents")
137 }
138
139 fn ci_component_contents(&self, stamp_file: &str) -> Vec<String> {
140 assert!(self.download_rustc());
141 if self.dry_run() {
142 return vec![];
143 }
144
145 let ci_rustc_dir = self.ci_rustc_dir();
146 let stamp_file = ci_rustc_dir.join(stamp_file);
147 let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string());
148 t!(BufReader::new(contents_file).lines().collect())
149 }
150
151 pub(crate) fn download_ci_rustc(&self, commit: &str) {
152 self.verbose(|| println!("using downloaded stage2 artifacts from CI (commit {commit})"));
153
154 let version = self.artifact_version_part(commit);
155 let extra_components = ["rustc-dev"];
158
159 self.download_toolchain(
160 &version,
161 "ci-rustc",
162 &format!("{commit}-{}", self.llvm_assertions),
163 &extra_components,
164 Self::download_ci_component,
165 );
166 }
167
168 fn download_toolchain(
169 &self,
170 version: &str,
171 sysroot: &str,
172 stamp_key: &str,
173 extra_components: &[&str],
174 download_component: fn(&Config, String, &str, &str),
175 ) {
176 let host = self.host_target.triple;
177 let bin_root = self.out.join(host).join(sysroot);
178 let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").add_stamp(stamp_key);
179
180 if !bin_root.join("bin").join(exe("rustc", self.host_target)).exists()
181 || !rustc_stamp.is_up_to_date()
182 {
183 if bin_root.exists() {
184 t!(fs::remove_dir_all(&bin_root));
185 }
186 let filename = format!("rust-std-{version}-{host}.tar.xz");
187 let pattern = format!("rust-std-{host}");
188 download_component(self, filename, &pattern, stamp_key);
189 let filename = format!("rustc-{version}-{host}.tar.xz");
190 download_component(self, filename, "rustc", stamp_key);
191
192 for component in extra_components {
193 let filename = format!("{component}-{version}-{host}.tar.xz");
194 download_component(self, filename, component, stamp_key);
195 }
196
197 if self.should_fix_bins_and_dylibs() {
198 self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
199 self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
200 self.fix_bin_or_dylib(
201 &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
202 );
203 let lib_dir = bin_root.join("lib");
204 for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
205 let lib = t!(lib);
206 if path_is_dylib(&lib.path()) {
207 self.fix_bin_or_dylib(&lib.path());
208 }
209 }
210 }
211
212 t!(rustc_stamp.write());
213 }
214 }
215
216 fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) {
219 Self::download_component(
220 self,
221 DownloadSource::CI,
222 filename,
223 prefix,
224 commit_with_assertions,
225 "ci-rustc",
226 )
227 }
228
229 fn download_component(
230 &self,
231 mode: DownloadSource,
232 filename: String,
233 prefix: &str,
234 key: &str,
235 destination: &str,
236 ) {
237 let dwn_ctx: DownloadContext<'_> = self.into();
238 download_component(dwn_ctx, mode, filename, prefix, key, destination);
239 }
240
241 #[cfg(test)]
242 pub(crate) fn maybe_download_ci_llvm(&self) {}
243
244 #[cfg(not(test))]
245 pub(crate) fn maybe_download_ci_llvm(&self) {
246 use build_helper::exit;
247 use build_helper::git::PathFreshness;
248
249 use crate::core::build_steps::llvm::detect_llvm_freshness;
250 use crate::core::config::toml::llvm::check_incompatible_options_for_ci_llvm;
251
252 if !self.llvm_from_ci {
253 return;
254 }
255
256 let llvm_root = self.ci_llvm_root();
257 let llvm_freshness =
258 detect_llvm_freshness(self, self.rust_info.is_managed_git_subrepository());
259 self.verbose(|| {
260 eprintln!("LLVM freshness: {llvm_freshness:?}");
261 });
262 let llvm_sha = match llvm_freshness {
263 PathFreshness::LastModifiedUpstream { upstream } => upstream,
264 PathFreshness::HasLocalModifications { upstream } => upstream,
265 PathFreshness::MissingUpstream => {
266 eprintln!("error: could not find commit hash for downloading LLVM");
267 eprintln!("HELP: maybe your repository history is too shallow?");
268 eprintln!("HELP: consider disabling `download-ci-llvm`");
269 eprintln!("HELP: or fetch enough history to include one upstream commit");
270 crate::exit!(1);
271 }
272 };
273 let stamp_key = format!("{}{}", llvm_sha, self.llvm_assertions);
274 let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm").add_stamp(stamp_key);
275 if !llvm_stamp.is_up_to_date() && !self.dry_run() {
276 self.download_ci_llvm(&llvm_sha);
277
278 if self.should_fix_bins_and_dylibs() {
279 for entry in t!(fs::read_dir(llvm_root.join("bin"))) {
280 self.fix_bin_or_dylib(&t!(entry).path());
281 }
282 }
283
284 let now = std::time::SystemTime::now();
293 let file_times = fs::FileTimes::new().set_accessed(now).set_modified(now);
294
295 let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.host_target));
296 t!(crate::utils::helpers::set_file_times(llvm_config, file_times));
297
298 if self.should_fix_bins_and_dylibs() {
299 let llvm_lib = llvm_root.join("lib");
300 for entry in t!(fs::read_dir(llvm_lib)) {
301 let lib = t!(entry).path();
302 if path_is_dylib(&lib) {
303 self.fix_bin_or_dylib(&lib);
304 }
305 }
306 }
307
308 t!(llvm_stamp.write());
309 }
310
311 if let Some(config_path) = &self.config {
312 let current_config_toml = Self::get_toml(config_path).unwrap();
313
314 match self.get_builder_toml("ci-llvm") {
315 Ok(ci_config_toml) => {
316 t!(check_incompatible_options_for_ci_llvm(current_config_toml, ci_config_toml));
317 }
318 Err(e) if e.to_string().contains("unknown field") => {
319 println!(
320 "WARNING: CI LLVM has some fields that are no longer supported in bootstrap; download-ci-llvm will be disabled."
321 );
322 println!("HELP: Consider rebasing to a newer commit if available.");
323 }
324 Err(e) => {
325 eprintln!("ERROR: Failed to parse CI LLVM bootstrap.toml: {e}");
326 exit!(2);
327 }
328 };
329 };
330 }
331
332 #[cfg(not(test))]
333 fn download_ci_llvm(&self, llvm_sha: &str) {
334 let llvm_assertions = self.llvm_assertions;
335
336 let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}");
337 let cache_dst =
338 self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
339
340 let rustc_cache = cache_dst.join(cache_prefix);
341 if !rustc_cache.exists() {
342 t!(fs::create_dir_all(&rustc_cache));
343 }
344 let base = if llvm_assertions {
345 &self.stage0_metadata.config.artifacts_with_llvm_assertions_server
346 } else {
347 &self.stage0_metadata.config.artifacts_server
348 };
349 let version = self.artifact_version_part(llvm_sha);
350 let filename = format!("rust-dev-{}-{}.tar.xz", version, self.host_target.triple);
351 let tarball = rustc_cache.join(&filename);
352 if !tarball.exists() {
353 let help_on_error = "ERROR: failed to download llvm from ci
354
355 HELP: There could be two reasons behind this:
356 1) The host triple is not supported for `download-ci-llvm`.
357 2) Old builds get deleted after a certain time.
358 HELP: In either case, disable `download-ci-llvm` in your bootstrap.toml:
359
360 [llvm]
361 download-ci-llvm = false
362 ";
363 self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error);
364 }
365 let llvm_root = self.ci_llvm_root();
366 self.unpack(&tarball, &llvm_root, "rust-dev");
367 }
368
369 pub fn download_ci_gcc(&self, gcc_sha: &str, root_dir: &Path) {
370 let cache_prefix = format!("gcc-{gcc_sha}");
371 let cache_dst =
372 self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
373
374 let gcc_cache = cache_dst.join(cache_prefix);
375 if !gcc_cache.exists() {
376 t!(fs::create_dir_all(&gcc_cache));
377 }
378 let base = &self.stage0_metadata.config.artifacts_server;
379 let version = self.artifact_version_part(gcc_sha);
380 let filename = format!("gcc-{version}-{}.tar.xz", self.host_target.triple);
381 let tarball = gcc_cache.join(&filename);
382 if !tarball.exists() {
383 let help_on_error = "ERROR: failed to download gcc from ci
384
385 HELP: There could be two reasons behind this:
386 1) The host triple is not supported for `download-ci-gcc`.
387 2) Old builds get deleted after a certain time.
388 HELP: In either case, disable `download-ci-gcc` in your bootstrap.toml:
389
390 [gcc]
391 download-ci-gcc = false
392 ";
393 self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error);
394 }
395 self.unpack(&tarball, root_dir, "gcc");
396 }
397}
398
399pub(crate) struct DownloadContext<'a> {
401 host_target: TargetSelection,
402 out: &'a Path,
403 patch_binaries_for_nix: Option<bool>,
404 exec_ctx: &'a ExecutionContext,
405 stage0_metadata: &'a build_helper::stage0_parser::Stage0,
406 llvm_assertions: bool,
407 bootstrap_cache_path: &'a Option<PathBuf>,
408 is_running_on_ci: bool,
409}
410
411impl<'a> AsRef<DownloadContext<'a>> for DownloadContext<'a> {
412 fn as_ref(&self) -> &DownloadContext<'a> {
413 self
414 }
415}
416
417impl<'a> From<&'a Config> for DownloadContext<'a> {
418 fn from(value: &'a Config) -> Self {
419 DownloadContext {
420 host_target: value.host_target,
421 out: &value.out,
422 patch_binaries_for_nix: value.patch_binaries_for_nix,
423 exec_ctx: &value.exec_ctx,
424 stage0_metadata: &value.stage0_metadata,
425 llvm_assertions: value.llvm_assertions,
426 bootstrap_cache_path: &value.bootstrap_cache_path,
427 is_running_on_ci: value.is_running_on_ci,
428 }
429 }
430}
431
432fn path_is_dylib(path: &Path) -> bool {
433 path.to_str().is_some_and(|path| path.contains(".so"))
435}
436
437pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool {
439 const SUPPORTED_PLATFORMS: &[&str] = &[
441 "aarch64-apple-darwin",
442 "aarch64-pc-windows-msvc",
443 "aarch64-unknown-linux-gnu",
444 "aarch64-unknown-linux-musl",
445 "arm-unknown-linux-gnueabi",
446 "arm-unknown-linux-gnueabihf",
447 "armv7-unknown-linux-gnueabihf",
448 "i686-pc-windows-gnu",
449 "i686-pc-windows-msvc",
450 "i686-unknown-linux-gnu",
451 "loongarch64-unknown-linux-gnu",
452 "powerpc-unknown-linux-gnu",
453 "powerpc64-unknown-linux-gnu",
454 "powerpc64le-unknown-linux-gnu",
455 "powerpc64le-unknown-linux-musl",
456 "riscv64gc-unknown-linux-gnu",
457 "s390x-unknown-linux-gnu",
458 "x86_64-apple-darwin",
459 "x86_64-pc-windows-gnu",
460 "x86_64-pc-windows-msvc",
461 "x86_64-unknown-freebsd",
462 "x86_64-unknown-illumos",
463 "x86_64-unknown-linux-gnu",
464 "x86_64-unknown-linux-musl",
465 "x86_64-unknown-netbsd",
466 ];
467
468 const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] =
469 &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"];
470
471 if llvm_assertions {
472 SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple)
473 } else {
474 SUPPORTED_PLATFORMS.contains(&target_triple)
475 }
476}
477
478#[cfg(test)]
479pub(crate) fn maybe_download_rustfmt<'a>(
480 dwn_ctx: impl AsRef<DownloadContext<'a>>,
481) -> Option<PathBuf> {
482 Some(PathBuf::new())
483}
484
485#[cfg(not(test))]
488pub(crate) fn maybe_download_rustfmt<'a>(
489 dwn_ctx: impl AsRef<DownloadContext<'a>>,
490) -> Option<PathBuf> {
491 use build_helper::stage0_parser::VersionMetadata;
492
493 let dwn_ctx = dwn_ctx.as_ref();
494
495 if dwn_ctx.exec_ctx.dry_run() {
496 return Some(PathBuf::new());
497 }
498
499 let VersionMetadata { date, version } = dwn_ctx.stage0_metadata.rustfmt.as_ref()?;
500 let channel = format!("{version}-{date}");
501
502 let host = dwn_ctx.host_target;
503 let bin_root = dwn_ctx.out.join(host).join("rustfmt");
504 let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host));
505 let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").add_stamp(channel);
506 if rustfmt_path.exists() && rustfmt_stamp.is_up_to_date() {
507 return Some(rustfmt_path);
508 }
509
510 download_component(
511 dwn_ctx,
512 DownloadSource::Dist,
513 format!("rustfmt-{version}-{build}.tar.xz", build = host.triple),
514 "rustfmt-preview",
515 date,
516 "rustfmt",
517 );
518
519 download_component(
520 dwn_ctx,
521 DownloadSource::Dist,
522 format!("rustc-{version}-{build}.tar.xz", build = host.triple),
523 "rustc",
524 date,
525 "rustfmt",
526 );
527
528 if should_fix_bins_and_dylibs(dwn_ctx.patch_binaries_for_nix, dwn_ctx.exec_ctx) {
529 fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("rustfmt"), dwn_ctx.exec_ctx);
530 fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("cargo-fmt"), dwn_ctx.exec_ctx);
531 let lib_dir = bin_root.join("lib");
532 for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
533 let lib = t!(lib);
534 if path_is_dylib(&lib.path()) {
535 fix_bin_or_dylib(dwn_ctx.out, &lib.path(), dwn_ctx.exec_ctx);
536 }
537 }
538 }
539
540 t!(rustfmt_stamp.write());
541 Some(rustfmt_path)
542}
543
544#[cfg(test)]
545pub(crate) fn download_beta_toolchain<'a>(dwn_ctx: impl AsRef<DownloadContext<'a>>) {}
546
547#[cfg(not(test))]
548pub(crate) fn download_beta_toolchain<'a>(dwn_ctx: impl AsRef<DownloadContext<'a>>) {
549 let dwn_ctx = dwn_ctx.as_ref();
550 dwn_ctx.exec_ctx.verbose(|| {
551 println!("downloading stage0 beta artifacts");
552 });
553
554 let date = dwn_ctx.stage0_metadata.compiler.date.clone();
555 let version = dwn_ctx.stage0_metadata.compiler.version.clone();
556 let extra_components = ["cargo"];
557 let sysroot = "stage0";
558 download_toolchain(
559 dwn_ctx,
560 &version,
561 sysroot,
562 &date,
563 &extra_components,
564 "stage0",
565 DownloadSource::Dist,
566 );
567}
568
569fn download_toolchain<'a>(
570 dwn_ctx: impl AsRef<DownloadContext<'a>>,
571 version: &str,
572 sysroot: &str,
573 stamp_key: &str,
574 extra_components: &[&str],
575 destination: &str,
576 mode: DownloadSource,
577) {
578 let dwn_ctx = dwn_ctx.as_ref();
579 let host = dwn_ctx.host_target.triple;
580 let bin_root = dwn_ctx.out.join(host).join(sysroot);
581 let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").add_stamp(stamp_key);
582
583 if !bin_root.join("bin").join(exe("rustc", dwn_ctx.host_target)).exists()
584 || !rustc_stamp.is_up_to_date()
585 {
586 if bin_root.exists() {
587 t!(fs::remove_dir_all(&bin_root));
588 }
589 let filename = format!("rust-std-{version}-{host}.tar.xz");
590 let pattern = format!("rust-std-{host}");
591 download_component(dwn_ctx, mode.clone(), filename, &pattern, stamp_key, destination);
592 let filename = format!("rustc-{version}-{host}.tar.xz");
593 download_component(dwn_ctx, mode.clone(), filename, "rustc", stamp_key, destination);
594
595 for component in extra_components {
596 let filename = format!("{component}-{version}-{host}.tar.xz");
597 download_component(dwn_ctx, mode.clone(), filename, component, stamp_key, destination);
598 }
599
600 if should_fix_bins_and_dylibs(dwn_ctx.patch_binaries_for_nix, dwn_ctx.exec_ctx) {
601 fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("rustc"), dwn_ctx.exec_ctx);
602 fix_bin_or_dylib(dwn_ctx.out, &bin_root.join("bin").join("rustdoc"), dwn_ctx.exec_ctx);
603 fix_bin_or_dylib(
604 dwn_ctx.out,
605 &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"),
606 dwn_ctx.exec_ctx,
607 );
608 let lib_dir = bin_root.join("lib");
609 for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) {
610 let lib = t!(lib);
611 if path_is_dylib(&lib.path()) {
612 fix_bin_or_dylib(dwn_ctx.out, &lib.path(), dwn_ctx.exec_ctx);
613 }
614 }
615 }
616
617 t!(rustc_stamp.write());
618 }
619}
620
621pub(crate) fn remove(exec_ctx: &ExecutionContext, f: &Path) {
622 if exec_ctx.dry_run() {
623 return;
624 }
625 fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {f:?}"));
626}
627
628fn fix_bin_or_dylib(out: &Path, fname: &Path, exec_ctx: &ExecutionContext) {
629 assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true));
630 println!("attempting to patch {}", fname.display());
631
632 static NIX_DEPS_DIR: OnceLock<PathBuf> = OnceLock::new();
634 let mut nix_build_succeeded = true;
635 let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
636 let nix_deps_dir = out.join(".nix-deps");
648 const NIX_EXPR: &str = "
649 with (import <nixpkgs> {});
650 symlinkJoin {
651 name = \"rust-stage0-dependencies\";
652 paths = [
653 zlib
654 patchelf
655 stdenv.cc.bintools
656 ];
657 }
658 ";
659 nix_build_succeeded = command("nix-build")
660 .allow_failure()
661 .args([Path::new("-E"), Path::new(NIX_EXPR), Path::new("-o"), &nix_deps_dir])
662 .run_capture_stdout(exec_ctx)
663 .is_success();
664 nix_deps_dir
665 });
666 if !nix_build_succeeded {
667 return;
668 }
669
670 let mut patchelf = command(nix_deps_dir.join("bin/patchelf"));
671 patchelf.args(&[
672 OsString::from("--add-rpath"),
673 OsString::from(t!(fs::canonicalize(nix_deps_dir)).join("lib")),
674 ]);
675 if !path_is_dylib(fname) {
676 let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
678 let dynamic_linker = t!(fs::read_to_string(dynamic_linker_path));
679 patchelf.args(["--set-interpreter", dynamic_linker.trim_end()]);
680 }
681 patchelf.arg(fname);
682 let _ = patchelf.allow_failure().run_capture_stdout(exec_ctx);
683}
684
685fn should_fix_bins_and_dylibs(
686 patch_binaries_for_nix: Option<bool>,
687 exec_ctx: &ExecutionContext,
688) -> bool {
689 let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| {
690 let uname = command("uname").allow_failure().arg("-s").run_capture_stdout(exec_ctx);
691 if uname.is_failure() {
692 return false;
693 }
694 let output = uname.stdout();
695 if !output.starts_with("Linux") {
696 return false;
697 }
698 if let Some(explicit_value) = patch_binaries_for_nix {
704 return explicit_value;
705 }
706
707 let is_nixos = match File::open("/etc/os-release") {
710 Err(e) if e.kind() == ErrorKind::NotFound => false,
711 Err(e) => panic!("failed to access /etc/os-release: {e}"),
712 Ok(os_release) => BufReader::new(os_release).lines().any(|l| {
713 let l = l.expect("reading /etc/os-release");
714 matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"")
715 }),
716 };
717 if !is_nixos {
718 let in_nix_shell = env::var("IN_NIX_SHELL");
719 if let Ok(in_nix_shell) = in_nix_shell {
720 eprintln!(
721 "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \
722 you may need to set `patch-binaries-for-nix=true` in bootstrap.toml"
723 );
724 }
725 }
726 is_nixos
727 });
728 if val {
729 eprintln!("INFO: You seem to be using Nix.");
730 }
731 val
732}
733
734fn download_component<'a>(
735 dwn_ctx: impl AsRef<DownloadContext<'a>>,
736 mode: DownloadSource,
737 filename: String,
738 prefix: &str,
739 key: &str,
740 destination: &str,
741) {
742 let dwn_ctx = dwn_ctx.as_ref();
743
744 if dwn_ctx.exec_ctx.dry_run() {
745 return;
746 }
747
748 let cache_dst =
749 dwn_ctx.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| dwn_ctx.out.join("cache"));
750
751 let cache_dir = cache_dst.join(key);
752 if !cache_dir.exists() {
753 t!(fs::create_dir_all(&cache_dir));
754 }
755
756 let bin_root = dwn_ctx.out.join(dwn_ctx.host_target).join(destination);
757 let tarball = cache_dir.join(&filename);
758 let (base_url, url, should_verify) = match mode {
759 DownloadSource::CI => {
760 let dist_server = if dwn_ctx.llvm_assertions {
761 dwn_ctx.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone()
762 } else {
763 dwn_ctx.stage0_metadata.config.artifacts_server.clone()
764 };
765 let url = format!(
766 "{}/{filename}",
767 key.strip_suffix(&format!("-{}", dwn_ctx.llvm_assertions)).unwrap()
768 );
769 (dist_server, url, false)
770 }
771 DownloadSource::Dist => {
772 let dist_server = env::var("RUSTUP_DIST_SERVER")
773 .unwrap_or(dwn_ctx.stage0_metadata.config.dist_server.to_string());
774 (dist_server, format!("dist/{key}/{filename}"), true)
776 }
777 };
778
779 let checksum = if should_verify {
781 let error = format!(
782 "src/stage0 doesn't contain a checksum for {url}. \
783 Pre-built artifacts might not be available for this \
784 target at this time, see https://doc.rust-lang.org/nightly\
785 /rustc/platform-support.html for more information."
786 );
787 let sha256 = dwn_ctx.stage0_metadata.checksums_sha256.get(&url).expect(&error);
788 if tarball.exists() {
789 if verify(dwn_ctx.exec_ctx, &tarball, sha256) {
790 unpack(dwn_ctx.exec_ctx, &tarball, &bin_root, prefix);
791 return;
792 } else {
793 dwn_ctx.exec_ctx.verbose(|| {
794 println!(
795 "ignoring cached file {} due to failed verification",
796 tarball.display()
797 )
798 });
799 remove(dwn_ctx.exec_ctx, &tarball);
800 }
801 }
802 Some(sha256)
803 } else if tarball.exists() {
804 unpack(dwn_ctx.exec_ctx, &tarball, &bin_root, prefix);
805 return;
806 } else {
807 None
808 };
809
810 let mut help_on_error = "";
811 if destination == "ci-rustc" {
812 help_on_error = "ERROR: failed to download pre-built rustc from CI
813
814NOTE: old builds get deleted after a certain time
815HELP: if trying to compile an old commit of rustc, disable `download-rustc` in bootstrap.toml:
816
817[rust]
818download-rustc = false
819";
820 }
821 download_file(dwn_ctx, &format!("{base_url}/{url}"), &tarball, help_on_error);
822 if let Some(sha256) = checksum
823 && !verify(dwn_ctx.exec_ctx, &tarball, sha256)
824 {
825 panic!("failed to verify {}", tarball.display());
826 }
827
828 unpack(dwn_ctx.exec_ctx, &tarball, &bin_root, prefix);
829}
830
831pub(crate) fn verify(exec_ctx: &ExecutionContext, path: &Path, expected: &str) -> bool {
832 use sha2::Digest;
833
834 exec_ctx.verbose(|| {
835 println!("verifying {}", path.display());
836 });
837
838 if exec_ctx.dry_run() {
839 return false;
840 }
841
842 let mut hasher = sha2::Sha256::new();
843
844 let file = t!(File::open(path));
845 let mut reader = BufReader::new(file);
846
847 loop {
848 let buffer = t!(reader.fill_buf());
849 let l = buffer.len();
850 if l == 0 {
852 break;
853 }
854 hasher.update(buffer);
855 reader.consume(l);
856 }
857
858 let checksum = hex_encode(hasher.finalize().as_slice());
859 let verified = checksum == expected;
860
861 if !verified {
862 println!(
863 "invalid checksum: \n\
864 found: {checksum}\n\
865 expected: {expected}",
866 );
867 }
868
869 verified
870}
871
872fn unpack(exec_ctx: &ExecutionContext, tarball: &Path, dst: &Path, pattern: &str) {
873 eprintln!("extracting {} to {}", tarball.display(), dst.display());
874 if !dst.exists() {
875 t!(fs::create_dir_all(dst));
876 }
877
878 let uncompressed_filename =
881 Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
882 let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
883
884 let data = t!(File::open(tarball), format!("file {} not found", tarball.display()));
886 let decompressor = XzDecoder::new(BufReader::new(data));
887
888 let mut tar = tar::Archive::new(decompressor);
889
890 let is_ci_rustc = dst.ends_with("ci-rustc");
891 let is_ci_llvm = dst.ends_with("ci-llvm");
892
893 let mut recorded_entries = if is_ci_rustc { recorded_entries(dst, pattern) } else { None };
897
898 for member in t!(tar.entries()) {
899 let mut member = t!(member);
900 let original_path = t!(member.path()).into_owned();
901 if original_path == directory_prefix {
903 continue;
904 }
905 let mut short_path = t!(original_path.strip_prefix(directory_prefix));
906 let is_builder_config = short_path.to_str() == Some(BUILDER_CONFIG_FILENAME);
907
908 if !(short_path.starts_with(pattern) || ((is_ci_rustc || is_ci_llvm) && is_builder_config))
909 {
910 continue;
911 }
912 short_path = short_path.strip_prefix(pattern).unwrap_or(short_path);
913 let dst_path = dst.join(short_path);
914
915 exec_ctx.verbose(|| {
916 println!("extracting {} to {}", original_path.display(), dst.display());
917 });
918
919 if !t!(member.unpack_in(dst)) {
920 panic!("path traversal attack ??");
921 }
922 if let Some(record) = &mut recorded_entries {
923 t!(writeln!(record, "{}", short_path.to_str().unwrap()));
924 }
925 let src_path = dst.join(original_path);
926 if src_path.is_dir() && dst_path.exists() {
927 continue;
928 }
929 t!(move_file(src_path, dst_path));
930 }
931 let dst_dir = dst.join(directory_prefix);
932 if dst_dir.exists() {
933 t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display()));
934 }
935}
936
937fn download_file<'a>(
938 dwn_ctx: impl AsRef<DownloadContext<'a>>,
939 url: &str,
940 dest_path: &Path,
941 help_on_error: &str,
942) {
943 let dwn_ctx = dwn_ctx.as_ref();
944
945 dwn_ctx.exec_ctx.verbose(|| {
946 println!("download {url}");
947 });
948 let tempfile = tempdir(dwn_ctx.out).join(dest_path.file_name().unwrap());
950 match url.split_once("://").map(|(proto, _)| proto) {
954 Some("http") | Some("https") => download_http_with_retries(
955 dwn_ctx.host_target,
956 dwn_ctx.is_running_on_ci,
957 dwn_ctx.exec_ctx,
958 &tempfile,
959 url,
960 help_on_error,
961 ),
962 Some(other) => panic!("unsupported protocol {other} in {url}"),
963 None => panic!("no protocol in {url}"),
964 }
965 t!(move_file(&tempfile, dest_path), format!("failed to rename {tempfile:?} to {dest_path:?}"));
966}
967
968pub(crate) fn tempdir(out: &Path) -> PathBuf {
973 let tmp = out.join("tmp");
974 t!(fs::create_dir_all(&tmp));
975 tmp
976}
977
978fn download_http_with_retries(
979 host_target: TargetSelection,
980 is_running_on_ci: bool,
981 exec_ctx: &ExecutionContext,
982 tempfile: &Path,
983 url: &str,
984 help_on_error: &str,
985) {
986 println!("downloading {url}");
987 let mut curl = command("curl").allow_failure();
992 curl.args([
993 "--location",
995 "--speed-time",
997 "30",
998 "--speed-limit",
999 "10",
1000 "--connect-timeout",
1002 "30",
1003 "--output",
1005 tempfile.to_str().unwrap(),
1006 "--continue-at",
1009 "-",
1010 "--retry",
1013 "3",
1014 "--show-error",
1016 "--remote-time",
1018 "--fail",
1020 ]);
1021 if is_running_on_ci {
1023 curl.arg("--silent");
1024 } else {
1025 curl.arg("--progress-bar");
1026 }
1027 if curl_version(exec_ctx) >= semver::Version::new(7, 71, 0) {
1029 curl.arg("--retry-all-errors");
1030 }
1031 curl.arg(url);
1032 if !curl.run(exec_ctx) {
1033 if host_target.contains("windows-msvc") {
1034 eprintln!("Fallback to PowerShell");
1035 for _ in 0..3 {
1036 let powershell = command("PowerShell.exe").allow_failure().args([
1037 "/nologo",
1038 "-Command",
1039 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
1040 &format!(
1041 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
1042 url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
1043 ),
1044 ]).run_capture_stdout(exec_ctx);
1045
1046 if powershell.is_failure() {
1047 return;
1048 }
1049
1050 eprintln!("\nspurious failure, trying again");
1051 }
1052 }
1053 if !help_on_error.is_empty() {
1054 eprintln!("{help_on_error}");
1055 }
1056 crate::exit!(1);
1057 }
1058}
1059
1060fn curl_version(exec_ctx: &ExecutionContext) -> semver::Version {
1061 let mut curl = command("curl");
1062 curl.arg("-V");
1063 let curl = curl.run_capture_stdout(exec_ctx);
1064 if curl.is_failure() {
1065 return semver::Version::new(1, 0, 0);
1066 }
1067 let output = curl.stdout();
1068 extract_curl_version(output)
1069}