bootstrap/utils/
tarball.rs

1//! Facilitates the management and generation of tarballs.
2//!
3//! Tarballs efficiently hold Rust compiler build artifacts and
4//! capture a snapshot of each bootstrap stage.
5//! In uplifting, a tarball from Stage N captures essential components
6//! to assemble Stage N + 1 compiler.
7
8use std::path::{Path, PathBuf};
9
10use crate::FileType;
11use crate::core::build_steps::dist::distdir;
12use crate::core::builder::{Builder, Kind};
13use crate::core::config::BUILDER_CONFIG_FILENAME;
14use crate::utils::exec::BootstrapCommand;
15use crate::utils::helpers::{move_file, t};
16use crate::utils::{channel, helpers};
17
18#[derive(Copy, Clone)]
19pub(crate) enum OverlayKind {
20    Rust,
21    Llvm,
22    Cargo,
23    Clippy,
24    Miri,
25    Rustfmt,
26    RustAnalyzer,
27    RustcCodegenCranelift,
28    RustcCodegenGcc,
29    Gcc,
30    LlvmBitcodeLinker,
31    Enzyme,
32}
33
34impl OverlayKind {
35    fn legal_and_readme(&self) -> &[&str] {
36        match self {
37            OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"],
38            OverlayKind::Llvm => {
39                &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"]
40            }
41            OverlayKind::Enzyme => &["src/tools/enzyme/LICENSE", "src/tools/enzyme/Readme.md"],
42            OverlayKind::Cargo => &[
43                "src/tools/cargo/README.md",
44                "src/tools/cargo/LICENSE-MIT",
45                "src/tools/cargo/LICENSE-APACHE",
46                "src/tools/cargo/LICENSE-THIRD-PARTY",
47            ],
48            OverlayKind::Clippy => &[
49                "src/tools/clippy/README.md",
50                "src/tools/clippy/LICENSE-APACHE",
51                "src/tools/clippy/LICENSE-MIT",
52            ],
53            OverlayKind::Miri => &[
54                "src/tools/miri/README.md",
55                "src/tools/miri/LICENSE-APACHE",
56                "src/tools/miri/LICENSE-MIT",
57            ],
58            OverlayKind::Rustfmt => &[
59                "src/tools/rustfmt/README.md",
60                "src/tools/rustfmt/LICENSE-APACHE",
61                "src/tools/rustfmt/LICENSE-MIT",
62            ],
63            OverlayKind::RustAnalyzer => &[
64                "src/tools/rust-analyzer/README.md",
65                "src/tools/rust-analyzer/LICENSE-APACHE",
66                "src/tools/rust-analyzer/LICENSE-MIT",
67            ],
68            OverlayKind::RustcCodegenCranelift => &[
69                "compiler/rustc_codegen_cranelift/Readme.md",
70                "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
71                "compiler/rustc_codegen_cranelift/LICENSE-MIT",
72            ],
73            OverlayKind::RustcCodegenGcc => &[
74                "compiler/rustc_codegen_gcc/Readme.md",
75                "compiler/rustc_codegen_gcc/LICENSE-APACHE",
76                "compiler/rustc_codegen_gcc/LICENSE-MIT",
77            ],
78            OverlayKind::LlvmBitcodeLinker => &[
79                "COPYRIGHT",
80                "LICENSE-APACHE",
81                "LICENSE-MIT",
82                "src/tools/llvm-bitcode-linker/README.md",
83            ],
84            OverlayKind::Gcc => &[
85                "src/gcc/README",
86                "src/gcc/COPYING",
87                "src/gcc/COPYING.LIB",
88                "src/gcc/COPYING.RUNTIME",
89                "src/gcc/COPYING3",
90                "src/gcc/COPYING3.LIB",
91            ],
92        }
93    }
94
95    fn version(&self, builder: &Builder<'_>) -> String {
96        match self {
97            OverlayKind::Rust => builder.rust_version(),
98            OverlayKind::Llvm => builder.rust_version(),
99            OverlayKind::Cargo => {
100                builder.cargo_info.version(builder, &builder.release_num("cargo"))
101            }
102            OverlayKind::Clippy => {
103                builder.clippy_info.version(builder, &builder.release_num("clippy"))
104            }
105            OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")),
106            OverlayKind::Rustfmt => {
107                builder.rustfmt_info.version(builder, &builder.release_num("rustfmt"))
108            }
109            OverlayKind::RustAnalyzer => builder
110                .rust_analyzer_info
111                .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
112            OverlayKind::RustcCodegenCranelift => builder.rust_version(),
113            OverlayKind::RustcCodegenGcc => builder.rust_version(),
114            OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
115            OverlayKind::Gcc => builder.rust_version(),
116            OverlayKind::Enzyme => builder.rust_version(),
117        }
118    }
119}
120
121pub(crate) struct Tarball<'a> {
122    builder: &'a Builder<'a>,
123
124    pkgname: String,
125    component: String,
126    target: Option<String>,
127    product_name: String,
128    overlay: OverlayKind,
129
130    temp_dir: PathBuf,
131    image_dir: PathBuf,
132    overlay_dir: PathBuf,
133    bulk_dirs: Vec<PathBuf>,
134
135    include_target_in_component_name: bool,
136    is_preview: bool,
137    permit_symlinks: bool,
138}
139
140impl<'a> Tarball<'a> {
141    pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self {
142        Self::new_inner(builder, component, Some(target.into()))
143    }
144
145    pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self {
146        Self::new_inner(builder, component, None)
147    }
148
149    fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self {
150        let pkgname = crate::core::build_steps::dist::pkgname(builder, component);
151
152        let mut temp_dir = builder.out.join("tmp").join("tarball").join(component);
153        if let Some(target) = &target {
154            temp_dir = temp_dir.join(target);
155        }
156        let _ = std::fs::remove_dir_all(&temp_dir);
157
158        let image_dir = temp_dir.join("image");
159        let overlay_dir = temp_dir.join("overlay");
160
161        Self {
162            builder,
163
164            pkgname,
165            component: component.into(),
166            target,
167            product_name: "Rust".into(),
168            overlay: OverlayKind::Rust,
169
170            temp_dir,
171            image_dir,
172            overlay_dir,
173            bulk_dirs: Vec::new(),
174
175            include_target_in_component_name: false,
176            is_preview: false,
177            permit_symlinks: false,
178        }
179    }
180
181    pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) {
182        self.overlay = overlay;
183    }
184
185    pub(crate) fn set_product_name(&mut self, name: &str) {
186        self.product_name = name.into();
187    }
188
189    pub(crate) fn include_target_in_component_name(&mut self, include: bool) {
190        self.include_target_in_component_name = include;
191    }
192
193    pub(crate) fn is_preview(&mut self, is: bool) {
194        self.is_preview = is;
195    }
196
197    pub(crate) fn permit_symlinks(&mut self, flag: bool) {
198        self.permit_symlinks = flag;
199    }
200
201    pub(crate) fn image_dir(&self) -> &Path {
202        t!(std::fs::create_dir_all(&self.image_dir));
203        &self.image_dir
204    }
205
206    pub(crate) fn add_file(
207        &self,
208        src: impl AsRef<Path>,
209        destdir: impl AsRef<Path>,
210        file_type: FileType,
211    ) {
212        // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply
213        // uses the base directory as the destination directory.
214        let destdir = if destdir.as_ref() == Path::new(".") {
215            self.image_dir.clone()
216        } else {
217            self.image_dir.join(destdir.as_ref())
218        };
219
220        t!(std::fs::create_dir_all(&destdir));
221        self.builder.install(src.as_ref(), &destdir, file_type);
222    }
223
224    pub(crate) fn add_renamed_file(
225        &self,
226        src: impl AsRef<Path>,
227        destdir: impl AsRef<Path>,
228        new_name: &str,
229        file_type: FileType,
230    ) {
231        let destdir = self.image_dir.join(destdir.as_ref());
232        t!(std::fs::create_dir_all(&destdir));
233        self.builder.copy_link(src.as_ref(), &destdir.join(new_name), file_type);
234    }
235
236    pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) {
237        for file in self.overlay.legal_and_readme() {
238            self.add_file(self.builder.src.join(file), destdir.as_ref(), FileType::Regular);
239        }
240    }
241
242    pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
243        let dest = self.image_dir.join(dest.as_ref());
244
245        t!(std::fs::create_dir_all(&dest));
246        self.builder.cp_link_r(src.as_ref(), &dest);
247    }
248
249    pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
250        self.bulk_dirs.push(dest.as_ref().to_path_buf());
251        self.add_dir(src, dest);
252    }
253
254    pub(crate) fn generate(self) -> GeneratedTarball {
255        let mut component_name = self.component.clone();
256        if self.is_preview {
257            component_name.push_str("-preview");
258        }
259        if self.include_target_in_component_name {
260            component_name.push('-');
261            component_name.push_str(
262                self.target
263                    .as_ref()
264                    .expect("include_target_in_component_name used in a targetless tarball"),
265            );
266        }
267
268        self.run(|this, cmd| {
269            cmd.arg("generate")
270                .arg("--image-dir")
271                .arg(&this.image_dir)
272                .arg(format!("--component-name={component_name}"));
273
274            if let Some((dir, dirs)) = this.bulk_dirs.split_first() {
275                let mut arg = dir.as_os_str().to_os_string();
276                for dir in dirs {
277                    arg.push(",");
278                    arg.push(dir);
279                }
280                cmd.arg("--bulk-dirs").arg(&arg);
281            }
282
283            this.non_bare_args(cmd);
284        })
285    }
286
287    pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball {
288        let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string();
289        for tarball in &tarballs[1..] {
290            input_tarballs.push(",");
291            input_tarballs.push(&tarball.path);
292        }
293
294        self.run(|this, cmd| {
295            cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs);
296            this.non_bare_args(cmd);
297        })
298    }
299
300    pub(crate) fn bare(self) -> GeneratedTarball {
301        // Bare tarballs should have the top level directory match the package
302        // name, not "image". We rename the image directory just before passing
303        // into rust-installer.
304        let dest = self.temp_dir.join(self.package_name());
305        t!(move_file(&self.image_dir, &dest));
306
307        self.run(|this, cmd| {
308            let distdir = distdir(this.builder);
309            t!(std::fs::create_dir_all(&distdir));
310            cmd.arg("tarball")
311                .arg("--input")
312                .arg(&dest)
313                .arg("--output")
314                .arg(distdir.join(this.package_name()));
315        })
316    }
317
318    fn package_name(&self) -> String {
319        if let Some(target) = &self.target {
320            format!("{}-{}", self.pkgname, target)
321        } else {
322            self.pkgname.clone()
323        }
324    }
325
326    fn non_bare_args(&self, cmd: &mut BootstrapCommand) {
327        cmd.arg("--rel-manifest-dir=rustlib")
328            .arg("--legacy-manifest-dirs=rustlib,cargo")
329            .arg(format!("--product-name={}", self.product_name))
330            .arg(format!("--success-message={} installed.", self.component))
331            .arg(format!("--package-name={}", self.package_name()))
332            .arg("--non-installed-overlay")
333            .arg(&self.overlay_dir)
334            .arg("--output-dir")
335            .arg(distdir(self.builder));
336    }
337
338    fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut BootstrapCommand)) -> GeneratedTarball {
339        t!(std::fs::create_dir_all(&self.overlay_dir));
340        self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
341        if let Some(info) = self.builder.rust_info().info() {
342            channel::write_commit_hash_file(&self.overlay_dir, &info.sha);
343            channel::write_commit_info_file(&self.overlay_dir, info);
344        }
345
346        // Add config file if present.
347        if let Some(config) = &self.builder.config.config {
348            self.add_renamed_file(
349                config,
350                &self.overlay_dir,
351                BUILDER_CONFIG_FILENAME,
352                FileType::Regular,
353            );
354        }
355
356        for file in self.overlay.legal_and_readme() {
357            self.builder.install(
358                &self.builder.src.join(file),
359                &self.overlay_dir,
360                FileType::Regular,
361            );
362        }
363
364        let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller);
365
366        let package_name = self.package_name();
367        self.builder.info(&format!("Dist {package_name}"));
368        let _time = crate::utils::helpers::timeit(self.builder);
369
370        build_cli(&self, &mut cmd);
371        cmd.arg("--work-dir").arg(&self.temp_dir);
372        if let Some(formats) = &self.builder.config.dist_compression_formats {
373            assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
374            cmd.arg("--compression-formats").arg(formats.join(","));
375        }
376
377        // For `x install` tarball files aren't needed, so we can speed up the process by not producing them.
378        let compression_profile = if self.builder.kind == Kind::Install {
379            self.builder.do_if_verbose(|| {
380                println!("Forcing dist.compression-profile = 'no-op' for `x install`.")
381            });
382            // "no-op" indicates that the rust-installer won't produce compressed tarball sources.
383            "no-op"
384        } else {
385            assert!(
386                self.builder.config.dist_compression_profile != "no-op",
387                "dist.compression-profile = 'no-op' can only be used for `x install`"
388            );
389
390            &self.builder.config.dist_compression_profile
391        };
392
393        cmd.args(["--compression-profile", compression_profile]);
394
395        // We want to use a pinned modification time for files in the archive
396        // to achieve better reproducibility. However, using the same mtime for all
397        // releases is not ideal, because it can break e.g. Cargo mtime checking
398        // (https://github.com/rust-lang/rust/issues/125578).
399        // Therefore, we set mtime to the date of the latest commit (if we're managed
400        // by git). In this way, the archive will still be always the same for a given commit
401        // (achieving reproducibility), but it will also change between different commits and
402        // Rust versions, so that it won't break mtime-based caches.
403        //
404        // Note that this only overrides the mtime of files, not directories, due to the
405        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
406
407        // Get the UTC timestamp of the last git commit, if we're under git.
408        // We need to use UTC, so that anyone who tries to rebuild from the same commit
409        // gets the same timestamp.
410        if self.builder.rust_info().is_managed_git_subrepository() {
411            // %ct means committer date
412            let timestamp = helpers::git(Some(&self.builder.src))
413                .arg("log")
414                .arg("-1")
415                .arg("--format=%ct")
416                .run_capture_stdout(self.builder)
417                .stdout();
418            cmd.args(["--override-file-mtime", timestamp.trim()]);
419        }
420
421        cmd.run(self.builder);
422
423        // Ensure there are no symbolic links in the tarball. In particular,
424        // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links.
425        let decompressed_output = self.temp_dir.join(&package_name);
426        if !self.builder.config.dry_run() && !self.permit_symlinks {
427            for entry in walkdir::WalkDir::new(&decompressed_output) {
428                let entry = t!(entry);
429                if entry.path_is_symlink() {
430                    panic!("generated a symlink in a tarball: {}", entry.path().display());
431                }
432            }
433        }
434
435        // Use either the first compression format defined, or "gz" as the default.
436        let ext = self
437            .builder
438            .config
439            .dist_compression_formats
440            .as_ref()
441            .and_then(|formats| formats.first())
442            .map(|s| s.as_str())
443            .unwrap_or("gz");
444
445        GeneratedTarball {
446            path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")),
447            decompressed_output,
448            work: self.temp_dir,
449        }
450    }
451}
452
453#[derive(Debug, Clone)]
454pub struct GeneratedTarball {
455    path: PathBuf,
456    decompressed_output: PathBuf,
457    work: PathBuf,
458}
459
460impl GeneratedTarball {
461    pub(crate) fn tarball(&self) -> &Path {
462        &self.path
463    }
464
465    pub(crate) fn decompressed_output(&self) -> &Path {
466        &self.decompressed_output
467    }
468
469    pub(crate) fn work_dir(&self) -> &Path {
470        &self.work
471    }
472}
bootstrap/utils/tarball.rs

bootstrap/utils/
tarball.rs
