bootstrap/utils/
tarball.rs

1//! Facilitates the management and generation of tarballs.
2//!
3//! Tarballs efficiently hold Rust compiler build artifacts and
4//! capture a snapshot of each bootstrap stage.
5//! In uplifting, a tarball from Stage N captures essential components
6//! to assemble Stage N + 1 compiler.
7
8use std::path::{Path, PathBuf};
9
10use crate::FileType;
11use crate::core::build_steps::dist::distdir;
12use crate::core::builder::{Builder, Kind};
13use crate::core::config::BUILDER_CONFIG_FILENAME;
14use crate::utils::exec::BootstrapCommand;
15use crate::utils::helpers::{move_file, t};
16use crate::utils::{channel, helpers};
17
18#[derive(Copy, Clone)]
19pub(crate) enum OverlayKind {
20    Rust,
21    Llvm,
22    Cargo,
23    Clippy,
24    Miri,
25    Rustfmt,
26    RustAnalyzer,
27    RustcCodegenCranelift,
28    RustcCodegenGcc,
29    Gcc,
30    LlvmBitcodeLinker,
31    Enzyme,
32    Offload,
33}
34
35impl OverlayKind {
36    fn legal_and_readme(&self) -> &[&str] {
37        match self {
38            OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"],
39            OverlayKind::Llvm => {
40                &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"]
41            }
42            OverlayKind::Enzyme => &["src/tools/enzyme/LICENSE", "src/tools/enzyme/Readme.md"],
43            OverlayKind::Offload => {
44                &["src/llvm-project/openmp/LICENSE.TXT", "src/llvm-project/offload/README.md"]
45            }
46            OverlayKind::Cargo => &[
47                "src/tools/cargo/README.md",
48                "src/tools/cargo/LICENSE-MIT",
49                "src/tools/cargo/LICENSE-APACHE",
50                "src/tools/cargo/LICENSE-THIRD-PARTY",
51            ],
52            OverlayKind::Clippy => &[
53                "src/tools/clippy/README.md",
54                "src/tools/clippy/LICENSE-APACHE",
55                "src/tools/clippy/LICENSE-MIT",
56            ],
57            OverlayKind::Miri => &[
58                "src/tools/miri/README.md",
59                "src/tools/miri/LICENSE-APACHE",
60                "src/tools/miri/LICENSE-MIT",
61            ],
62            OverlayKind::Rustfmt => &[
63                "src/tools/rustfmt/README.md",
64                "src/tools/rustfmt/LICENSE-APACHE",
65                "src/tools/rustfmt/LICENSE-MIT",
66            ],
67            OverlayKind::RustAnalyzer => &[
68                "src/tools/rust-analyzer/README.md",
69                "src/tools/rust-analyzer/LICENSE-APACHE",
70                "src/tools/rust-analyzer/LICENSE-MIT",
71            ],
72            OverlayKind::RustcCodegenCranelift => &[
73                "compiler/rustc_codegen_cranelift/Readme.md",
74                "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
75                "compiler/rustc_codegen_cranelift/LICENSE-MIT",
76            ],
77            OverlayKind::RustcCodegenGcc => &[
78                "compiler/rustc_codegen_gcc/Readme.md",
79                "compiler/rustc_codegen_gcc/LICENSE-APACHE",
80                "compiler/rustc_codegen_gcc/LICENSE-MIT",
81            ],
82            OverlayKind::LlvmBitcodeLinker => &[
83                "COPYRIGHT",
84                "LICENSE-APACHE",
85                "LICENSE-MIT",
86                "src/tools/llvm-bitcode-linker/README.md",
87            ],
88            OverlayKind::Gcc => &[
89                "src/gcc/README",
90                "src/gcc/COPYING",
91                "src/gcc/COPYING.LIB",
92                "src/gcc/COPYING.RUNTIME",
93                "src/gcc/COPYING3",
94                "src/gcc/COPYING3.LIB",
95            ],
96        }
97    }
98
99    fn version(&self, builder: &Builder<'_>) -> String {
100        match self {
101            OverlayKind::Rust => builder.rust_version(),
102            OverlayKind::Llvm => builder.rust_version(),
103            OverlayKind::Cargo => {
104                builder.cargo_info.version(builder, &builder.release_num("cargo"))
105            }
106            OverlayKind::Clippy => {
107                builder.clippy_info.version(builder, &builder.release_num("clippy"))
108            }
109            OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")),
110            OverlayKind::Rustfmt => {
111                builder.rustfmt_info.version(builder, &builder.release_num("rustfmt"))
112            }
113            OverlayKind::RustAnalyzer => builder
114                .rust_analyzer_info
115                .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
116            OverlayKind::RustcCodegenCranelift => builder.rust_version(),
117            OverlayKind::RustcCodegenGcc => builder.rust_version(),
118            OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
119            OverlayKind::Gcc => builder.rust_version(),
120            OverlayKind::Enzyme => builder.rust_version(),
121            OverlayKind::Offload => builder.rust_version(),
122        }
123    }
124}
125
126pub(crate) struct Tarball<'a> {
127    builder: &'a Builder<'a>,
128
129    pkgname: String,
130    component: String,
131    target: Option<String>,
132    product_name: String,
133    overlay: OverlayKind,
134
135    temp_dir: PathBuf,
136    image_dir: PathBuf,
137    overlay_dir: PathBuf,
138    bulk_dirs: Vec<PathBuf>,
139
140    include_target_in_component_name: bool,
141    is_preview: bool,
142    permit_symlinks: bool,
143}
144
145impl<'a> Tarball<'a> {
146    pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self {
147        Self::new_inner(builder, component, Some(target.into()))
148    }
149
150    pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self {
151        Self::new_inner(builder, component, None)
152    }
153
154    fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self {
155        let pkgname = crate::core::build_steps::dist::pkgname(builder, component);
156
157        let mut temp_dir = builder.out.join("tmp").join("tarball").join(component);
158        if let Some(target) = &target {
159            temp_dir = temp_dir.join(target);
160        }
161        let _ = std::fs::remove_dir_all(&temp_dir);
162
163        let image_dir = temp_dir.join("image");
164        let overlay_dir = temp_dir.join("overlay");
165
166        Self {
167            builder,
168
169            pkgname,
170            component: component.into(),
171            target,
172            product_name: "Rust".into(),
173            overlay: OverlayKind::Rust,
174
175            temp_dir,
176            image_dir,
177            overlay_dir,
178            bulk_dirs: Vec::new(),
179
180            include_target_in_component_name: false,
181            is_preview: false,
182            permit_symlinks: false,
183        }
184    }
185
186    pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) {
187        self.overlay = overlay;
188    }
189
190    pub(crate) fn set_product_name(&mut self, name: &str) {
191        self.product_name = name.into();
192    }
193
194    pub(crate) fn include_target_in_component_name(&mut self, include: bool) {
195        self.include_target_in_component_name = include;
196    }
197
198    pub(crate) fn is_preview(&mut self, is: bool) {
199        self.is_preview = is;
200    }
201
202    pub(crate) fn permit_symlinks(&mut self, flag: bool) {
203        self.permit_symlinks = flag;
204    }
205
206    pub(crate) fn image_dir(&self) -> &Path {
207        t!(std::fs::create_dir_all(&self.image_dir));
208        &self.image_dir
209    }
210
211    pub(crate) fn add_file(
212        &self,
213        src: impl AsRef<Path>,
214        destdir: impl AsRef<Path>,
215        file_type: FileType,
216    ) {
217        // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply
218        // uses the base directory as the destination directory.
219        let destdir = if destdir.as_ref() == Path::new(".") {
220            self.image_dir.clone()
221        } else {
222            self.image_dir.join(destdir.as_ref())
223        };
224
225        t!(std::fs::create_dir_all(&destdir));
226        self.builder.install(src.as_ref(), &destdir, file_type);
227    }
228
229    pub(crate) fn add_renamed_file(
230        &self,
231        src: impl AsRef<Path>,
232        destdir: impl AsRef<Path>,
233        new_name: &str,
234        file_type: FileType,
235    ) {
236        let destdir = self.image_dir.join(destdir.as_ref());
237        t!(std::fs::create_dir_all(&destdir));
238        self.builder.copy_link(src.as_ref(), &destdir.join(new_name), file_type);
239    }
240
241    pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) {
242        for file in self.overlay.legal_and_readme() {
243            self.add_file(self.builder.src.join(file), destdir.as_ref(), FileType::Regular);
244        }
245    }
246
247    pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
248        let dest = self.image_dir.join(dest.as_ref());
249
250        t!(std::fs::create_dir_all(&dest));
251        self.builder.cp_link_r(src.as_ref(), &dest);
252    }
253
254    pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
255        self.bulk_dirs.push(dest.as_ref().to_path_buf());
256        self.add_dir(src, dest);
257    }
258
259    pub(crate) fn generate(self) -> GeneratedTarball {
260        let mut component_name = self.component.clone();
261        if self.is_preview {
262            component_name.push_str("-preview");
263        }
264        if self.include_target_in_component_name {
265            component_name.push('-');
266            component_name.push_str(
267                self.target
268                    .as_ref()
269                    .expect("include_target_in_component_name used in a targetless tarball"),
270            );
271        }
272
273        self.run(|this, cmd| {
274            cmd.arg("generate")
275                .arg("--image-dir")
276                .arg(&this.image_dir)
277                .arg(format!("--component-name={component_name}"));
278
279            if let Some((dir, dirs)) = this.bulk_dirs.split_first() {
280                let mut arg = dir.as_os_str().to_os_string();
281                for dir in dirs {
282                    arg.push(",");
283                    arg.push(dir);
284                }
285                cmd.arg("--bulk-dirs").arg(&arg);
286            }
287
288            this.non_bare_args(cmd);
289        })
290    }
291
292    pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball {
293        let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string();
294        for tarball in &tarballs[1..] {
295            input_tarballs.push(",");
296            input_tarballs.push(&tarball.path);
297        }
298
299        self.run(|this, cmd| {
300            cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs);
301            this.non_bare_args(cmd);
302        })
303    }
304
305    pub(crate) fn bare(self) -> GeneratedTarball {
306        // Bare tarballs should have the top level directory match the package
307        // name, not "image". We rename the image directory just before passing
308        // into rust-installer.
309        let dest = self.temp_dir.join(self.package_name());
310        t!(move_file(&self.image_dir, &dest));
311
312        self.run(|this, cmd| {
313            let distdir = distdir(this.builder);
314            t!(std::fs::create_dir_all(&distdir));
315            cmd.arg("tarball")
316                .arg("--input")
317                .arg(&dest)
318                .arg("--output")
319                .arg(distdir.join(this.package_name()));
320        })
321    }
322
323    fn package_name(&self) -> String {
324        if let Some(target) = &self.target {
325            format!("{}-{}", self.pkgname, target)
326        } else {
327            self.pkgname.clone()
328        }
329    }
330
331    fn non_bare_args(&self, cmd: &mut BootstrapCommand) {
332        cmd.arg("--rel-manifest-dir=rustlib")
333            .arg("--legacy-manifest-dirs=rustlib,cargo")
334            .arg(format!("--product-name={}", self.product_name))
335            .arg(format!("--success-message={} installed.", self.component))
336            .arg(format!("--package-name={}", self.package_name()))
337            .arg("--non-installed-overlay")
338            .arg(&self.overlay_dir)
339            .arg("--output-dir")
340            .arg(distdir(self.builder));
341    }
342
343    fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut BootstrapCommand)) -> GeneratedTarball {
344        t!(std::fs::create_dir_all(&self.overlay_dir));
345        self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
346        if let Some(info) = self.builder.rust_info().info() {
347            channel::write_commit_hash_file(&self.overlay_dir, &info.sha);
348            channel::write_commit_info_file(&self.overlay_dir, info);
349        }
350
351        // Add config file if present.
352        if let Some(config) = &self.builder.config.config {
353            self.add_renamed_file(
354                config,
355                &self.overlay_dir,
356                BUILDER_CONFIG_FILENAME,
357                FileType::Regular,
358            );
359        }
360
361        for file in self.overlay.legal_and_readme() {
362            self.builder.install(
363                &self.builder.src.join(file),
364                &self.overlay_dir,
365                FileType::Regular,
366            );
367        }
368
369        let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller);
370
371        let package_name = self.package_name();
372        self.builder.info(&format!("Dist {package_name}"));
373        let _time = crate::utils::helpers::timeit(self.builder);
374
375        build_cli(&self, &mut cmd);
376        cmd.arg("--work-dir").arg(&self.temp_dir);
377        if let Some(formats) = &self.builder.config.dist_compression_formats {
378            assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
379            cmd.arg("--compression-formats").arg(formats.join(","));
380        }
381
382        // For `x install` tarball files aren't needed, so we can speed up the process by not producing them.
383        let compression_profile = if self.builder.kind == Kind::Install {
384            self.builder.do_if_verbose(|| {
385                println!("Forcing dist.compression-profile = 'no-op' for `x install`.")
386            });
387            // "no-op" indicates that the rust-installer won't produce compressed tarball sources.
388            "no-op"
389        } else {
390            assert!(
391                self.builder.config.dist_compression_profile != "no-op",
392                "dist.compression-profile = 'no-op' can only be used for `x install`"
393            );
394
395            &self.builder.config.dist_compression_profile
396        };
397
398        cmd.args(["--compression-profile", compression_profile]);
399
400        // We want to use a pinned modification time for files in the archive
401        // to achieve better reproducibility. However, using the same mtime for all
402        // releases is not ideal, because it can break e.g. Cargo mtime checking
403        // (https://github.com/rust-lang/rust/issues/125578).
404        // Therefore, we set mtime to the date of the latest commit (if we're managed
405        // by git). In this way, the archive will still be always the same for a given commit
406        // (achieving reproducibility), but it will also change between different commits and
407        // Rust versions, so that it won't break mtime-based caches.
408        //
409        // Note that this only overrides the mtime of files, not directories, due to the
410        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
411
412        // Get the UTC timestamp of the last git commit, if we're under git.
413        // We need to use UTC, so that anyone who tries to rebuild from the same commit
414        // gets the same timestamp.
415        if self.builder.rust_info().is_managed_git_subrepository() {
416            // %ct means committer date
417            let timestamp = helpers::git(Some(&self.builder.src))
418                .arg("log")
419                .arg("-1")
420                .arg("--format=%ct")
421                .run_capture_stdout(self.builder)
422                .stdout();
423            cmd.args(["--override-file-mtime", timestamp.trim()]);
424        }
425
426        cmd.run(self.builder);
427
428        // Ensure there are no symbolic links in the tarball. In particular,
429        // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links.
430        let decompressed_output = self.temp_dir.join(&package_name);
431        if !self.builder.config.dry_run() && !self.permit_symlinks {
432            for entry in walkdir::WalkDir::new(&decompressed_output) {
433                let entry = t!(entry);
434                if entry.path_is_symlink() {
435                    panic!("generated a symlink in a tarball: {}", entry.path().display());
436                }
437            }
438        }
439
440        // Use either the first compression format defined, or "gz" as the default.
441        let ext = self
442            .builder
443            .config
444            .dist_compression_formats
445            .as_ref()
446            .and_then(|formats| formats.first())
447            .map(|s| s.as_str())
448            .unwrap_or("gz");
449
450        GeneratedTarball {
451            path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")),
452            decompressed_output,
453            work: self.temp_dir,
454        }
455    }
456}
457
458#[derive(Debug, Clone)]
459pub struct GeneratedTarball {
460    path: PathBuf,
461    decompressed_output: PathBuf,
462    work: PathBuf,
463}
464
465impl GeneratedTarball {
466    pub(crate) fn tarball(&self) -> &Path {
467        &self.path
468    }
469
470    pub(crate) fn decompressed_output(&self) -> &Path {
471        &self.decompressed_output
472    }
473
474    pub(crate) fn work_dir(&self) -> &Path {
475        &self.work
476    }
477}
bootstrap/utils/tarball.rs

bootstrap/utils/
tarball.rs
