bootstrap/utils/
tarball.rs

1//! Facilitates the management and generation of tarballs.
2//!
3//! Tarballs efficiently hold Rust compiler build artifacts and
4//! capture a snapshot of each bootstrap stage.
5//! In uplifting, a tarball from Stage N captures essential components
6//! to assemble Stage N + 1 compiler.
7
8use std::path::{Path, PathBuf};
9
10use crate::FileType;
11use crate::core::build_steps::dist::distdir;
12use crate::core::builder::{Builder, Kind};
13use crate::core::config::BUILDER_CONFIG_FILENAME;
14use crate::utils::exec::BootstrapCommand;
15use crate::utils::helpers::{move_file, t};
16use crate::utils::{channel, helpers};
17
18#[derive(Copy, Clone)]
19pub(crate) enum OverlayKind {
20    Rust,
21    Llvm,
22    Cargo,
23    Clippy,
24    Miri,
25    Rustfmt,
26    RustAnalyzer,
27    RustcCodegenCranelift,
28    RustcCodegenGcc,
29    LlvmBitcodeLinker,
30}
31
32impl OverlayKind {
33    fn legal_and_readme(&self) -> &[&str] {
34        match self {
35            OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"],
36            OverlayKind::Llvm => {
37                &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"]
38            }
39            OverlayKind::Cargo => &[
40                "src/tools/cargo/README.md",
41                "src/tools/cargo/LICENSE-MIT",
42                "src/tools/cargo/LICENSE-APACHE",
43                "src/tools/cargo/LICENSE-THIRD-PARTY",
44            ],
45            OverlayKind::Clippy => &[
46                "src/tools/clippy/README.md",
47                "src/tools/clippy/LICENSE-APACHE",
48                "src/tools/clippy/LICENSE-MIT",
49            ],
50            OverlayKind::Miri => &[
51                "src/tools/miri/README.md",
52                "src/tools/miri/LICENSE-APACHE",
53                "src/tools/miri/LICENSE-MIT",
54            ],
55            OverlayKind::Rustfmt => &[
56                "src/tools/rustfmt/README.md",
57                "src/tools/rustfmt/LICENSE-APACHE",
58                "src/tools/rustfmt/LICENSE-MIT",
59            ],
60            OverlayKind::RustAnalyzer => &[
61                "src/tools/rust-analyzer/README.md",
62                "src/tools/rust-analyzer/LICENSE-APACHE",
63                "src/tools/rust-analyzer/LICENSE-MIT",
64            ],
65            OverlayKind::RustcCodegenCranelift => &[
66                "compiler/rustc_codegen_cranelift/Readme.md",
67                "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
68                "compiler/rustc_codegen_cranelift/LICENSE-MIT",
69            ],
70            OverlayKind::RustcCodegenGcc => &[
71                "compiler/rustc_codegen_gcc/Readme.md",
72                "compiler/rustc_codegen_gcc/LICENSE-APACHE",
73                "compiler/rustc_codegen_gcc/LICENSE-MIT",
74            ],
75            OverlayKind::LlvmBitcodeLinker => &[
76                "COPYRIGHT",
77                "LICENSE-APACHE",
78                "LICENSE-MIT",
79                "src/tools/llvm-bitcode-linker/README.md",
80            ],
81        }
82    }
83
84    fn version(&self, builder: &Builder<'_>) -> String {
85        match self {
86            OverlayKind::Rust => builder.rust_version(),
87            OverlayKind::Llvm => builder.rust_version(),
88            OverlayKind::Cargo => {
89                builder.cargo_info.version(builder, &builder.release_num("cargo"))
90            }
91            OverlayKind::Clippy => {
92                builder.clippy_info.version(builder, &builder.release_num("clippy"))
93            }
94            OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")),
95            OverlayKind::Rustfmt => {
96                builder.rustfmt_info.version(builder, &builder.release_num("rustfmt"))
97            }
98            OverlayKind::RustAnalyzer => builder
99                .rust_analyzer_info
100                .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
101            OverlayKind::RustcCodegenCranelift => builder.rust_version(),
102            OverlayKind::RustcCodegenGcc => builder.rust_version(),
103            OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
104        }
105    }
106}
107
108pub(crate) struct Tarball<'a> {
109    builder: &'a Builder<'a>,
110
111    pkgname: String,
112    component: String,
113    target: Option<String>,
114    product_name: String,
115    overlay: OverlayKind,
116
117    temp_dir: PathBuf,
118    image_dir: PathBuf,
119    overlay_dir: PathBuf,
120    bulk_dirs: Vec<PathBuf>,
121
122    include_target_in_component_name: bool,
123    is_preview: bool,
124    permit_symlinks: bool,
125}
126
127impl<'a> Tarball<'a> {
128    pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self {
129        Self::new_inner(builder, component, Some(target.into()))
130    }
131
132    pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self {
133        Self::new_inner(builder, component, None)
134    }
135
136    fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self {
137        let pkgname = crate::core::build_steps::dist::pkgname(builder, component);
138
139        let mut temp_dir = builder.out.join("tmp").join("tarball").join(component);
140        if let Some(target) = &target {
141            temp_dir = temp_dir.join(target);
142        }
143        let _ = std::fs::remove_dir_all(&temp_dir);
144
145        let image_dir = temp_dir.join("image");
146        let overlay_dir = temp_dir.join("overlay");
147
148        Self {
149            builder,
150
151            pkgname,
152            component: component.into(),
153            target,
154            product_name: "Rust".into(),
155            overlay: OverlayKind::Rust,
156
157            temp_dir,
158            image_dir,
159            overlay_dir,
160            bulk_dirs: Vec::new(),
161
162            include_target_in_component_name: false,
163            is_preview: false,
164            permit_symlinks: false,
165        }
166    }
167
168    pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) {
169        self.overlay = overlay;
170    }
171
172    pub(crate) fn set_product_name(&mut self, name: &str) {
173        self.product_name = name.into();
174    }
175
176    pub(crate) fn include_target_in_component_name(&mut self, include: bool) {
177        self.include_target_in_component_name = include;
178    }
179
180    pub(crate) fn is_preview(&mut self, is: bool) {
181        self.is_preview = is;
182    }
183
184    pub(crate) fn permit_symlinks(&mut self, flag: bool) {
185        self.permit_symlinks = flag;
186    }
187
188    pub(crate) fn image_dir(&self) -> &Path {
189        t!(std::fs::create_dir_all(&self.image_dir));
190        &self.image_dir
191    }
192
193    pub(crate) fn add_file(
194        &self,
195        src: impl AsRef<Path>,
196        destdir: impl AsRef<Path>,
197        file_type: FileType,
198    ) {
199        // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply
200        // uses the base directory as the destination directory.
201        let destdir = if destdir.as_ref() == Path::new(".") {
202            self.image_dir.clone()
203        } else {
204            self.image_dir.join(destdir.as_ref())
205        };
206
207        t!(std::fs::create_dir_all(&destdir));
208        self.builder.install(src.as_ref(), &destdir, file_type);
209    }
210
211    pub(crate) fn add_renamed_file(
212        &self,
213        src: impl AsRef<Path>,
214        destdir: impl AsRef<Path>,
215        new_name: &str,
216        file_type: FileType,
217    ) {
218        let destdir = self.image_dir.join(destdir.as_ref());
219        t!(std::fs::create_dir_all(&destdir));
220        self.builder.copy_link(src.as_ref(), &destdir.join(new_name), file_type);
221    }
222
223    pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) {
224        for file in self.overlay.legal_and_readme() {
225            self.add_file(self.builder.src.join(file), destdir.as_ref(), FileType::Regular);
226        }
227    }
228
229    pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
230        let dest = self.image_dir.join(dest.as_ref());
231
232        t!(std::fs::create_dir_all(&dest));
233        self.builder.cp_link_r(src.as_ref(), &dest);
234    }
235
236    pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
237        self.bulk_dirs.push(dest.as_ref().to_path_buf());
238        self.add_dir(src, dest);
239    }
240
241    pub(crate) fn generate(self) -> GeneratedTarball {
242        let mut component_name = self.component.clone();
243        if self.is_preview {
244            component_name.push_str("-preview");
245        }
246        if self.include_target_in_component_name {
247            component_name.push('-');
248            component_name.push_str(
249                self.target
250                    .as_ref()
251                    .expect("include_target_in_component_name used in a targetless tarball"),
252            );
253        }
254
255        self.run(|this, cmd| {
256            cmd.arg("generate")
257                .arg("--image-dir")
258                .arg(&this.image_dir)
259                .arg(format!("--component-name={component_name}"));
260
261            if let Some((dir, dirs)) = this.bulk_dirs.split_first() {
262                let mut arg = dir.as_os_str().to_os_string();
263                for dir in dirs {
264                    arg.push(",");
265                    arg.push(dir);
266                }
267                cmd.arg("--bulk-dirs").arg(&arg);
268            }
269
270            this.non_bare_args(cmd);
271        })
272    }
273
274    pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball {
275        let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string();
276        for tarball in &tarballs[1..] {
277            input_tarballs.push(",");
278            input_tarballs.push(&tarball.path);
279        }
280
281        self.run(|this, cmd| {
282            cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs);
283            this.non_bare_args(cmd);
284        })
285    }
286
287    pub(crate) fn bare(self) -> GeneratedTarball {
288        // Bare tarballs should have the top level directory match the package
289        // name, not "image". We rename the image directory just before passing
290        // into rust-installer.
291        let dest = self.temp_dir.join(self.package_name());
292        t!(move_file(&self.image_dir, &dest));
293
294        self.run(|this, cmd| {
295            let distdir = distdir(this.builder);
296            t!(std::fs::create_dir_all(&distdir));
297            cmd.arg("tarball")
298                .arg("--input")
299                .arg(&dest)
300                .arg("--output")
301                .arg(distdir.join(this.package_name()));
302        })
303    }
304
305    fn package_name(&self) -> String {
306        if let Some(target) = &self.target {
307            format!("{}-{}", self.pkgname, target)
308        } else {
309            self.pkgname.clone()
310        }
311    }
312
313    fn non_bare_args(&self, cmd: &mut BootstrapCommand) {
314        cmd.arg("--rel-manifest-dir=rustlib")
315            .arg("--legacy-manifest-dirs=rustlib,cargo")
316            .arg(format!("--product-name={}", self.product_name))
317            .arg(format!("--success-message={} installed.", self.component))
318            .arg(format!("--package-name={}", self.package_name()))
319            .arg("--non-installed-overlay")
320            .arg(&self.overlay_dir)
321            .arg("--output-dir")
322            .arg(distdir(self.builder));
323    }
324
325    fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut BootstrapCommand)) -> GeneratedTarball {
326        t!(std::fs::create_dir_all(&self.overlay_dir));
327        self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
328        if let Some(info) = self.builder.rust_info().info() {
329            channel::write_commit_hash_file(&self.overlay_dir, &info.sha);
330            channel::write_commit_info_file(&self.overlay_dir, info);
331        }
332
333        // Add config file if present.
334        if let Some(config) = &self.builder.config.config {
335            self.add_renamed_file(
336                config,
337                &self.overlay_dir,
338                BUILDER_CONFIG_FILENAME,
339                FileType::Regular,
340            );
341        }
342
343        for file in self.overlay.legal_and_readme() {
344            self.builder.install(
345                &self.builder.src.join(file),
346                &self.overlay_dir,
347                FileType::Regular,
348            );
349        }
350
351        let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller);
352
353        let package_name = self.package_name();
354        self.builder.info(&format!("Dist {package_name}"));
355        let _time = crate::utils::helpers::timeit(self.builder);
356
357        build_cli(&self, &mut cmd);
358        cmd.arg("--work-dir").arg(&self.temp_dir);
359        if let Some(formats) = &self.builder.config.dist_compression_formats {
360            assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
361            cmd.arg("--compression-formats").arg(formats.join(","));
362        }
363
364        // For `x install` tarball files aren't needed, so we can speed up the process by not producing them.
365        let compression_profile = if self.builder.kind == Kind::Install {
366            self.builder.do_if_verbose(|| {
367                println!("Forcing dist.compression-profile = 'no-op' for `x install`.")
368            });
369            // "no-op" indicates that the rust-installer won't produce compressed tarball sources.
370            "no-op"
371        } else {
372            assert!(
373                self.builder.config.dist_compression_profile != "no-op",
374                "dist.compression-profile = 'no-op' can only be used for `x install`"
375            );
376
377            &self.builder.config.dist_compression_profile
378        };
379
380        cmd.args(["--compression-profile", compression_profile]);
381
382        // We want to use a pinned modification time for files in the archive
383        // to achieve better reproducibility. However, using the same mtime for all
384        // releases is not ideal, because it can break e.g. Cargo mtime checking
385        // (https://github.com/rust-lang/rust/issues/125578).
386        // Therefore, we set mtime to the date of the latest commit (if we're managed
387        // by git). In this way, the archive will still be always the same for a given commit
388        // (achieving reproducibility), but it will also change between different commits and
389        // Rust versions, so that it won't break mtime-based caches.
390        //
391        // Note that this only overrides the mtime of files, not directories, due to the
392        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
393
394        // Get the UTC timestamp of the last git commit, if we're under git.
395        // We need to use UTC, so that anyone who tries to rebuild from the same commit
396        // gets the same timestamp.
397        if self.builder.rust_info().is_managed_git_subrepository() {
398            // %ct means committer date
399            let timestamp = helpers::git(Some(&self.builder.src))
400                .arg("log")
401                .arg("-1")
402                .arg("--format=%ct")
403                .run_capture_stdout(self.builder)
404                .stdout();
405            cmd.args(["--override-file-mtime", timestamp.trim()]);
406        }
407
408        cmd.run(self.builder);
409
410        // Ensure there are no symbolic links in the tarball. In particular,
411        // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links.
412        let decompressed_output = self.temp_dir.join(&package_name);
413        if !self.builder.config.dry_run() && !self.permit_symlinks {
414            for entry in walkdir::WalkDir::new(&decompressed_output) {
415                let entry = t!(entry);
416                if entry.path_is_symlink() {
417                    panic!("generated a symlink in a tarball: {}", entry.path().display());
418                }
419            }
420        }
421
422        // Use either the first compression format defined, or "gz" as the default.
423        let ext = self
424            .builder
425            .config
426            .dist_compression_formats
427            .as_ref()
428            .and_then(|formats| formats.first())
429            .map(|s| s.as_str())
430            .unwrap_or("gz");
431
432        GeneratedTarball {
433            path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")),
434            decompressed_output,
435            work: self.temp_dir,
436        }
437    }
438}
439
440#[derive(Debug, Clone)]
441pub struct GeneratedTarball {
442    path: PathBuf,
443    decompressed_output: PathBuf,
444    work: PathBuf,
445}
446
447impl GeneratedTarball {
448    pub(crate) fn tarball(&self) -> &Path {
449        &self.path
450    }
451
452    pub(crate) fn decompressed_output(&self) -> &Path {
453        &self.decompressed_output
454    }
455
456    pub(crate) fn work_dir(&self) -> &Path {
457        &self.work
458    }
459}
bootstrap/utils/tarball.rs

bootstrap/utils/
tarball.rs
