bootstrap/utils/
tarball.rs

Help
1//! Facilitates the management and generation of tarballs.
2//!
3//! Tarballs efficiently hold Rust compiler build artifacts and
4//! capture a snapshot of each bootstrap stage.
5//! In uplifting, a tarball from Stage N captures essential components
6//! to assemble Stage N + 1 compiler.
7
8use std::path::{Path, PathBuf};
9
10use crate::FileType;
11use crate::core::build_steps::dist::distdir;
12use crate::core::builder::{Builder, Kind};
13use crate::core::config::BUILDER_CONFIG_FILENAME;
14use crate::utils::exec::BootstrapCommand;
15use crate::utils::helpers::{move_file, t};
16use crate::utils::{channel, helpers};
17
18#[derive(Copy, Clone)]
19pub(crate) enum OverlayKind {
20    Rust,
21    Llvm,
22    Cargo,
23    Clippy,
24    Miri,
25    Rustfmt,
26    RustAnalyzer,
27    RustcCodegenCranelift,
28    LlvmBitcodeLinker,
29}
30
31impl OverlayKind {
32    fn legal_and_readme(&self) -> &[&str] {
33        match self {
34            OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"],
35            OverlayKind::Llvm => {
36                &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"]
37            }
38            OverlayKind::Cargo => &[
39                "src/tools/cargo/README.md",
40                "src/tools/cargo/LICENSE-MIT",
41                "src/tools/cargo/LICENSE-APACHE",
42                "src/tools/cargo/LICENSE-THIRD-PARTY",
43            ],
44            OverlayKind::Clippy => &[
45                "src/tools/clippy/README.md",
46                "src/tools/clippy/LICENSE-APACHE",
47                "src/tools/clippy/LICENSE-MIT",
48            ],
49            OverlayKind::Miri => &[
50                "src/tools/miri/README.md",
51                "src/tools/miri/LICENSE-APACHE",
52                "src/tools/miri/LICENSE-MIT",
53            ],
54            OverlayKind::Rustfmt => &[
55                "src/tools/rustfmt/README.md",
56                "src/tools/rustfmt/LICENSE-APACHE",
57                "src/tools/rustfmt/LICENSE-MIT",
58            ],
59            OverlayKind::RustAnalyzer => &[
60                "src/tools/rust-analyzer/README.md",
61                "src/tools/rust-analyzer/LICENSE-APACHE",
62                "src/tools/rust-analyzer/LICENSE-MIT",
63            ],
64            OverlayKind::RustcCodegenCranelift => &[
65                "compiler/rustc_codegen_cranelift/Readme.md",
66                "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
67                "compiler/rustc_codegen_cranelift/LICENSE-MIT",
68            ],
69            OverlayKind::LlvmBitcodeLinker => &[
70                "COPYRIGHT",
71                "LICENSE-APACHE",
72                "LICENSE-MIT",
73                "src/tools/llvm-bitcode-linker/README.md",
74            ],
75        }
76    }
77
78    fn version(&self, builder: &Builder<'_>) -> String {
79        match self {
80            OverlayKind::Rust => builder.rust_version(),
81            OverlayKind::Llvm => builder.rust_version(),
82            OverlayKind::Cargo => {
83                builder.cargo_info.version(builder, &builder.release_num("cargo"))
84            }
85            OverlayKind::Clippy => {
86                builder.clippy_info.version(builder, &builder.release_num("clippy"))
87            }
88            OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")),
89            OverlayKind::Rustfmt => {
90                builder.rustfmt_info.version(builder, &builder.release_num("rustfmt"))
91            }
92            OverlayKind::RustAnalyzer => builder
93                .rust_analyzer_info
94                .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
95            OverlayKind::RustcCodegenCranelift => builder.rust_version(),
96            OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
97        }
98    }
99}
100
101pub(crate) struct Tarball<'a> {
102    builder: &'a Builder<'a>,
103
104    pkgname: String,
105    component: String,
106    target: Option<String>,
107    product_name: String,
108    overlay: OverlayKind,
109
110    temp_dir: PathBuf,
111    image_dir: PathBuf,
112    overlay_dir: PathBuf,
113    bulk_dirs: Vec<PathBuf>,
114
115    include_target_in_component_name: bool,
116    is_preview: bool,
117    permit_symlinks: bool,
118}
119
120impl<'a> Tarball<'a> {
121    pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self {
122        Self::new_inner(builder, component, Some(target.into()))
123    }
124
125    pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self {
126        Self::new_inner(builder, component, None)
127    }
128
129    fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self {
130        let pkgname = crate::core::build_steps::dist::pkgname(builder, component);
131
132        let mut temp_dir = builder.out.join("tmp").join("tarball").join(component);
133        if let Some(target) = &target {
134            temp_dir = temp_dir.join(target);
135        }
136        let _ = std::fs::remove_dir_all(&temp_dir);
137
138        let image_dir = temp_dir.join("image");
139        let overlay_dir = temp_dir.join("overlay");
140
141        Self {
142            builder,
143
144            pkgname,
145            component: component.into(),
146            target,
147            product_name: "Rust".into(),
148            overlay: OverlayKind::Rust,
149
150            temp_dir,
151            image_dir,
152            overlay_dir,
153            bulk_dirs: Vec::new(),
154
155            include_target_in_component_name: false,
156            is_preview: false,
157            permit_symlinks: false,
158        }
159    }
160
161    pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) {
162        self.overlay = overlay;
163    }
164
165    pub(crate) fn set_product_name(&mut self, name: &str) {
166        self.product_name = name.into();
167    }
168
169    pub(crate) fn include_target_in_component_name(&mut self, include: bool) {
170        self.include_target_in_component_name = include;
171    }
172
173    pub(crate) fn is_preview(&mut self, is: bool) {
174        self.is_preview = is;
175    }
176
177    pub(crate) fn permit_symlinks(&mut self, flag: bool) {
178        self.permit_symlinks = flag;
179    }
180
181    pub(crate) fn image_dir(&self) -> &Path {
182        t!(std::fs::create_dir_all(&self.image_dir));
183        &self.image_dir
184    }
185
186    pub(crate) fn add_file(
187        &self,
188        src: impl AsRef<Path>,
189        destdir: impl AsRef<Path>,
190        file_type: FileType,
191    ) {
192        // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply
193        // uses the base directory as the destination directory.
194        let destdir = if destdir.as_ref() == Path::new(".") {
195            self.image_dir.clone()
196        } else {
197            self.image_dir.join(destdir.as_ref())
198        };
199
200        t!(std::fs::create_dir_all(&destdir));
201        self.builder.install(src.as_ref(), &destdir, file_type);
202    }
203
204    pub(crate) fn add_renamed_file(
205        &self,
206        src: impl AsRef<Path>,
207        destdir: impl AsRef<Path>,
208        new_name: &str,
209        file_type: FileType,
210    ) {
211        let destdir = self.image_dir.join(destdir.as_ref());
212        t!(std::fs::create_dir_all(&destdir));
213        self.builder.copy_link(src.as_ref(), &destdir.join(new_name), file_type);
214    }
215
216    pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) {
217        for file in self.overlay.legal_and_readme() {
218            self.add_file(self.builder.src.join(file), destdir.as_ref(), FileType::Regular);
219        }
220    }
221
222    pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
223        let dest = self.image_dir.join(dest.as_ref());
224
225        t!(std::fs::create_dir_all(&dest));
226        self.builder.cp_link_r(src.as_ref(), &dest);
227    }
228
229    pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef<Path>, dest: impl AsRef<Path>) {
230        self.bulk_dirs.push(dest.as_ref().to_path_buf());
231        self.add_dir(src, dest);
232    }
233
234    pub(crate) fn generate(self) -> GeneratedTarball {
235        let mut component_name = self.component.clone();
236        if self.is_preview {
237            component_name.push_str("-preview");
238        }
239        if self.include_target_in_component_name {
240            component_name.push('-');
241            component_name.push_str(
242                self.target
243                    .as_ref()
244                    .expect("include_target_in_component_name used in a targetless tarball"),
245            );
246        }
247
248        self.run(|this, cmd| {
249            cmd.arg("generate")
250                .arg("--image-dir")
251                .arg(&this.image_dir)
252                .arg(format!("--component-name={component_name}"));
253
254            if let Some((dir, dirs)) = this.bulk_dirs.split_first() {
255                let mut arg = dir.as_os_str().to_os_string();
256                for dir in dirs {
257                    arg.push(",");
258                    arg.push(dir);
259                }
260                cmd.arg("--bulk-dirs").arg(&arg);
261            }
262
263            this.non_bare_args(cmd);
264        })
265    }
266
267    pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball {
268        let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string();
269        for tarball in &tarballs[1..] {
270            input_tarballs.push(",");
271            input_tarballs.push(&tarball.path);
272        }
273
274        self.run(|this, cmd| {
275            cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs);
276            this.non_bare_args(cmd);
277        })
278    }
279
280    pub(crate) fn bare(self) -> GeneratedTarball {
281        // Bare tarballs should have the top level directory match the package
282        // name, not "image". We rename the image directory just before passing
283        // into rust-installer.
284        let dest = self.temp_dir.join(self.package_name());
285        t!(move_file(&self.image_dir, &dest));
286
287        self.run(|this, cmd| {
288            let distdir = distdir(this.builder);
289            t!(std::fs::create_dir_all(&distdir));
290            cmd.arg("tarball")
291                .arg("--input")
292                .arg(&dest)
293                .arg("--output")
294                .arg(distdir.join(this.package_name()));
295        })
296    }
297
298    fn package_name(&self) -> String {
299        if let Some(target) = &self.target {
300            format!("{}-{}", self.pkgname, target)
301        } else {
302            self.pkgname.clone()
303        }
304    }
305
306    fn non_bare_args(&self, cmd: &mut BootstrapCommand) {
307        cmd.arg("--rel-manifest-dir=rustlib")
308            .arg("--legacy-manifest-dirs=rustlib,cargo")
309            .arg(format!("--product-name={}", self.product_name))
310            .arg(format!("--success-message={} installed.", self.component))
311            .arg(format!("--package-name={}", self.package_name()))
312            .arg("--non-installed-overlay")
313            .arg(&self.overlay_dir)
314            .arg("--output-dir")
315            .arg(distdir(self.builder));
316    }
317
318    fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut BootstrapCommand)) -> GeneratedTarball {
319        t!(std::fs::create_dir_all(&self.overlay_dir));
320        self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder));
321        if let Some(info) = self.builder.rust_info().info() {
322            channel::write_commit_hash_file(&self.overlay_dir, &info.sha);
323            channel::write_commit_info_file(&self.overlay_dir, info);
324        }
325
326        // Add config file if present.
327        if let Some(config) = &self.builder.config.config {
328            self.add_renamed_file(
329                config,
330                &self.overlay_dir,
331                BUILDER_CONFIG_FILENAME,
332                FileType::Regular,
333            );
334        }
335
336        for file in self.overlay.legal_and_readme() {
337            self.builder.install(
338                &self.builder.src.join(file),
339                &self.overlay_dir,
340                FileType::Regular,
341            );
342        }
343
344        let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller);
345
346        let package_name = self.package_name();
347        self.builder.info(&format!("Dist {package_name}"));
348        let _time = crate::utils::helpers::timeit(self.builder);
349
350        build_cli(&self, &mut cmd);
351        cmd.arg("--work-dir").arg(&self.temp_dir);
352        if let Some(formats) = &self.builder.config.dist_compression_formats {
353            assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
354            cmd.arg("--compression-formats").arg(formats.join(","));
355        }
356
357        // For `x install` tarball files aren't needed, so we can speed up the process by not producing them.
358        let compression_profile = if self.builder.kind == Kind::Install {
359            self.builder.verbose(|| {
360                println!("Forcing dist.compression-profile = 'no-op' for `x install`.")
361            });
362            // "no-op" indicates that the rust-installer won't produce compressed tarball sources.
363            "no-op"
364        } else {
365            assert!(
366                self.builder.config.dist_compression_profile != "no-op",
367                "dist.compression-profile = 'no-op' can only be used for `x install`"
368            );
369
370            &self.builder.config.dist_compression_profile
371        };
372
373        cmd.args(["--compression-profile", compression_profile]);
374
375        // We want to use a pinned modification time for files in the archive
376        // to achieve better reproducibility. However, using the same mtime for all
377        // releases is not ideal, because it can break e.g. Cargo mtime checking
378        // (https://github.com/rust-lang/rust/issues/125578).
379        // Therefore, we set mtime to the date of the latest commit (if we're managed
380        // by git). In this way, the archive will still be always the same for a given commit
381        // (achieving reproducibility), but it will also change between different commits and
382        // Rust versions, so that it won't break mtime-based caches.
383        //
384        // Note that this only overrides the mtime of files, not directories, due to the
385        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
386
387        // Get the UTC timestamp of the last git commit, if we're under git.
388        // We need to use UTC, so that anyone who tries to rebuild from the same commit
389        // gets the same timestamp.
390        if self.builder.rust_info().is_managed_git_subrepository() {
391            // %ct means committer date
392            let timestamp = helpers::git(Some(&self.builder.src))
393                .arg("log")
394                .arg("-1")
395                .arg("--format=%ct")
396                .run_capture_stdout(self.builder)
397                .stdout();
398            cmd.args(["--override-file-mtime", timestamp.trim()]);
399        }
400
401        cmd.run(self.builder);
402
403        // Ensure there are no symbolic links in the tarball. In particular,
404        // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links.
405        let decompressed_output = self.temp_dir.join(&package_name);
406        if !self.builder.config.dry_run() && !self.permit_symlinks {
407            for entry in walkdir::WalkDir::new(&decompressed_output) {
408                let entry = t!(entry);
409                if entry.path_is_symlink() {
410                    panic!("generated a symlink in a tarball: {}", entry.path().display());
411                }
412            }
413        }
414
415        // Use either the first compression format defined, or "gz" as the default.
416        let ext = self
417            .builder
418            .config
419            .dist_compression_formats
420            .as_ref()
421            .and_then(|formats| formats.first())
422            .map(|s| s.as_str())
423            .unwrap_or("gz");
424
425        GeneratedTarball {
426            path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")),
427            decompressed_output,
428            work: self.temp_dir,
429        }
430    }
431}
432
433#[derive(Debug, Clone)]
434pub struct GeneratedTarball {
435    path: PathBuf,
436    decompressed_output: PathBuf,
437    work: PathBuf,
438}
439
440impl GeneratedTarball {
441    pub(crate) fn tarball(&self) -> &Path {
442        &self.path
443    }
444
445    pub(crate) fn decompressed_output(&self) -> &Path {
446        &self.decompressed_output
447    }
448
449    pub(crate) fn work_dir(&self) -> &Path {
450        &self.work
451    }
452}
bootstrap/utils/tarball.rs

bootstrap/utils/
tarball.rs
