cargo/compiler/

layout.rs

//! Management of the directory layout of a build
//!
//! The directory layout is a little tricky at times, hence a separate file to
//! house this logic. Cargo stores build artifacts in two directories: `artifact-dir` and
//! `build-dir`
//!
//! ## `artifact-dir` layout
//!
//! `artifact-dir` is where final artifacts like binaries are stored.
//! The `artifact-dir` layout is consider part of the public API and
//! cannot be easily changed.
//!
//! ```text
//! <artifact-dir>/
//!
//!     # Compilation files are grouped by build target and profile.
//!     # The target is omitted if not explicitly specified.
//!     [<target>]/<profile>/ # e.g. `debug` / `release`
//!
//!         # File used to lock the directory to prevent multiple cargo processes
//!         # from using it at the same time.
//!         .cargo-lock
//!
//!         # Root directory for all compiled examples.
//!         examples/
//!
//!     # Output from rustdoc
//!     doc/
//!
//!     # Output from `cargo package` to build a `.crate` file.
//!     package/
//! ```
//!
//! ## `build-dir` layout
//!
//! `build-dir` is where intermediate build artifacts are stored.
//! The `build-dir` layout is considered an internal implementation detail of Cargo
//! meaning that we can change this if needed. However, in reality many tools rely on
//! implementation details of Cargo so breaking changes need to be done carefully.
//!
//! ```text
//! <build-dir>/
//!
//!     # Cache of `rustc -Vv` output for performance.
//!     .rustc-info.json
//!
//!     # Compilation files are grouped by build target and profile.
//!     # The target is omitted if not explicitly specified.
//!     [<target>]/<profile>/ # e.g. `debug` / `release`
//!
//!         # File used to lock the directory to prevent multiple cargo processes
//!         # from using it at the same time.
//!         .cargo-lock
//!
//!         # Hidden directory that holds all of the fingerprint files for all
//!         # packages
//!         .fingerprint/
//!             # Each package is in a separate directory.
//!             # Note that different target kinds have different filename prefixes.
//!             $pkgname-$META/
//!                 # Set of source filenames for this package.
//!                 dep-lib-$targetname
//!                 # Timestamp when this package was last built.
//!                 invoked.timestamp
//!                 # The fingerprint hash.
//!                 lib-$targetname
//!                 # Detailed information used for logging the reason why
//!                 # something is being recompiled.
//!                 lib-$targetname.json
//!                 # The console output from the compiler. This is cached
//!                 # so that warnings can be redisplayed for "fresh" units.
//!                 output-lib-$targetname
//!
//!         # This is the root directory for all rustc artifacts except build
//!         # scripts, examples, and test and bench executables. Almost every
//!         # artifact should have a metadata hash added to its filename to
//!         # prevent collisions. One notable exception is dynamic libraries.
//!         deps/
//!
//!             # Each artifact dependency gets in its own directory.
//!             /artifact/$pkgname-$META/$kind
//!
//!         # Root directory for all compiled examples.
//!         examples/
//!
//!         # Directory used to store incremental data for the compiler (when
//!         # incremental is enabled.
//!         incremental/
//!
//!         # This is the location at which the output of all custom build
//!         # commands are rooted.
//!         build/
//!
//!             # Each package gets its own directory where its build script and
//!             # script output are placed
//!             $pkgname-$META/    # For the build script itself.
//!                 # The build script executable (name may be changed by user).
//!                 build-script-build-$META
//!                 # Hard link to build-script-build-$META.
//!                 build-script-build
//!                 # Dependency information generated by rustc.
//!                 build-script-build-$META.d
//!                 # Debug information, depending on platform and profile
//!                 # settings.
//!                 <debug symbols>
//!
//!             # The package shows up twice with two different metadata hashes.
//!             $pkgname-$META/  # For the output of the build script.
//!                 # Timestamp when the build script was last executed.
//!                 invoked.timestamp
//!                 # Directory where script can output files ($OUT_DIR).
//!                 out/
//!                 # Output from the build script.
//!                 output
//!                 # Path to `out`, used to help when the target directory is
//!                 # moved.
//!                 root-output
//!                 # Stderr output from the build script.
//!                 stderr
//!
//!     # Used by `cargo package` and `cargo publish` to build a `.crate` file.
//!     package/
//!
//!     # Experimental feature for generated build scripts.
//!     .metabuild/
//! ```
//!
//! ### New `build-dir` layout
//!
//! `build-dir` supports a new "build unit" based layout that is unstable.
//! It can be enabled via `-Zbuild-dir-new-layout`.
//! For more info about the layout transition see: [#15010](https://github.com/rust-lang/cargo/issues/15010)
//!
//! ```text
//! <build-dir>/
//!
//!     # Cache of `rustc -Vv` output for performance.
//!     .rustc-info.json
//!
//!     # Compilation files are grouped by build target and profile.
//!     # The target is omitted if not explicitly specified.
//!     [<target>]/<profile>/ # e.g. `debug` / `release`
//!
//!         # File used to lock the directory to prevent multiple cargo processes
//!         # from using it at the same time.
//!         .cargo-lock
//!
//!         # Directory used to store incremental data for the compiler (when
//!         # incremental is enabled.
//!         incremental/
//!
//!         # Main directory for storing build unit related files.
//!         # Files are organized by Cargo build unit (`$pkgname/$META`) so that
//!         # related files are stored in a single directory.
//!         build/
//!
//!             # This is the location at which the output of all files related to
//!             # a given build unit. These files are organized together so that we can
//!             # treat this directly like a single unit for locking and caching.
//!             $pkgname/
//!                 $META/
//!                     # The general purpose output directory for build units.
//!                     # For compilation units, the rustc artifact will be located here.
//!                     # For build script run units, this is the $OUT_DIR
//!                     out/
//!
//!                         # For artifact dependency units, the output is nested by the kind
//!                         artifact/$kind
//!
//!                     # Directory that holds all of the fingerprint files for the build unit.
//!                     fingerprint/
//!                         # Set of source filenames for this package.
//!                         dep-lib-$targetname
//!                         # Timestamp when this package was last built.
//!                         invoked.timestamp
//!                         # The fingerprint hash.
//!                         lib-$targetname
//!                         # Detailed information used for logging the reason why
//!                         # something is being recompiled.
//!                         lib-$targetname.json
//!                         # The console output from the compiler. This is cached
//!                         # so that warnings can be redisplayed for "fresh" units.
//!                         output-lib-$targetname
//!
//!                     # Directory for "execution" units that spawn a process (excluding compilation with
//!                     # rustc). Contains the process execution details.
//!                     # Currently the only execution unit Cargo supports is running build script
//!                     # binaries.
//!                     run/
//!                         # Timestamp of last execution.
//!                         invoked.timestamp
//!                         # Stdout output from the process.
//!                         stdout
//!                         # Stderr output from the process.
//!                         stderr
//!                         # Path to `out`, used to help when the target directory is
//!                         # moved. (build scripts)
//!                         root-output
//!
//!     # Used by `cargo package` and `cargo publish` to build a `.crate` file.
//!     package/
//!
//!     # Experimental feature for generated build scripts.
//!     .metabuild/
//! ```
//!
//! When cross-compiling, the layout is the same, except it appears in
//! `target/$TUPLE`.

use crate::compiler::CompileTarget;
use crate::util::flock::is_on_nfs_mount;
use crate::util::{CargoResult, FileLock};
use crate::workspace::Workspace;
use cargo_util::paths;
use std::path::{Path, PathBuf};

/// Contains the paths of all target output locations.
///
/// See module docs for more information.
pub struct Layout {
    artifact_dir: Option<ArtifactDirLayout>,
    build_dir: BuildDirLayout,
    _lock: Option<FileLock>,
}

impl Layout {
    /// Calculate the paths for build output, lock the build directory, and return as a Layout.
    ///
    /// This function will block if the directory is already locked.
    ///
    /// `dest` should be the final artifact directory name. Currently either
    /// "debug" or "release".
    pub fn new(
        ws: &Workspace<'_>,
        target: Option<CompileTarget>,
        dest: &str,
        must_take_artifact_dir_lock: bool,
        must_take_build_dir_lock_exclusively: bool,
    ) -> CargoResult<Layout> {
        let is_new_layout = ws.gctx().cli_unstable().build_dir_new_layout;
        let mut root = ws.target_dir();
        let mut build_root = ws.build_dir();
        if let Some(target) = target {
            root.push(target.short_name());
            build_root.push(target.short_name());
        }
        let build_dest = build_root.join(dest);
        let dest = root.join(dest);
        // If the root directory doesn't already exist go ahead and create it
        // here. Use this opportunity to exclude it from backups as well if the
        // system supports it since this is a freshly created folder.
        //
        // Only the created directory is excluded, not the parents created
        // along the way, so when one root contains the other, as with a
        // `--target-dir` inside `build.build-dir`, the outer one is created
        // first.
        for dir in outer_first(root.as_path_unlocked(), build_root.as_path_unlocked()) {
            paths::create_dir_all_excluded_from_backups_atomic(dir)?;
        }

        // Now that the excluded from backups target root is created we can create the
        // actual destination (sub)subdirectory.
        paths::create_dir_all(dest.as_path_unlocked())?;

        let build_dir_lock = if is_on_nfs_mount(build_root.as_path_unlocked()) {
            None
        } else {
            if ws.gctx().cli_unstable().fine_grain_locking && !must_take_build_dir_lock_exclusively
            {
                Some(build_dest.open_ro_shared_create(
                    ".cargo-build-lock",
                    ws.gctx(),
                    "build directory",
                )?)
            } else {
                Some(build_dest.open_rw_exclusive_create(
                    ".cargo-build-lock",
                    ws.gctx(),
                    "build directory",
                )?)
            }
        };
        let build_root = build_root.into_path_unlocked();
        let build_dest = build_dest.as_path_unlocked();
        let deps = build_dest.join("deps");
        let artifact = deps.join("artifact");

        // We take a shared lock on `.cargo-lock` to make sure we don't run currently with
        // older versions of Cargo (including tools that use Cargo as a library) that don't support
        // `.cargo-build-lock`.
        let lock = if is_on_nfs_mount(root.as_path_unlocked()) {
            None
        } else {
            Some(dest.open_ro_shared_create(".cargo-lock", ws.gctx(), "artifact directory")?)
        };

        let artifact_dir = if must_take_artifact_dir_lock {
            // For now we don't do any more finer-grained locking on the artifact
            // directory, so just lock the entire thing for the duration of this
            // compile.
            let artifact_dir_lock = if is_on_nfs_mount(root.as_path_unlocked()) {
                None
            } else {
                Some(dest.open_rw_exclusive_create(
                    ".cargo-artifact-lock",
                    ws.gctx(),
                    "artifact directory",
                )?)
            };
            let root = root.into_path_unlocked();
            let dest = dest.into_path_unlocked();
            Some(ArtifactDirLayout {
                dest: dest.clone(),
                examples: dest.join("examples"),
                doc: root.join("doc"),
                timings: root.join("cargo-timings"),
                _lock: artifact_dir_lock,
            })
        } else {
            None
        };
        Ok(Layout {
            artifact_dir,
            build_dir: BuildDirLayout {
                root: build_root.clone(),
                deps,
                build: build_dest.join("build"),
                artifact,
                incremental: build_dest.join("incremental"),
                fingerprint: build_dest.join(".fingerprint"),
                examples: build_dest.join("examples"),
                tmp: build_root.join("tmp"),
                _lock: build_dir_lock,
                is_new_layout,
            },
            _lock: lock,
        })
    }

    /// Makes sure all directories stored in the Layout exist on the filesystem.
    pub fn prepare(&mut self) -> CargoResult<()> {
        if let Some(ref mut artifact_dir) = self.artifact_dir {
            artifact_dir.prepare()?;
        }
        self.build_dir.prepare()?;

        Ok(())
    }

    pub fn artifact_dir(&self) -> Option<&ArtifactDirLayout> {
        self.artifact_dir.as_ref()
    }

    pub fn build_dir(&self) -> &BuildDirLayout {
        &self.build_dir
    }
}

/// Orders two directories so that one that contains the other comes first.
pub fn outer_first<'a>(a: &'a Path, b: &'a Path) -> [&'a Path; 2] {
    if a.starts_with(b) { [b, a] } else { [a, b] }
}

pub struct ArtifactDirLayout {
    /// The final artifact destination: `<artifact-dir>/debug` (or `release`).
    dest: PathBuf,
    /// The directory for examples
    examples: PathBuf,
    /// The directory for rustdoc output
    doc: PathBuf,
    /// The directory for --timings output
    timings: PathBuf,
    /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
    /// struct is `drop`ped.
    _lock: Option<FileLock>,
}

impl ArtifactDirLayout {
    /// Makes sure all directories stored in the Layout exist on the filesystem.
    pub fn prepare(&mut self) -> CargoResult<()> {
        paths::create_dir_all(&self.examples)?;

        Ok(())
    }
    /// Fetch the destination path for final artifacts  (`/…/target/debug`).
    pub fn dest(&self) -> &Path {
        &self.dest
    }
    /// Fetch the examples path.
    pub fn examples(&self) -> &Path {
        &self.examples
    }
    /// Fetch the doc path.
    pub fn doc(&self) -> &Path {
        &self.doc
    }
    /// Fetch the cargo-timings path.
    pub fn timings(&self) -> &Path {
        &self.timings
    }
}

pub struct BuildDirLayout {
    /// The root directory: `/path/to/build-dir`.
    /// If cross compiling: `/path/to/build-dir/$TUPLE`.
    root: PathBuf,
    /// The directory with rustc artifacts
    deps: PathBuf,
    /// The primary directory for build files
    build: PathBuf,
    /// The directory for artifacts, i.e. binaries, cdylibs, staticlibs
    artifact: PathBuf,
    /// The directory for incremental files
    incremental: PathBuf,
    /// The directory for fingerprints
    fingerprint: PathBuf,
    /// The directory for pre-uplifted examples: `build-dir/debug/examples`
    examples: PathBuf,
    /// The directory for temporary data of integration tests and benches
    tmp: PathBuf,
    /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
    /// struct is `drop`ped.
    ///
    /// Will be `None` when the build-dir and target-dir are the same path as we cannot
    /// lock the same path twice.
    _lock: Option<FileLock>,
    is_new_layout: bool,
}

impl BuildDirLayout {
    /// Makes sure all directories stored in the Layout exist on the filesystem.
    pub fn prepare(&mut self) -> CargoResult<()> {
        if !self.is_new_layout {
            paths::create_dir_all(&self.deps)?;
            paths::create_dir_all(&self.fingerprint)?;
            paths::create_dir_all(&self.examples)?;
        }
        paths::create_dir_all(&self.incremental)?;
        paths::create_dir_all(&self.build)?;

        Ok(())
    }
    /// Fetch the deps path.
    pub fn deps(&self, pkg_dir: &str) -> PathBuf {
        if self.is_new_layout {
            self.out_force_new_layout(pkg_dir)
        } else {
            self.legacy_deps().to_path_buf()
        }
    }
    /// Fetch the output path for build units. (new layout only)
    ///
    /// New features should consider using this so we can avoid their migrations.
    pub fn out_force_new_layout(&self, pkg_dir: &str) -> PathBuf {
        let mut build_dir = self.build_unit(pkg_dir);
        // This function can be somewhat hot for packages with many dependencies, so reuse the
        // PathBuf allocation here.
        build_dir.push("out");
        build_dir
    }
    /// Fetch the deps path. (old layout)
    pub fn legacy_deps(&self) -> &Path {
        &self.deps
    }
    pub fn root(&self) -> &Path {
        &self.root
    }
    /// Fetch the build examples path.
    pub fn examples(&self) -> &Path {
        &self.examples
    }
    /// Fetch the incremental path.
    pub fn incremental(&self) -> &Path {
        &self.incremental
    }
    /// Fetch the fingerprint path.
    pub fn fingerprint(&self, pkg_dir: &str) -> PathBuf {
        if self.is_new_layout {
            self.build_unit(pkg_dir).join("fingerprint")
        } else {
            self.legacy_fingerprint().join(pkg_dir)
        }
    }
    /// Fetch the fingerprint path. (old layout)
    pub fn legacy_fingerprint(&self) -> &Path {
        &self.fingerprint
    }
    /// Fetch the build path.
    pub fn build(&self) -> &Path {
        &self.build
    }
    /// Fetch the build script path.
    pub fn build_script(&self, pkg_dir: &str) -> PathBuf {
        if self.is_new_layout {
            self.deps(pkg_dir)
        } else {
            self.build().join(pkg_dir)
        }
    }
    /// Fetch the build script execution path.
    pub fn build_script_execution(&self, pkg_dir: &str) -> PathBuf {
        if self.is_new_layout {
            self.build_unit(pkg_dir).join("run")
        } else {
            self.build().join(pkg_dir)
        }
    }
    /// Fetch the artifact path.
    pub fn artifact(&self, pkg_dir: &str, kind: &str) -> PathBuf {
        if self.is_new_layout {
            self.build_unit(pkg_dir).join("artifact").join(kind)
        } else {
            self.artifact.join(pkg_dir).join(kind)
        }
    }
    /// Fetch the build unit path
    pub fn build_unit(&self, pkg_dir: &str) -> PathBuf {
        self.build().join(pkg_dir)
    }
    /// Create and return the tmp path.
    pub fn prepare_tmp(&self) -> CargoResult<&Path> {
        paths::create_dir_all(&self.tmp)?;
        Ok(&self.tmp)
    }
}