rustc_incremental/persist/

file_format.rs

//! This module defines a generic file format that allows to check if a given
//! file generated by incremental compilation was generated by a compatible
//! compiler version. This file format is used for the on-disk version of the
//! dependency graph and the exported metadata hashes.
//!
//! In practice "compatible compiler version" means "exactly the same compiler
//! version", since the header encodes the git commit hash of the compiler.
//! Since we can always just ignore the incremental compilation cache and
//! compiler versions don't change frequently for the typical user, being
//! conservative here practically has no downside.

use std::borrow::Cow;
use std::io::{self, Read};
use std::path::{Path, PathBuf};
use std::{array, env, fs};

use rustc_data_structures::memmap::Mmap;
use rustc_serialize::Encoder;
use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
use rustc_session::Session;
use tracing::debug;

use crate::errors;

/// The first few bytes of files generated by incremental compilation.
const FILE_MAGIC: &[u8] = b"RSIC";

/// Change this if the header format changes.
const HEADER_FORMAT_VERSION: u16 = 0;

pub(crate) fn write_file_header(stream: &mut FileEncoder, sess: &Session) {
    stream.emit_raw_bytes(FILE_MAGIC);
    stream.emit_raw_bytes(&u16::to_le_bytes(HEADER_FORMAT_VERSION));

    let rustc_version = rustc_version(sess);
    let rustc_version_len =
        u8::try_from(rustc_version.len()).expect("version string should not exceed 255 bytes");
    stream.emit_raw_bytes(&[rustc_version_len]);
    stream.emit_raw_bytes(rustc_version.as_bytes());
}

pub(crate) fn save_in<F>(sess: &Session, path_buf: PathBuf, name: &str, encode: F)
where
    F: FnOnce(FileEncoder) -> FileEncodeResult,
{
    debug!("save: storing data in {}", path_buf.display());

    // Delete the old file, if any.
    // Note: It's important that we actually delete the old file and not just
    // truncate and overwrite it, since it might be a shared hard-link, the
    // underlying data of which we don't want to modify.
    //
    // We have to ensure we have dropped the memory maps to this file
    // before performing this removal.
    match fs::remove_file(&path_buf) {
        Ok(()) => {
            debug!("save: remove old file");
        }
        Err(err) if err.kind() == io::ErrorKind::NotFound => (),
        Err(err) => sess.dcx().emit_fatal(errors::DeleteOld { name, path: path_buf, err }),
    }

    let mut encoder = match FileEncoder::new(&path_buf) {
        Ok(encoder) => encoder,
        Err(err) => sess.dcx().emit_fatal(errors::CreateNew { name, path: path_buf, err }),
    };

    write_file_header(&mut encoder, sess);

    match encode(encoder) {
        Ok(position) => {
            sess.prof.artifact_size(
                &name.replace(' ', "_"),
                path_buf.file_name().unwrap().to_string_lossy(),
                position as u64,
            );
            debug!("save: data written to disk successfully");
        }
        Err((path, err)) => sess.dcx().emit_fatal(errors::WriteNew { name, path, err }),
    }
}

pub(crate) struct OpenFile {
    /// A read-only mmap view of the file contents.
    pub(crate) mmap: Mmap,
    /// File position to start reading normal data from, just after the end of the file header.
    pub(crate) start_pos: usize,
}

pub(crate) enum OpenFileError {
    /// Either the file was not found, or one of the header checks failed.
    ///
    /// These conditions prevent us from reading the file contents, but should
    /// not trigger an error or even a warning, because they routinely happen
    /// during normal operation:
    /// - File-not-found occurs in a fresh build, or after clearing the build directory.
    /// - Header-mismatch occurs after upgrading or switching compiler versions.
    NotFoundOrHeaderMismatch,

    /// An unexpected I/O error occurred while opening or checking the file.
    IoError { err: io::Error },
}

impl From<io::Error> for OpenFileError {
    fn from(err: io::Error) -> Self {
        OpenFileError::IoError { err }
    }
}

/// Tries to open a file that was written by the previous incremental-compilation
/// session, and checks that it was produced by a matching compiler version.
pub(crate) fn open_incremental_file(
    sess: &Session,
    path: &Path,
) -> Result<OpenFile, OpenFileError> {
    let file = fs::File::open(path).map_err(|err| {
        if err.kind() == io::ErrorKind::NotFound {
            OpenFileError::NotFoundOrHeaderMismatch
        } else {
            OpenFileError::IoError { err }
        }
    })?;

    // SAFETY: This process must not modify nor remove the backing file while the memory map lives.
    // For the dep-graph and the work product index, it is as soon as the decoding is done.
    // For the query result cache, the memory map is dropped in save_dep_graph before calling
    // save_in and trying to remove the backing file.
    //
    // There is no way to prevent another process from modifying this file.
    let mmap = unsafe { Mmap::map(file) }?;

    let mut file = io::Cursor::new(&*mmap);

    // Check FILE_MAGIC
    {
        debug_assert!(FILE_MAGIC.len() == 4);
        let mut file_magic = [0u8; 4];
        file.read_exact(&mut file_magic)?;
        if file_magic != FILE_MAGIC {
            report_format_mismatch(sess, path, "Wrong FILE_MAGIC");
            return Err(OpenFileError::NotFoundOrHeaderMismatch);
        }
    }

    // Check HEADER_FORMAT_VERSION
    {
        debug_assert!(size_of_val(&HEADER_FORMAT_VERSION) == 2);
        let mut header_format_version = [0u8; 2];
        file.read_exact(&mut header_format_version)?;
        let header_format_version = u16::from_le_bytes(header_format_version);

        if header_format_version != HEADER_FORMAT_VERSION {
            report_format_mismatch(sess, path, "Wrong HEADER_FORMAT_VERSION");
            return Err(OpenFileError::NotFoundOrHeaderMismatch);
        }
    }

    // Check RUSTC_VERSION
    {
        let mut rustc_version_str_len = 0u8;
        file.read_exact(array::from_mut(&mut rustc_version_str_len))?;
        let mut buffer = vec![0; usize::from(rustc_version_str_len)];
        file.read_exact(&mut buffer)?;

        if buffer != rustc_version(sess).as_bytes() {
            report_format_mismatch(sess, path, "Different compiler version");
            return Err(OpenFileError::NotFoundOrHeaderMismatch);
        }
    }

    let start_pos = file.position() as usize;
    Ok(OpenFile { mmap, start_pos })
}

fn report_format_mismatch(sess: &Session, file: &Path, message: &str) {
    debug!("read_file: {}", message);

    if sess.opts.unstable_opts.incremental_info {
        eprintln!(
            "[incremental] ignoring cache artifact `{}`: {}",
            file.file_name().unwrap().to_string_lossy(),
            message
        );
    }
}

/// A version string that hopefully is always different for compiler versions
/// with different encodings of incremental compilation artifacts. Contains
/// the Git commit hash.
fn rustc_version(sess: &Session) -> Cow<'static, str> {
    // Allow version string overrides so that tests can produce a header-mismatch on demand.
    if sess.is_nightly_build()
        && let Ok(env_version) = env::var("RUSTC_FORCE_RUSTC_VERSION")
    {
        Cow::Owned(env_version)
    } else {
        Cow::Borrowed(sess.cfg_version)
    }
}