cargo/sources/registry/

remote.rs

//! Access to a Git index based registry. See [`RemoteRegistry`] for details.

use crate::core::global_cache_tracker;
use crate::core::{GitReference, PackageId, SourceId};
use crate::sources::git;
use crate::sources::git::fetch::RemoteKind;
use crate::sources::git::resolve_ref;
use crate::sources::registry::MaybeLock;
use crate::sources::registry::download;
use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
use crate::util::cache_lock::CacheLockMode;
use crate::util::errors::CargoResult;
use crate::util::interning::InternedString;
use crate::util::{Filesystem, GlobalContext};
use anyhow::Context as _;
use cargo_util::paths;
use std::cell::{Cell, Ref, RefCell};
use std::fs::File;
use std::mem;
use std::path::Path;
use std::str;
use tracing::{debug, trace};

/// A remote registry is a registry that lives at a remote URL (such as
/// crates.io). The git index is cloned locally, and `.crate` files are
/// downloaded as needed and cached locally.
///
/// This type is primarily accessed through the [`RegistryData`] trait.
///
/// See the [module-level documentation](super) for the index format and layout.
///
/// ## History of Git-based index registry
///
/// Using Git to host this index used to be quite efficient. The full index can
/// be stored efficiently locally on disk, and once it is downloaded, all
/// queries of a registry can happen locally and needn't touch the network.
/// Git-based index was a reasonable design choice at the time when HTTP/2
/// was just introduced.
///
/// However, the full index keeps growing as crates.io grows. It becomes
/// relatively big and slows down the first use of Cargo. Git (specifically
/// libgit2) is not efficient at handling huge amounts of small files either.
/// On the other hand, newer protocols like HTTP/2 are prevalent and capable to
/// serve a bunch of tiny files. Today, it is encouraged to use [`HttpRegistry`],
/// which is the default from 1.70.0. That being said, Cargo will continue
/// supporting Git-based index for a pretty long while.
///
/// [`HttpRegistry`]: super::http_remote::HttpRegistry
pub struct RemoteRegistry<'gctx> {
    /// The name of this source, a unique string (across all sources) used as
    /// the directory name where its cached content is stored.
    name: InternedString,
    /// Path to the registry index (`$CARGO_HOME/registry/index/$REG-HASH`).
    index_path: Filesystem,
    /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/cache/$REG-HASH`).
    cache_path: Filesystem,
    /// The unique identifier of this registry source.
    source_id: SourceId,
    /// This reference is stored so that when a registry needs update, it knows
    /// where to fetch from.
    index_git_ref: GitReference,
    gctx: &'gctx GlobalContext,
    /// A Git [tree object] to help this registry find crate metadata from the
    /// underlying Git repository.
    ///
    /// This is stored here to prevent Git from repeatedly creating a tree object
    /// during each call into `load()`.
    ///
    /// [tree object]: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects#_tree_objects
    tree: RefCell<Option<git2::Tree<'static>>>,
    /// A Git repository that contains the actual index we want.
    repo: RefCell<Option<git2::Repository>>,
    /// The current HEAD commit of the underlying Git repository.
    head: Cell<Option<git2::Oid>>,
    /// This stores sha value of the current HEAD commit for convenience.
    current_sha: Cell<Option<InternedString>>,
    /// Whether this registry needs to update package information.
    ///
    /// See [`RemoteRegistry::mark_updated`] on how to make sure a registry
    /// index is updated only once per session.
    needs_update: Cell<bool>,
    /// Disables status messages.
    quiet: bool,
}

impl<'gctx> RemoteRegistry<'gctx> {
    /// Creates a Git-rebased remote registry for `source_id`.
    ///
    /// * `name` --- Name of a path segment where `.crate` tarballs and the
    ///   registry index are stored. Expect to be unique.
    pub fn new(
        source_id: SourceId,
        gctx: &'gctx GlobalContext,
        name: &str,
    ) -> RemoteRegistry<'gctx> {
        RemoteRegistry {
            name: name.into(),
            index_path: gctx.registry_index_path().join(name),
            cache_path: gctx.registry_cache_path().join(name),
            source_id,
            gctx,
            index_git_ref: GitReference::DefaultBranch,
            tree: RefCell::new(None),
            repo: RefCell::new(None),
            head: Cell::new(None),
            current_sha: Cell::new(None),
            needs_update: Cell::new(false),
            quiet: false,
        }
    }

    /// Creates intermediate dirs and initialize the repository.
    fn repo(&self) -> CargoResult<Ref<'_, Option<git2::Repository>>> {
        if self.repo.borrow().is_none() {
            trace!("acquiring registry index lock");
            let path = self
                .gctx
                .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path);

            self.repo.replace(Some(match git2::Repository::open(&path) {
                Ok(repo) => repo,
                Err(_) => {
                    drop(paths::remove_dir_all(&path));
                    paths::create_dir_all(&path)?;

                    // Note that we'd actually prefer to use a bare repository
                    // here as we're not actually going to check anything out.
                    // All versions of Cargo, though, share the same CARGO_HOME,
                    // so for compatibility with older Cargo which *does* do
                    // checkouts we make sure to initialize a new full
                    // repository (not a bare one).
                    //
                    // We should change this to `init_bare` whenever we feel
                    // like enough time has passed or if we change the directory
                    // that the folder is located in, such as by changing the
                    // hash at the end of the directory.
                    //
                    // Note that in the meantime we also skip `init.templatedir`
                    // as it can be misconfigured sometimes or otherwise add
                    // things that we don't want.
                    let mut opts = git2::RepositoryInitOptions::new();
                    opts.external_template(false);
                    git2::Repository::init_opts(&path, &opts).with_context(|| {
                        format!("failed to initialize index git repository (in {:?})", path)
                    })?
                }
            }));
        }

        Ok(self.repo.borrow())
    }

    /// Get the object ID of the HEAD commit from the underlying Git repository.
    fn head(&self) -> CargoResult<git2::Oid> {
        if self.head.get().is_none() {
            let repo = self.repo()?;
            let repo = repo.as_ref().unwrap();
            let oid = resolve_ref(&self.index_git_ref, repo)?;
            self.head.set(Some(oid));
        }
        Ok(self.head.get().unwrap())
    }

    /// Returns a [`git2::Tree`] object of the current HEAD commit of the
    /// underlying Git repository.
    fn tree(&self) -> CargoResult<Ref<'_, git2::Tree<'_>>> {
        {
            let tree = self.tree.borrow();
            if tree.is_some() {
                return Ok(Ref::map(tree, |s| s.as_ref().unwrap()));
            }
        }
        let repo = self.repo()?;
        let repo = repo.as_ref().unwrap();
        let commit = repo.find_commit(self.head()?)?;
        let tree = commit.tree()?;

        // SAFETY:
        // Unfortunately in libgit2 the tree objects look like they've got a
        // reference to the repository object which means that a tree cannot
        // outlive the repository that it came from. Here we want to cache this
        // tree, though, so to accomplish this we transmute it to a static
        // lifetime.
        //
        // Note that we don't actually hand out the static lifetime, instead we
        // only return a scoped one from this function. Additionally the repo
        // we loaded from (above) lives as long as this object
        // (`RemoteRegistry`) so we then just need to ensure that the tree is
        // destroyed first in the destructor, hence the destructor on
        // `RemoteRegistry` below.
        let tree = unsafe { mem::transmute::<git2::Tree<'_>, git2::Tree<'static>>(tree) };
        *self.tree.borrow_mut() = Some(tree);
        Ok(Ref::map(self.tree.borrow(), |s| s.as_ref().unwrap()))
    }

    /// Gets the current version of the registry index.
    ///
    /// It is usually sha of the HEAD commit from the underlying Git repository.
    fn current_version(&self) -> Option<InternedString> {
        if let Some(sha) = self.current_sha.get() {
            return Some(sha);
        }
        let sha = self.head().ok()?.to_string().into();
        self.current_sha.set(Some(sha));
        Some(sha)
    }

    /// Whether the registry is up-to-date. See [`Self::mark_updated`] for more.
    fn is_updated(&self) -> bool {
        self.gctx.updated_sources().contains(&self.source_id)
    }

    /// Marks this registry as up-to-date.
    ///
    /// This makes sure the index is only updated once per session since it is
    /// an expensive operation. This generally only happens when the resolver
    /// is run multiple times, such as during `cargo publish`.
    fn mark_updated(&self) {
        self.gctx.updated_sources().insert(self.source_id);
    }

    fn update(&self) -> CargoResult<()> {
        if !self.needs_update.get() {
            return Ok(());
        }

        self.needs_update.set(false);

        if self.is_updated() {
            return Ok(());
        }
        self.mark_updated();

        if !self.gctx.network_allowed() {
            return Ok(());
        }
        if self.gctx.cli_unstable().no_index_update {
            return Ok(());
        }

        debug!("updating the index");

        // Ensure that we'll actually be able to acquire an HTTP handle later on
        // once we start trying to download crates. This will weed out any
        // problems with `.cargo/config` configuration related to HTTP.
        //
        // This way if there's a problem the error gets printed before we even
        // hit the index, which may not actually read this configuration.
        self.gctx.http()?;

        self.prepare()?;
        self.head.set(None);
        *self.tree.borrow_mut() = None;
        self.current_sha.set(None);
        let _path = self
            .gctx
            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path);
        if !self.quiet {
            self.gctx
                .shell()
                .status("Updating", self.source_id.display_index())?;
        }

        // Fetch the latest version of our `index_git_ref` into the index
        // checkout.
        let url = self.source_id.url();
        let mut repo = self.repo.borrow_mut();
        let repo = repo.as_mut().unwrap();
        git::fetch(
            repo,
            url.as_str(),
            &self.index_git_ref,
            &self.index_git_ref,
            self.gctx,
            RemoteKind::Registry,
        )
        .with_context(|| format!("failed to fetch `{}`", url))?;

        Ok(())
    }
}

#[async_trait::async_trait(?Send)]
impl<'gctx> RegistryData for RemoteRegistry<'gctx> {
    fn prepare(&self) -> CargoResult<()> {
        self.repo()?;
        self.gctx
            .deferred_global_last_use()?
            .mark_registry_index_used(global_cache_tracker::RegistryIndex {
                encoded_registry_name: self.name,
            });
        Ok(())
    }

    fn index_path(&self) -> &Filesystem {
        &self.index_path
    }

    fn cache_path(&self) -> &Filesystem {
        &self.cache_path
    }

    fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path {
        self.gctx
            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, path)
    }

    /// Read the general concept for `load()` on [`RegistryData::load`].
    ///
    /// `index_version` is a string representing the version of the file used
    /// to construct the cached copy.
    ///
    /// Older versions of Cargo used the single value of the hash of the HEAD
    /// commit as a `index_version`. This is technically correct but a little
    /// too conservative. If a new commit is fetched all cached files need to
    /// be regenerated even if a particular file was not changed.
    ///
    /// However if an old cargo has written such a file we still know how to
    /// read it, as long as we check for that hash value.
    ///
    /// Cargo now uses a hash of the file's contents as provided by git.
    async fn load(
        &self,
        _root: &Path,
        path: &Path,
        index_version: Option<&str>,
    ) -> CargoResult<LoadResponse> {
        if self.needs_update.get() {
            self.update()?;
        }
        // Check if the cache is valid.
        let git_commit_hash = self.current_version();
        if index_version.is_some() && index_version == git_commit_hash.as_deref() {
            // This file was written by an old version of cargo, but it is
            // still up-to-date.
            return Ok(LoadResponse::CacheValid);
        }
        // Note that the index calls this method and the filesystem is locked
        // in the index, so we don't need to worry about an `update_index`
        // happening in a different process.
        fn load_helper(
            registry: &RemoteRegistry<'_>,
            path: &Path,
            index_version: Option<&str>,
        ) -> CargoResult<LoadResponse> {
            let repo = registry.repo()?;
            let repo = repo.as_ref().unwrap();
            let tree = registry.tree()?;
            let entry = tree.get_path(path);
            let entry = entry?;
            let git_file_hash = Some(entry.id().to_string());

            // Check if the cache is valid.
            if index_version.is_some() && index_version == git_file_hash.as_deref() {
                return Ok(LoadResponse::CacheValid);
            }

            let object = entry.to_object(repo)?;
            let Some(blob) = object.as_blob() else {
                anyhow::bail!("path `{}` is not a blob in the git repo", path.display())
            };

            Ok(LoadResponse::Data {
                raw_data: blob.content().to_vec(),
                index_version: git_file_hash,
            })
        }

        loop {
            return match load_helper(&self, path, index_version) {
                Ok(result) => Ok(result),
                Err(_) if !self.is_updated() => {
                    // If git returns an error and we haven't updated the repo,
                    // return pending to allow an update to try again.
                    self.needs_update.set(true);
                    self.update()?;
                    continue;
                }
                Err(e)
                    if e.downcast_ref::<git2::Error>()
                        .map(|e| e.code() == git2::ErrorCode::NotFound)
                        .unwrap_or_default() =>
                {
                    // The repo has been updated and the file does not exist.
                    Ok(LoadResponse::NotFound)
                }
                Err(e) => Err(e),
            };
        }
    }

    async fn config(&self) -> CargoResult<Option<RegistryConfig>> {
        debug!("loading config");
        self.prepare()?;
        self.gctx
            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &self.index_path);
        match self
            .load(Path::new(""), Path::new(RegistryConfig::NAME), None)
            .await?
        {
            LoadResponse::Data { raw_data, .. } => {
                trace!("config loaded");
                let cfg: RegistryConfig = serde_json::from_slice(&raw_data)?;
                Ok(Some(cfg))
            }
            _ => Ok(None),
        }
    }

    /// Read the general concept for `invalidate_cache()` on
    /// [`RegistryData::invalidate_cache`].
    ///
    /// To fully invalidate, undo [`RemoteRegistry::mark_updated`]'s work.
    fn invalidate_cache(&self) {
        self.needs_update.set(true);
    }

    fn set_quiet(&mut self, quiet: bool) {
        self.quiet = quiet;
    }

    fn is_updated(&self) -> bool {
        self.is_updated()
    }

    fn download(&self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock> {
        let registry_config = crate::util::block_on(self.config())?.unwrap();

        download::download(
            &self.cache_path,
            &self.gctx,
            self.name,
            pkg,
            checksum,
            registry_config,
        )
    }

    fn finish_download(&self, pkg: PackageId, checksum: &str, data: &[u8]) -> CargoResult<File> {
        download::finish_download(
            &self.cache_path,
            &self.gctx,
            self.name.clone(),
            pkg,
            checksum,
            data,
        )
    }

    fn is_crate_downloaded(&self, pkg: PackageId) -> bool {
        download::is_crate_downloaded(&self.cache_path, &self.gctx, pkg)
    }
}

/// Implemented to just be sure to drop `tree` field before our other fields.
/// See SAFETY inside [`RemoteRegistry::tree()`] for more.
impl<'gctx> Drop for RemoteRegistry<'gctx> {
    fn drop(&mut self) {
        self.tree.borrow_mut().take();
    }
}