cargo/sources/registry/index/mod.rs
1//! Management of the index of a registry source.
2//!
3//! This module contains management of the index and various operations, such as
4//! actually parsing the index, looking for crates, etc. This is intended to be
5//! abstract over remote indices (downloaded via Git or HTTP) and local registry
6//! indices (which are all just present on the filesystem).
7//!
8//! ## How the index works
9//!
10//! Here is a simple flow when loading a [`Summary`] (metadata) from the index:
11//!
12//! 1. A query is fired via [`RegistryIndex::query_inner`].
13//! 2. Tries loading all summaries via [`RegistryIndex::load_summaries`], and
14//! under the hood calling [`Summaries::parse`] to parse an index file.
15//! 1. If an on-disk index cache is present, loads it via
16//! [`Summaries::parse_cache`].
17//! 2. Otherwise goes to the slower path [`RegistryData::load`] to get the
18//! specific index file.
19//! 3. A [`Summary`] is now ready in callback `f` in [`RegistryIndex::query_inner`].
20//!
21//! To learn the rationale behind this multi-layer index metadata loading,
22//! see [the documentation of the on-disk index cache](cache).
23use crate::core::Dependency;
24use crate::core::dependency::{Artifact, DepKind};
25use crate::core::{PackageId, SourceId, Summary};
26use crate::sources::registry::{LoadResponse, RegistryData};
27use crate::util::IntoUrl;
28use crate::util::interning::InternedString;
29use crate::util::{CargoResult, Filesystem, GlobalContext, OptVersionReq, internal};
30use cargo_util::registry::make_dep_path;
31use cargo_util_schemas::index::{IndexPackage, RegistryDependency};
32use cargo_util_schemas::manifest::RustVersion;
33use semver::Version;
34use serde::{Deserialize, Serialize};
35use std::borrow::Cow;
36use std::collections::BTreeMap;
37use std::collections::HashMap;
38use std::path::Path;
39use std::str;
40use std::task::{Poll, ready};
41use tracing::info;
42
43mod cache;
44use self::cache::CacheManager;
45use self::cache::SummariesCache;
46
47/// The maximum schema version of the `v` field in the index this version of
48/// cargo understands. See [`IndexPackage::v`] for the detail.
49const INDEX_V_MAX: u32 = 2;
50
51/// Manager for handling the on-disk index.
52///
53/// Different kinds of registries store the index differently:
54///
55/// * [`LocalRegistry`] is a simple on-disk tree of files of the raw index.
56/// * [`RemoteRegistry`] is stored as a raw git repository.
57/// * [`HttpRegistry`] fills the on-disk index cache directly without keeping
58/// any raw index.
59///
60/// These means of access are handled via the [`RegistryData`] trait abstraction.
61/// This transparently handles caching of the index in a more efficient format.
62///
63/// [`LocalRegistry`]: super::local::LocalRegistry
64/// [`RemoteRegistry`]: super::remote::RemoteRegistry
65/// [`HttpRegistry`]: super::http_remote::HttpRegistry
66pub struct RegistryIndex<'gctx> {
67 source_id: SourceId,
68 /// Root directory of the index for the registry.
69 path: Filesystem,
70 /// In-memory cache of summary data.
71 ///
72 /// This is keyed off the package name. The [`Summaries`] value handles
73 /// loading the summary data. It keeps an optimized on-disk representation
74 /// of the JSON files, which is created in an as-needed fashion. If it
75 /// hasn't been cached already, it uses [`RegistryData::load`] to access
76 /// to JSON files from the index, and the creates the optimized on-disk
77 /// summary cache.
78 summaries_cache: HashMap<InternedString, Summaries>,
79 /// [`GlobalContext`] reference for convenience.
80 gctx: &'gctx GlobalContext,
81 /// Manager of on-disk caches.
82 cache_manager: CacheManager<'gctx>,
83}
84
85/// An internal cache of summaries for a particular package.
86///
87/// A list of summaries are loaded from disk via one of two methods:
88///
89/// 1. From raw registry index --- Primarily Cargo will parse the corresponding
90/// file for a crate in the upstream crates.io registry. That's just a JSON
91/// blob per line which we can parse, extract the version, and then store here.
92/// See [`IndexPackage`] and [`IndexSummary::parse`].
93///
94/// 2. From on-disk index cache --- If Cargo has previously run, we'll have a
95/// cached index of dependencies for the upstream index. This is a file that
96/// Cargo maintains lazily on the local filesystem and is much faster to
97/// parse since it doesn't involve parsing all of the JSON.
98/// See [`SummariesCache`].
99///
100/// The outward-facing interface of this doesn't matter too much where it's
101/// loaded from, but it's important when reading the implementation to note that
102/// we try to parse as little as possible!
103#[derive(Default)]
104struct Summaries {
105 /// A raw vector of uninterpreted bytes. This is what `Unparsed` start/end
106 /// fields are indexes into. If a `Summaries` is loaded from the crates.io
107 /// index then this field will be empty since nothing is `Unparsed`.
108 raw_data: Vec<u8>,
109
110 /// All known versions of a crate, keyed from their `Version` to the
111 /// possibly parsed or unparsed version of the full summary.
112 versions: HashMap<Version, MaybeIndexSummary>,
113}
114
115/// A lazily parsed [`IndexSummary`].
116enum MaybeIndexSummary {
117 /// A summary which has not been parsed, The `start` and `end` are pointers
118 /// into [`Summaries::raw_data`] which this is an entry of.
119 Unparsed { start: usize, end: usize },
120
121 /// An actually parsed summary.
122 Parsed(IndexSummary),
123}
124
125/// A parsed representation of a summary from the index. This is usually parsed
126/// from a line from a raw index file, or a JSON blob from on-disk index cache.
127///
128/// In addition to a full [`Summary`], we have information on whether it is `yanked`.
129#[derive(Clone, Debug)]
130pub enum IndexSummary {
131 /// Available for consideration
132 Candidate(Summary),
133 /// Yanked within its registry
134 Yanked(Summary),
135 /// Not available as we are offline and create is not downloaded yet
136 Offline(Summary),
137 /// From a newer schema version and is likely incomplete or inaccurate
138 Unsupported(Summary, u32),
139 /// An error was encountered despite being a supported schema version
140 Invalid(Summary),
141}
142
143impl IndexSummary {
144 /// Extract the summary from any variant
145 pub fn as_summary(&self) -> &Summary {
146 match self {
147 IndexSummary::Candidate(sum)
148 | IndexSummary::Yanked(sum)
149 | IndexSummary::Offline(sum)
150 | IndexSummary::Unsupported(sum, _)
151 | IndexSummary::Invalid(sum) => sum,
152 }
153 }
154
155 /// Extract the summary from any variant
156 pub fn into_summary(self) -> Summary {
157 match self {
158 IndexSummary::Candidate(sum)
159 | IndexSummary::Yanked(sum)
160 | IndexSummary::Offline(sum)
161 | IndexSummary::Unsupported(sum, _)
162 | IndexSummary::Invalid(sum) => sum,
163 }
164 }
165
166 pub fn map_summary(self, f: impl Fn(Summary) -> Summary) -> Self {
167 match self {
168 IndexSummary::Candidate(s) => IndexSummary::Candidate(f(s)),
169 IndexSummary::Yanked(s) => IndexSummary::Yanked(f(s)),
170 IndexSummary::Offline(s) => IndexSummary::Offline(f(s)),
171 IndexSummary::Unsupported(s, v) => IndexSummary::Unsupported(f(s), v.clone()),
172 IndexSummary::Invalid(s) => IndexSummary::Invalid(f(s)),
173 }
174 }
175
176 /// Extract the package id from any variant
177 pub fn package_id(&self) -> PackageId {
178 self.as_summary().package_id()
179 }
180
181 /// Returns `true` if the index summary is [`Yanked`].
182 ///
183 /// [`Yanked`]: IndexSummary::Yanked
184 #[must_use]
185 pub fn is_yanked(&self) -> bool {
186 matches!(self, Self::Yanked(..))
187 }
188
189 /// Returns `true` if the index summary is [`Offline`].
190 ///
191 /// [`Offline`]: IndexSummary::Offline
192 #[must_use]
193 pub fn is_offline(&self) -> bool {
194 matches!(self, Self::Offline(..))
195 }
196}
197
198fn index_package_to_summary(pkg: &IndexPackage<'_>, source_id: SourceId) -> CargoResult<Summary> {
199 // ****CAUTION**** Please be extremely careful with returning errors, see
200 // `IndexSummary::parse` for details
201 let pkgid = PackageId::new(pkg.name.as_ref().into(), pkg.vers.clone(), source_id);
202 let deps = pkg
203 .deps
204 .iter()
205 .map(|dep| registry_dependency_into_dep(dep.clone(), source_id))
206 .collect::<CargoResult<Vec<_>>>()?;
207 let mut features = pkg.features.clone();
208 if let Some(features2) = pkg.features2.clone() {
209 for (name, values) in features2 {
210 features.entry(name).or_default().extend(values);
211 }
212 }
213 let features = features
214 .into_iter()
215 .map(|(name, values)| (name.into(), values.into_iter().map(|v| v.into()).collect()))
216 .collect::<BTreeMap<_, _>>();
217 let links: Option<InternedString> = pkg.links.as_ref().map(|l| l.as_ref().into());
218 let mut summary = Summary::new(pkgid, deps, &features, links, pkg.rust_version.clone())?;
219 summary.set_checksum(pkg.cksum.clone());
220 Ok(summary)
221}
222
223#[derive(Deserialize, Serialize)]
224struct IndexPackageMinimum<'a> {
225 name: Cow<'a, str>,
226 vers: Version,
227}
228
229#[derive(Deserialize, Serialize, Default)]
230struct IndexPackageRustVersion {
231 rust_version: Option<RustVersion>,
232}
233
234#[derive(Deserialize, Serialize, Default)]
235struct IndexPackageV {
236 v: Option<u32>,
237}
238
239impl<'gctx> RegistryIndex<'gctx> {
240 /// Creates an empty registry index at `path`.
241 pub fn new(
242 source_id: SourceId,
243 path: &Filesystem,
244 gctx: &'gctx GlobalContext,
245 ) -> RegistryIndex<'gctx> {
246 RegistryIndex {
247 source_id,
248 path: path.clone(),
249 summaries_cache: HashMap::new(),
250 gctx,
251 cache_manager: CacheManager::new(path.join(".cache"), gctx),
252 }
253 }
254
255 /// Returns the hash listed for a specified `PackageId`. Primarily for
256 /// checking the integrity of a downloaded package matching the checksum in
257 /// the index file, aka [`IndexSummary`].
258 pub fn hash(&mut self, pkg: PackageId, load: &mut dyn RegistryData) -> Poll<CargoResult<&str>> {
259 let req = OptVersionReq::lock_to_exact(pkg.version());
260 let summary = self.summaries(pkg.name(), &req, load)?;
261 let summary = ready!(summary).next();
262 Poll::Ready(Ok(summary
263 .ok_or_else(|| internal(format!("no hash listed for {}", pkg)))?
264 .as_summary()
265 .checksum()
266 .ok_or_else(|| internal(format!("no hash listed for {}", pkg)))?))
267 }
268
269 /// Load a list of summaries for `name` package in this registry which
270 /// match `req`.
271 ///
272 /// This function will semantically
273 ///
274 /// 1. parse the index file (either raw or cache),
275 /// 2. match all versions,
276 /// 3. and then return an iterator over all summaries which matched.
277 ///
278 /// Internally there's quite a few layer of caching to amortize this cost
279 /// though since this method is called quite a lot on null builds in Cargo.
280 fn summaries<'a, 'b>(
281 &'a mut self,
282 name: InternedString,
283 req: &'b OptVersionReq,
284 load: &mut dyn RegistryData,
285 ) -> Poll<CargoResult<impl Iterator<Item = &'a IndexSummary> + 'b>>
286 where
287 'a: 'b,
288 {
289 let bindeps = self.gctx.cli_unstable().bindeps;
290
291 let source_id = self.source_id;
292
293 // First up parse what summaries we have available.
294 let summaries = ready!(self.load_summaries(name, load)?);
295
296 // Iterate over our summaries, extract all relevant ones which match our
297 // version requirement, and then parse all corresponding rows in the
298 // registry. As a reminder this `summaries` method is called for each
299 // entry in a lock file on every build, so we want to absolutely
300 // minimize the amount of work being done here and parse as little as
301 // necessary.
302 let raw_data = &summaries.raw_data;
303 Poll::Ready(Ok(summaries
304 .versions
305 .iter_mut()
306 .filter_map(move |(k, v)| if req.matches(k) { Some(v) } else { None })
307 .filter_map(move |maybe| {
308 match maybe.parse(raw_data, source_id, bindeps) {
309 Ok(sum) => Some(sum),
310 Err(e) => {
311 info!("failed to parse `{}` registry package: {}", name, e);
312 None
313 }
314 }
315 })))
316 }
317
318 /// Actually parses what summaries we have available.
319 ///
320 /// If Cargo has run previously, this tries in this order:
321 ///
322 /// 1. Returns from in-memory cache, aka [`RegistryIndex::summaries_cache`].
323 /// 2. If missing, hands over to [`Summaries::parse`] to parse an index file.
324 ///
325 /// The actual kind index file being parsed depends on which kind of
326 /// [`RegistryData`] the `load` argument is given. For example, a
327 /// Git-based [`RemoteRegistry`] will first try a on-disk index cache
328 /// file, and then try parsing registry raw index from Git repository.
329 ///
330 /// In effect, this is intended to be a quite cheap operation.
331 ///
332 /// [`RemoteRegistry`]: super::remote::RemoteRegistry
333 fn load_summaries(
334 &mut self,
335 name: InternedString,
336 load: &mut dyn RegistryData,
337 ) -> Poll<CargoResult<&mut Summaries>> {
338 // If we've previously loaded what versions are present for `name`, just
339 // return that since our in-memory cache should still be valid.
340 if self.summaries_cache.contains_key(&name) {
341 return Poll::Ready(Ok(self.summaries_cache.get_mut(&name).unwrap()));
342 }
343
344 // Prepare the `RegistryData` which will lazily initialize internal data
345 // structures.
346 load.prepare()?;
347
348 let root = load.assert_index_locked(&self.path);
349 let summaries = ready!(Summaries::parse(
350 root,
351 &name,
352 self.source_id,
353 load,
354 self.gctx.cli_unstable().bindeps,
355 &self.cache_manager,
356 ))?
357 .unwrap_or_default();
358 self.summaries_cache.insert(name, summaries);
359 Poll::Ready(Ok(self.summaries_cache.get_mut(&name).unwrap()))
360 }
361
362 /// Clears the in-memory summaries cache.
363 pub fn clear_summaries_cache(&mut self) {
364 self.summaries_cache.clear();
365 }
366
367 /// Attempts to find the packages that match a `name` and a version `req`.
368 ///
369 /// This is primarily used by [`Source::query`](super::Source).
370 pub fn query_inner(
371 &mut self,
372 name: InternedString,
373 req: &OptVersionReq,
374 load: &mut dyn RegistryData,
375 f: &mut dyn FnMut(IndexSummary),
376 ) -> Poll<CargoResult<()>> {
377 if !self.gctx.network_allowed() {
378 // This should only return `Poll::Ready(Ok(()))` if there is at least 1 match.
379 //
380 // If there are 0 matches it should fall through and try again with online.
381 // This is necessary for dependencies that are not used (such as
382 // target-cfg or optional), but are not downloaded. Normally the
383 // build should succeed if they are not downloaded and not used,
384 // but they still need to resolve. If they are actually needed
385 // then cargo will fail to download and an error message
386 // indicating that the required dependency is unavailable while
387 // offline will be displayed.
388 let mut called = false;
389 let callback = &mut |s: IndexSummary| {
390 if !s.is_offline() {
391 called = true;
392 f(s);
393 }
394 };
395 ready!(self.query_inner_with_online(name, req, load, callback, false)?);
396 if called {
397 return Poll::Ready(Ok(()));
398 }
399 }
400 self.query_inner_with_online(name, req, load, f, true)
401 }
402
403 /// Inner implementation of [`Self::query_inner`]. Returns the number of
404 /// summaries we've got.
405 ///
406 /// The `online` controls whether Cargo can access the network when needed.
407 fn query_inner_with_online(
408 &mut self,
409 name: InternedString,
410 req: &OptVersionReq,
411 load: &mut dyn RegistryData,
412 f: &mut dyn FnMut(IndexSummary),
413 online: bool,
414 ) -> Poll<CargoResult<()>> {
415 ready!(self.summaries(name, &req, load))?
416 // First filter summaries for `--offline`. If we're online then
417 // everything is a candidate, otherwise if we're offline we're only
418 // going to consider candidates which are actually present on disk.
419 //
420 // Note: This particular logic can cause problems with
421 // optional dependencies when offline. If at least 1 version
422 // of an optional dependency is downloaded, but that version
423 // does not satisfy the requirements, then resolution will
424 // fail. Unfortunately, whether or not something is optional
425 // is not known here.
426 .map(|s| {
427 if online || load.is_crate_downloaded(s.package_id()) {
428 s.clone()
429 } else {
430 IndexSummary::Offline(s.as_summary().clone())
431 }
432 })
433 .for_each(f);
434 Poll::Ready(Ok(()))
435 }
436
437 /// Looks into the summaries to check if a package has been yanked.
438 pub fn is_yanked(
439 &mut self,
440 pkg: PackageId,
441 load: &mut dyn RegistryData,
442 ) -> Poll<CargoResult<bool>> {
443 let req = OptVersionReq::lock_to_exact(pkg.version());
444 let found = ready!(self.summaries(pkg.name(), &req, load))?.any(|s| s.is_yanked());
445 Poll::Ready(Ok(found))
446 }
447}
448
449impl Summaries {
450 /// Parse out a [`Summaries`] instances from on-disk state.
451 ///
452 /// This will do the followings in order:
453 ///
454 /// 1. Attempt to prefer parsing a previous index cache file that already
455 /// exists from a previous invocation of Cargo (aka you're typing `cargo
456 /// build` again after typing it previously).
457 /// 2. If parsing fails, or the cache isn't found or is invalid, we then
458 /// take a slower path which loads the full descriptor for `relative`
459 /// from the underlying index (aka libgit2 with crates.io, or from a
460 /// remote HTTP index) and then parse everything in there.
461 ///
462 /// * `root` --- this is the root argument passed to `load`
463 /// * `name` --- the name of the package.
464 /// * `source_id` --- the registry's `SourceId` used when parsing JSON blobs
465 /// to create summaries.
466 /// * `load` --- the actual index implementation which may be very slow to
467 /// call. We avoid this if we can.
468 /// * `bindeps` --- whether the `-Zbindeps` unstable flag is enabled
469 pub fn parse(
470 root: &Path,
471 name: &str,
472 source_id: SourceId,
473 load: &mut dyn RegistryData,
474 bindeps: bool,
475 cache_manager: &CacheManager<'_>,
476 ) -> Poll<CargoResult<Option<Summaries>>> {
477 // This is the file we're loading from cache or the index data.
478 // See module comment in `registry/mod.rs` for why this is structured the way it is.
479 let lowered_name = &name.to_lowercase();
480 let relative = make_dep_path(&lowered_name, false);
481
482 let mut cached_summaries = None;
483 let mut index_version = None;
484 if let Some(contents) = cache_manager.get(lowered_name) {
485 match Summaries::parse_cache(contents) {
486 Ok((s, v)) => {
487 cached_summaries = Some(s);
488 index_version = Some(v);
489 }
490 Err(e) => {
491 tracing::debug!("failed to parse {lowered_name:?} cache: {e}");
492 }
493 }
494 }
495
496 let response = ready!(load.load(root, relative.as_ref(), index_version.as_deref())?);
497
498 match response {
499 LoadResponse::CacheValid => {
500 tracing::debug!("fast path for registry cache of {:?}", relative);
501 return Poll::Ready(Ok(cached_summaries));
502 }
503 LoadResponse::NotFound => {
504 cache_manager.invalidate(lowered_name);
505 return Poll::Ready(Ok(None));
506 }
507 LoadResponse::Data {
508 raw_data,
509 index_version,
510 } => {
511 // This is the fallback path where we actually talk to the registry backend to load
512 // information. Here we parse every single line in the index (as we need
513 // to find the versions)
514 tracing::debug!("slow path for {:?}", relative);
515 let mut cache = SummariesCache::default();
516 let mut ret = Summaries::default();
517 ret.raw_data = raw_data;
518 for line in split(&ret.raw_data, b'\n') {
519 // Attempt forwards-compatibility on the index by ignoring
520 // everything that we ourselves don't understand, that should
521 // allow future cargo implementations to break the
522 // interpretation of each line here and older cargo will simply
523 // ignore the new lines.
524 let summary = match IndexSummary::parse(line, source_id, bindeps) {
525 Ok(summary) => summary,
526 Err(e) => {
527 // This should only happen when there is an index
528 // entry from a future version of cargo that this
529 // version doesn't understand. Hopefully, those future
530 // versions of cargo correctly set INDEX_V_MAX and
531 // CURRENT_CACHE_VERSION, otherwise this will skip
532 // entries in the cache preventing those newer
533 // versions from reading them (that is, until the
534 // cache is rebuilt).
535 tracing::info!(
536 "failed to parse {:?} registry package: {}",
537 relative,
538 e
539 );
540 continue;
541 }
542 };
543 let version = summary.package_id().version().clone();
544 cache.versions.push((version.clone(), line));
545 ret.versions.insert(version, summary.into());
546 }
547 if let Some(index_version) = index_version {
548 tracing::trace!("caching index_version {}", index_version);
549 let cache_bytes = cache.serialize(index_version.as_str());
550 // Once we have our `cache_bytes` which represents the `Summaries` we're
551 // about to return, write that back out to disk so future Cargo
552 // invocations can use it.
553 cache_manager.put(lowered_name, &cache_bytes);
554
555 // If we've got debug assertions enabled read back in the cached values
556 // and assert they match the expected result.
557 #[cfg(debug_assertions)]
558 {
559 let readback = SummariesCache::parse(&cache_bytes)
560 .expect("failed to parse cache we just wrote");
561 assert_eq!(
562 readback.index_version, index_version,
563 "index_version mismatch"
564 );
565 assert_eq!(readback.versions, cache.versions, "versions mismatch");
566 }
567 }
568 Poll::Ready(Ok(Some(ret)))
569 }
570 }
571 }
572
573 /// Parses the contents of an on-disk cache, aka [`SummariesCache`], which
574 /// represents information previously cached by Cargo.
575 pub fn parse_cache(contents: Vec<u8>) -> CargoResult<(Summaries, InternedString)> {
576 let cache = SummariesCache::parse(&contents)?;
577 let index_version = cache.index_version.into();
578 let mut ret = Summaries::default();
579 for (version, summary) in cache.versions {
580 let (start, end) = subslice_bounds(&contents, summary);
581 ret.versions
582 .insert(version, MaybeIndexSummary::Unparsed { start, end });
583 }
584 ret.raw_data = contents;
585 return Ok((ret, index_version));
586
587 // Returns the start/end offsets of `inner` with `outer`. Asserts that
588 // `inner` is a subslice of `outer`.
589 fn subslice_bounds(outer: &[u8], inner: &[u8]) -> (usize, usize) {
590 let outer_start = outer.as_ptr() as usize;
591 let outer_end = outer_start + outer.len();
592 let inner_start = inner.as_ptr() as usize;
593 let inner_end = inner_start + inner.len();
594 assert!(inner_start >= outer_start);
595 assert!(inner_end <= outer_end);
596 (inner_start - outer_start, inner_end - outer_start)
597 }
598 }
599}
600
601impl MaybeIndexSummary {
602 /// Parses this "maybe a summary" into a `Parsed` for sure variant.
603 ///
604 /// Does nothing if this is already `Parsed`, and otherwise the `raw_data`
605 /// passed in is sliced with the bounds in `Unparsed` and then actually
606 /// parsed.
607 fn parse(
608 &mut self,
609 raw_data: &[u8],
610 source_id: SourceId,
611 bindeps: bool,
612 ) -> CargoResult<&IndexSummary> {
613 let (start, end) = match self {
614 MaybeIndexSummary::Unparsed { start, end } => (*start, *end),
615 MaybeIndexSummary::Parsed(summary) => return Ok(summary),
616 };
617 let summary = IndexSummary::parse(&raw_data[start..end], source_id, bindeps)?;
618 *self = MaybeIndexSummary::Parsed(summary);
619 match self {
620 MaybeIndexSummary::Unparsed { .. } => unreachable!(),
621 MaybeIndexSummary::Parsed(summary) => Ok(summary),
622 }
623 }
624}
625
626impl From<IndexSummary> for MaybeIndexSummary {
627 fn from(summary: IndexSummary) -> MaybeIndexSummary {
628 MaybeIndexSummary::Parsed(summary)
629 }
630}
631
632impl IndexSummary {
633 /// Parses a line from the registry's index file into an [`IndexSummary`]
634 /// for a package.
635 ///
636 /// The `line` provided is expected to be valid JSON. It is supposed to be
637 /// a [`IndexPackage`].
638 fn parse(line: &[u8], source_id: SourceId, bindeps: bool) -> CargoResult<IndexSummary> {
639 // ****CAUTION**** Please be extremely careful with returning errors
640 // from this function. Entries that error are not included in the
641 // index cache, and can cause cargo to get confused when switching
642 // between different versions that understand the index differently.
643 // Make sure to consider the INDEX_V_MAX and CURRENT_CACHE_VERSION
644 // values carefully when making changes here.
645 let index_summary = (|| {
646 let index = serde_json::from_slice::<IndexPackage<'_>>(line)?;
647 let summary = index_package_to_summary(&index, source_id)?;
648 Ok((index, summary))
649 })();
650 let (index, summary, valid) = match index_summary {
651 Ok((index, summary)) => (index, summary, true),
652 Err(err) => {
653 let Ok(IndexPackageMinimum { name, vers }) =
654 serde_json::from_slice::<IndexPackageMinimum<'_>>(line)
655 else {
656 // If we can't recover, prefer the original error
657 return Err(err);
658 };
659 tracing::info!(
660 "recoverying from failed parse of registry package {name}@{vers}: {err}"
661 );
662 let IndexPackageRustVersion { rust_version } =
663 serde_json::from_slice::<IndexPackageRustVersion>(line).unwrap_or_default();
664 let IndexPackageV { v } =
665 serde_json::from_slice::<IndexPackageV>(line).unwrap_or_default();
666 let index = IndexPackage {
667 name,
668 vers,
669 rust_version,
670 v,
671 deps: Default::default(),
672 features: Default::default(),
673 features2: Default::default(),
674 cksum: Default::default(),
675 yanked: Default::default(),
676 links: Default::default(),
677 };
678 let summary = index_package_to_summary(&index, source_id)?;
679 (index, summary, false)
680 }
681 };
682 let v = index.v.unwrap_or(1);
683 tracing::trace!("json parsed registry {}/{}", index.name, index.vers);
684
685 let v_max = if bindeps {
686 INDEX_V_MAX + 1
687 } else {
688 INDEX_V_MAX
689 };
690
691 if v_max < v {
692 Ok(IndexSummary::Unsupported(summary, v))
693 } else if !valid {
694 Ok(IndexSummary::Invalid(summary))
695 } else if index.yanked.unwrap_or(false) {
696 Ok(IndexSummary::Yanked(summary))
697 } else {
698 Ok(IndexSummary::Candidate(summary))
699 }
700 }
701}
702
703/// Converts an encoded dependency in the registry to a cargo dependency
704fn registry_dependency_into_dep(
705 dep: RegistryDependency<'_>,
706 default: SourceId,
707) -> CargoResult<Dependency> {
708 let RegistryDependency {
709 name,
710 req,
711 mut features,
712 optional,
713 default_features,
714 target,
715 kind,
716 registry,
717 package,
718 public,
719 artifact,
720 bindep_target,
721 lib,
722 } = dep;
723
724 let id = if let Some(registry) = ®istry {
725 SourceId::for_registry(®istry.into_url()?)?
726 } else {
727 default
728 };
729
730 let interned_name = InternedString::new(package.as_ref().unwrap_or(&name));
731 let mut dep = Dependency::parse(interned_name, Some(&req), id)?;
732 if package.is_some() {
733 dep.set_explicit_name_in_toml(name);
734 }
735 let kind = match kind.as_deref().unwrap_or("") {
736 "dev" => DepKind::Development,
737 "build" => DepKind::Build,
738 _ => DepKind::Normal,
739 };
740
741 let platform = match target {
742 Some(target) => Some(target.parse()?),
743 None => None,
744 };
745
746 // All dependencies are private by default
747 let public = public.unwrap_or(false);
748
749 // Unfortunately older versions of cargo and/or the registry ended up
750 // publishing lots of entries where the features array contained the
751 // empty feature, "", inside. This confuses the resolution process much
752 // later on and these features aren't actually valid, so filter them all
753 // out here.
754 features.retain(|s| !s.is_empty());
755
756 // In index, "registry" is null if it is from the same index.
757 // In Cargo.toml, "registry" is None if it is from the default
758 if !id.is_crates_io() {
759 dep.set_registry_id(id);
760 }
761
762 if let Some(artifacts) = artifact {
763 let artifact = Artifact::parse(&artifacts, lib, bindep_target.as_deref())?;
764 dep.set_artifact(artifact);
765 }
766
767 dep.set_optional(optional)
768 .set_default_features(default_features)
769 .set_features(features)
770 .set_platform(platform)
771 .set_kind(kind)
772 .set_public(public);
773
774 Ok(dep)
775}
776
777/// Like [`slice::split`] but is optimized by [`memchr`].
778fn split(haystack: &[u8], needle: u8) -> impl Iterator<Item = &[u8]> {
779 struct Split<'a> {
780 haystack: &'a [u8],
781 needle: u8,
782 }
783
784 impl<'a> Iterator for Split<'a> {
785 type Item = &'a [u8];
786
787 fn next(&mut self) -> Option<&'a [u8]> {
788 if self.haystack.is_empty() {
789 return None;
790 }
791 let (ret, remaining) = match memchr::memchr(self.needle, self.haystack) {
792 Some(pos) => (&self.haystack[..pos], &self.haystack[pos + 1..]),
793 None => (self.haystack, &[][..]),
794 };
795 self.haystack = remaining;
796 Some(ret)
797 }
798 }
799
800 Split { haystack, needle }
801}