cargo/util/network/
retry.rs

1//! Utilities for retrying a network operation.
2//!
3//! Some network errors are considered "spurious", meaning it is not a real
4//! error (such as a 404 not found) and is likely a transient error (like a
5//! bad network connection) that we can hope will resolve itself shortly. The
6//! [`Retry`] type offers a way to repeatedly perform some kind of network
7//! operation with a delay if it detects one of these possibly transient
8//! errors.
9//!
10//! This supports errors from [`git2`], [`gix`], [`curl`], and
11//! [`HttpNotSuccessful`] 5xx HTTP errors.
12//!
13//! The number of retries can be configured by the user via the `net.retry`
14//! config option. This indicates the number of times to retry the operation
15//! (default 3 times for a total of 4 attempts).
16//!
17//! There are hard-coded constants that indicate how long to sleep between
18//! retries. The constants are tuned to balance a few factors, such as the
19//! responsiveness to the user (we don't want cargo to hang for too long
20//! retrying things), and accommodating things like Cloudfront's default
21//! negative TTL of 10 seconds (if Cloudfront gets a 5xx error for whatever
22//! reason it won't try to fetch again for 10 seconds).
23//!
24//! The timeout also implements a primitive form of random jitter. This is so
25//! that if multiple requests fail at the same time that they don't all flood
26//! the server at the same time when they are retried. This jitter still has
27//! some clumping behavior, but should be good enough.
28//!
29//! [`Retry`] is the core type for implementing retry logic. The
30//! [`Retry::try`] method can be called with a callback, and it will
31//! indicate if it needs to be called again sometime in the future if there
32//! was a possibly transient error. The caller is responsible for sleeping the
33//! appropriate amount of time and then calling [`Retry::try`] again.
34//!
35//! [`with_retry`] is a convenience function that will create a [`Retry`] and
36//! handle repeatedly running a callback until it succeeds, or it runs out of
37//! retries.
38//!
39//! Some interesting resources about retries:
40//! - <https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/>
41//! - <https://en.wikipedia.org/wiki/Exponential_backoff>
42//! - <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After>
43
44use crate::util::errors::{GitCliError, HttpNotSuccessful};
45use crate::util::network::http_async;
46use crate::{CargoResult, GlobalContext};
47use anyhow::Error;
48use rand::RngExt;
49use std::cmp::min;
50use std::time::Duration;
51
52/// State for managing retrying a network operation.
53pub struct Retry<'a> {
54    gctx: &'a GlobalContext,
55    /// The number of failed attempts that have been done so far.
56    ///
57    /// Starts at 0, and increases by one each time an attempt fails.
58    retries: u64,
59    /// The maximum number of times the operation should be retried.
60    ///
61    /// 0 means it should never retry.
62    max_retries: u64,
63}
64
65/// The result of attempting some operation via [`Retry::try`].
66pub enum RetryResult<T> {
67    /// The operation was successful.
68    ///
69    /// The wrapped value is the return value of the callback function.
70    Success(T),
71    /// The operation was an error, and it should not be tried again.
72    Err(anyhow::Error),
73    /// The operation failed, and should be tried again in the future.
74    ///
75    /// The wrapped value is the number of milliseconds to wait before trying
76    /// again. The caller is responsible for waiting this long and then
77    /// calling [`Retry::try`] again.
78    Retry(u64),
79}
80
81/// Default for `net.retry`
82pub const MAX_RETRY_DEFAULT: u32 = 3;
83/// Maximum amount of time a single retry can be delayed (seconds).
84pub const MAX_RETRY_SLEEP_S: u64 = 10;
85/// Maximum amount of time a single retry can be delayed (milliseconds).
86const MAX_RETRY_SLEEP_MS: u64 = MAX_RETRY_SLEEP_S * 1000;
87/// The minimum initial amount of time a retry will be delayed (milliseconds).
88///
89/// The actual amount of time will be a random value above this.
90const INITIAL_RETRY_SLEEP_BASE_MS: u64 = 500;
91/// The maximum amount of additional time the initial retry will take (milliseconds).
92///
93/// The initial delay will be [`INITIAL_RETRY_SLEEP_BASE_MS`] plus a random range
94/// from 0 to this value.
95const INITIAL_RETRY_JITTER_MS: u64 = 1000;
96
97impl<'a> Retry<'a> {
98    pub fn new(gctx: &'a GlobalContext) -> CargoResult<Retry<'a>> {
99        Ok(Retry {
100            gctx,
101            retries: 0,
102            max_retries: gctx.net_config()?.retry.unwrap_or(MAX_RETRY_DEFAULT) as u64,
103        })
104    }
105
106    /// Calls the given callback, and returns a [`RetryResult`] which
107    /// indicates whether or not this needs to be called again at some point
108    /// in the future to retry the operation if it failed.
109    pub fn r#try<T>(&mut self, f: impl FnOnce() -> CargoResult<T>) -> RetryResult<T> {
110        match f() {
111            Err(ref e) if maybe_spurious(e) && self.retries < self.max_retries => {
112                let err = e.downcast_ref::<HttpNotSuccessful>();
113                let err_msg = err
114                    .map(|http_err| http_err.display_short())
115                    .unwrap_or_else(|| e.root_cause().to_string());
116                let left_retries = self.max_retries - self.retries;
117                let msg = format!(
118                    "spurious network error ({} {} remaining): {err_msg}",
119                    left_retries,
120                    if left_retries != 1 { "tries" } else { "try" }
121                );
122                if let Err(e) = self.gctx.shell().warn(msg) {
123                    return RetryResult::Err(e);
124                }
125                self.retries += 1;
126                let sleep = err
127                    .and_then(|v| Self::parse_retry_after(v, &jiff::Timestamp::now()))
128                    // Limit the Retry-After to a maximum value to avoid waiting too long.
129                    .map(|retry_after| retry_after.min(MAX_RETRY_SLEEP_MS))
130                    .unwrap_or_else(|| self.next_sleep_ms());
131                RetryResult::Retry(sleep)
132            }
133            Err(e) => RetryResult::Err(e),
134            Ok(r) => RetryResult::Success(r),
135        }
136    }
137
138    /// Gets the next sleep duration in milliseconds.
139    fn next_sleep_ms(&self) -> u64 {
140        if let Ok(sleep) = self.gctx.get_env("__CARGO_TEST_FIXED_RETRY_SLEEP_MS") {
141            return sleep.parse().expect("a u64");
142        }
143
144        if self.retries == 1 {
145            let mut rng = rand::rng();
146            INITIAL_RETRY_SLEEP_BASE_MS + rng.random_range(0..INITIAL_RETRY_JITTER_MS)
147        } else {
148            min(
149                ((self.retries - 1) * 3) * 1000 + INITIAL_RETRY_SLEEP_BASE_MS,
150                MAX_RETRY_SLEEP_MS,
151            )
152        }
153    }
154
155    /// Parse the HTTP `Retry-After` header.
156    /// Returns the number of milliseconds to wait before retrying according to the header.
157    fn parse_retry_after(response: &HttpNotSuccessful, now: &jiff::Timestamp) -> Option<u64> {
158        // Only applies to HTTP 429 (too many requests) and 503 (service unavailable).
159        if !matches!(response.code, 429 | 503) {
160            return None;
161        }
162
163        // Extract the Retry-After header value.
164        let retry_after = response
165            .headers
166            .iter()
167            .filter_map(|h| h.split_once(':'))
168            .map(|(k, v)| (k.trim(), v.trim()))
169            .find(|(k, _)| k.eq_ignore_ascii_case("retry-after"))?
170            .1;
171
172        // First option: Retry-After is a positive integer of seconds to wait.
173        if let Ok(delay_secs) = retry_after.parse::<u32>() {
174            return Some(delay_secs as u64 * 1000);
175        }
176
177        // Second option: Retry-After is a future HTTP date string that tells us when to retry.
178        if let Ok(retry_time) = jiff::fmt::rfc2822::parse(retry_after) {
179            let diff_ms = now
180                .until(&retry_time)
181                .unwrap()
182                .total(jiff::Unit::Millisecond)
183                .unwrap();
184            if diff_ms > 0.0 {
185                return Some(diff_ms as u64);
186            }
187        }
188        None
189    }
190}
191
192fn maybe_spurious(err: &Error) -> bool {
193    fn maybe_spurious_curl(curl_err: &curl::Error) -> bool {
194        curl_err.is_couldnt_connect()
195            || curl_err.is_couldnt_resolve_proxy()
196            || curl_err.is_couldnt_resolve_host()
197            || curl_err.is_operation_timedout()
198            || curl_err.is_recv_error()
199            || curl_err.is_send_error()
200            || curl_err.is_http2_error()
201            || curl_err.is_http2_stream_error()
202            || curl_err.is_ssl_connect_error()
203            || curl_err.is_partial_file()
204    }
205    if let Some(async_http_error) = err.downcast_ref::<http_async::Error>() {
206        match async_http_error {
207            http_async::Error::Easy(error) => return maybe_spurious_curl(error),
208            http_async::Error::TooSlow { .. } => return true,
209            http_async::Error::Multi(_) => {}
210            http_async::Error::BadHeader { .. } => {}
211        }
212    }
213    if let Some(git_err) = err.downcast_ref::<git2::Error>() {
214        match git_err.class() {
215            git2::ErrorClass::Net
216            | git2::ErrorClass::Os
217            | git2::ErrorClass::Zlib
218            | git2::ErrorClass::Http => return git_err.code() != git2::ErrorCode::Certificate,
219            _ => (),
220        }
221    }
222    if let Some(curl_err) = err.downcast_ref::<curl::Error>() {
223        if maybe_spurious_curl(curl_err) {
224            return true;
225        }
226    }
227    if let Some(not_200) = err.downcast_ref::<HttpNotSuccessful>() {
228        if 500 <= not_200.code && not_200.code < 600 || not_200.code == 429 {
229            return true;
230        }
231    }
232
233    if let Some(err) = err.downcast_ref::<crate::sources::git::fetch::Error>() {
234        if err.can_retry_lenient() {
235            return true;
236        }
237    }
238
239    if let Some(err) = err.downcast_ref::<GitCliError>() {
240        if err.is_spurious() {
241            return true;
242        }
243    }
244
245    false
246}
247
248/// Wrapper method for network call retry logic.
249///
250/// Retry counts provided by Config object `net.retry`. Config shell outputs
251/// a warning on per retry.
252///
253/// Closure must return a `CargoResult`.
254///
255/// # Examples
256///
257/// ```
258/// # use crate::cargo::util::{CargoResult, GlobalContext};
259/// # let download_something = || return Ok(());
260/// # let gctx = GlobalContext::default().unwrap();
261/// use cargo::util::network;
262/// let cargo_result = network::retry::with_retry(&gctx, || download_something());
263/// ```
264pub fn with_retry<T, F>(gctx: &GlobalContext, mut callback: F) -> CargoResult<T>
265where
266    F: FnMut() -> CargoResult<T>,
267{
268    let mut retry = Retry::new(gctx)?;
269    loop {
270        match retry.r#try(&mut callback) {
271            RetryResult::Success(r) => return Ok(r),
272            RetryResult::Err(e) => return Err(e),
273            RetryResult::Retry(sleep) => std::thread::sleep(Duration::from_millis(sleep)),
274        }
275    }
276}
277
278#[cfg(test)]
279mod tests {
280    use super::maybe_spurious;
281    use gix::error::ErrorExt;
282
283    #[test]
284    fn gitoxide_retry_classification_preserves_cancellation() {
285        let transient = std::io::Error::from(std::io::ErrorKind::ConnectionReset).raise();
286        assert!(maybe_spurious(&anyhow::Error::from(transient)));
287        let permanent = gix::error::validation("invalid refspec").raise();
288        assert!(!maybe_spurious(&anyhow::Error::from(permanent)));
289        let cancelled = std::io::Error::from(std::io::ErrorKind::ConnectionReset)
290            .and_raise(gix::error::cancelled("cancelled"));
291        assert!(!maybe_spurious(&anyhow::Error::from(cancelled)));
292    }
293}
294
295#[test]
296fn with_retry_repeats_the_call_then_works() {
297    use cargo_util_terminal::Shell;
298
299    //Error HTTP codes (5xx) are considered maybe_spurious and will prompt retry
300    let error1 = HttpNotSuccessful {
301        code: 501,
302        url: "Uri".to_string(),
303        ip: None,
304        body: Vec::new(),
305        headers: Vec::new(),
306    }
307    .into();
308    let error2 = HttpNotSuccessful {
309        code: 502,
310        url: "Uri".to_string(),
311        ip: None,
312        body: Vec::new(),
313        headers: Vec::new(),
314    }
315    .into();
316    let mut results: Vec<CargoResult<()>> = vec![Ok(()), Err(error1), Err(error2)];
317    let gctx = GlobalContext::default().unwrap();
318    *gctx.shell() = Shell::from_write(Box::new(Vec::new()));
319    let result = with_retry(&gctx, || results.pop().unwrap());
320    assert!(result.is_ok())
321}
322
323#[test]
324fn with_retry_finds_nested_spurious_errors() {
325    use cargo_util_terminal::Shell;
326
327    //Error HTTP codes (5xx) are considered maybe_spurious and will prompt retry
328    //String error messages are not considered spurious
329    let error1 = anyhow::Error::from(HttpNotSuccessful {
330        code: 501,
331        url: "Uri".to_string(),
332        ip: None,
333        body: Vec::new(),
334        headers: Vec::new(),
335    });
336    let error1 = anyhow::Error::from(error1.context("a non-spurious wrapping err"));
337    let error2 = anyhow::Error::from(HttpNotSuccessful {
338        code: 502,
339        url: "Uri".to_string(),
340        ip: None,
341        body: Vec::new(),
342        headers: Vec::new(),
343    });
344    let error2 = anyhow::Error::from(error2.context("a second chained error"));
345    let mut results: Vec<CargoResult<()>> = vec![Ok(()), Err(error1), Err(error2)];
346    let gctx = GlobalContext::default().unwrap();
347    *gctx.shell() = Shell::from_write(Box::new(Vec::new()));
348    let result = with_retry(&gctx, || results.pop().unwrap());
349    assert!(result.is_ok())
350}
351
352#[test]
353fn default_retry_schedule() {
354    use cargo_util_terminal::Shell;
355
356    let spurious = || -> CargoResult<()> {
357        Err(anyhow::Error::from(HttpNotSuccessful {
358            code: 500,
359            url: "Uri".to_string(),
360            ip: None,
361            body: Vec::new(),
362            headers: Vec::new(),
363        }))
364    };
365    let gctx = GlobalContext::default().unwrap();
366    *gctx.shell() = Shell::from_write(Box::new(Vec::new()));
367    let mut retry = Retry::new(&gctx).unwrap();
368    match retry.r#try(|| spurious()) {
369        RetryResult::Retry(sleep) => {
370            assert!(
371                sleep >= INITIAL_RETRY_SLEEP_BASE_MS
372                    && sleep < INITIAL_RETRY_SLEEP_BASE_MS + INITIAL_RETRY_JITTER_MS
373            );
374        }
375        _ => panic!("unexpected non-retry"),
376    }
377    match retry.r#try(|| spurious()) {
378        RetryResult::Retry(sleep) => assert_eq!(sleep, 3500),
379        _ => panic!("unexpected non-retry"),
380    }
381    match retry.r#try(|| spurious()) {
382        RetryResult::Retry(sleep) => assert_eq!(sleep, 6500),
383        _ => panic!("unexpected non-retry"),
384    }
385    match retry.r#try(|| spurious()) {
386        RetryResult::Err(_) => {}
387        _ => panic!("unexpected non-retry"),
388    }
389}
390
391#[test]
392fn curle_http2_stream_is_spurious() {
393    let code = curl_sys::CURLE_HTTP2_STREAM;
394    let err = curl::Error::new(code);
395    assert!(maybe_spurious(&err.into()));
396}
397
398#[test]
399fn retry_after_parsing() {
400    use cargo_util_terminal::Shell;
401    fn spurious(code: u32, header: &str) -> HttpNotSuccessful {
402        HttpNotSuccessful {
403            code,
404            url: "Uri".to_string(),
405            ip: None,
406            body: Vec::new(),
407            headers: vec![header.to_string()],
408        }
409    }
410
411    // Start of year 2025.
412    let now = jiff::Timestamp::new(1735689600, 0).unwrap();
413    let headers = spurious(429, "Retry-After: 10");
414    assert_eq!(Retry::parse_retry_after(&headers, &now), Some(10_000));
415    let headers = spurious(429, "retry-after: Wed, 01 Jan 2025 00:00:10 GMT");
416    let actual = Retry::parse_retry_after(&headers, &now).unwrap();
417    assert_eq!(10000, actual);
418
419    let headers = spurious(429, "Content-Type: text/html");
420    assert_eq!(Retry::parse_retry_after(&headers, &now), None);
421
422    let headers = spurious(429, "retry-after: Fri, 01 Jan 2000 00:00:00 GMT");
423    assert_eq!(Retry::parse_retry_after(&headers, &now), None);
424
425    let headers = spurious(429, "retry-after: -1");
426    assert_eq!(Retry::parse_retry_after(&headers, &now), None);
427
428    let headers = spurious(400, "retry-after: 1");
429    assert_eq!(Retry::parse_retry_after(&headers, &now), None);
430
431    let gctx = GlobalContext::default().unwrap();
432    *gctx.shell() = Shell::from_write(Box::new(Vec::new()));
433    let mut retry = Retry::new(&gctx).unwrap();
434    match retry
435        .r#try(|| -> CargoResult<()> { Err(anyhow::Error::from(spurious(429, "Retry-After: 7"))) })
436    {
437        RetryResult::Retry(sleep) => assert_eq!(sleep, 7_000),
438        _ => panic!("unexpected non-retry"),
439    }
440}
441
442#[test]
443fn git_cli_error_spurious() {
444    let error = GitCliError::new(Error::msg("test-git-cli-error")).spurious(false);
445    assert!(!maybe_spurious(&error.into()));
446
447    let error = GitCliError::new(Error::msg("test-git-cli-error")).spurious(true);
448    assert!(maybe_spurious(&error.into()));
449}
cargo/util/network/retry.rs

cargo/util/network/
retry.rs
