std/io/buffered/
bufreader.rs

1mod buffer;
2
3use buffer::Buffer;
4
5use crate::fmt;
6use crate::io::{
7    self, BorrowedCursor, BufRead, DEFAULT_BUF_SIZE, IoSliceMut, Read, Seek, SeekFrom, SizeHint,
8    SpecReadByte, uninlined_slow_read_byte,
9};
10
11/// The `BufReader<R>` struct adds buffering to any reader.
12///
13/// It can be excessively inefficient to work directly with a [`Read`] instance.
14/// For example, every call to [`read`][`TcpStream::read`] on [`TcpStream`]
15/// results in a system call. A `BufReader<R>` performs large, infrequent reads on
16/// the underlying [`Read`] and maintains an in-memory buffer of the results.
17///
18/// `BufReader<R>` can improve the speed of programs that make *small* and
19/// *repeated* read calls to the same file or network socket. It does not
20/// help when reading very large amounts at once, or reading just one or a few
21/// times. It also provides no advantage when reading from a source that is
22/// already in memory, like a <code>[Vec]\<u8></code>.
23///
24/// When the `BufReader<R>` is dropped, the contents of its buffer will be
25/// discarded. Creating multiple instances of a `BufReader<R>` on the same
26/// stream can cause data loss. Reading from the underlying reader after
27/// unwrapping the `BufReader<R>` with [`BufReader::into_inner`] can also cause
28/// data loss.
29///
30/// [`TcpStream::read`]: crate::net::TcpStream::read
31/// [`TcpStream`]: crate::net::TcpStream
32///
33/// # Examples
34///
35/// ```no_run
36/// use std::io::prelude::*;
37/// use std::io::BufReader;
38/// use std::fs::File;
39///
40/// fn main() -> std::io::Result<()> {
41///     let f = File::open("log.txt")?;
42///     let mut reader = BufReader::new(f);
43///
44///     let mut line = String::new();
45///     let len = reader.read_line(&mut line)?;
46///     println!("First line is {len} bytes long");
47///     Ok(())
48/// }
49/// ```
50#[stable(feature = "rust1", since = "1.0.0")]
51pub struct BufReader<R: ?Sized> {
52    buf: Buffer,
53    inner: R,
54}
55
56impl<R: Read> BufReader<R> {
57    /// Creates a new `BufReader<R>` with a default buffer capacity. The default is currently 8 KiB,
58    /// but may change in the future.
59    ///
60    /// # Examples
61    ///
62    /// ```no_run
63    /// use std::io::BufReader;
64    /// use std::fs::File;
65    ///
66    /// fn main() -> std::io::Result<()> {
67    ///     let f = File::open("log.txt")?;
68    ///     let reader = BufReader::new(f);
69    ///     Ok(())
70    /// }
71    /// ```
72    #[stable(feature = "rust1", since = "1.0.0")]
73    pub fn new(inner: R) -> BufReader<R> {
74        BufReader::with_capacity(DEFAULT_BUF_SIZE, inner)
75    }
76
77    pub(crate) fn try_new_buffer() -> io::Result<Buffer> {
78        Buffer::try_with_capacity(DEFAULT_BUF_SIZE)
79    }
80
81    pub(crate) fn with_buffer(inner: R, buf: Buffer) -> Self {
82        Self { inner, buf }
83    }
84
85    /// Creates a new `BufReader<R>` with the specified buffer capacity.
86    ///
87    /// # Examples
88    ///
89    /// Creating a buffer with ten bytes of capacity:
90    ///
91    /// ```no_run
92    /// use std::io::BufReader;
93    /// use std::fs::File;
94    ///
95    /// fn main() -> std::io::Result<()> {
96    ///     let f = File::open("log.txt")?;
97    ///     let reader = BufReader::with_capacity(10, f);
98    ///     Ok(())
99    /// }
100    /// ```
101    #[stable(feature = "rust1", since = "1.0.0")]
102    pub fn with_capacity(capacity: usize, inner: R) -> BufReader<R> {
103        BufReader { inner, buf: Buffer::with_capacity(capacity) }
104    }
105}
106
107impl<R: Read + ?Sized> BufReader<R> {
108    /// Attempt to look ahead `n` bytes.
109    ///
110    /// `n` must be less than or equal to `capacity`.
111    ///
112    /// the returned slice may be less than `n` bytes long if
113    /// end of file is reached.
114    ///
115    /// ## Examples
116    ///
117    /// ```rust
118    /// #![feature(bufreader_peek)]
119    /// use std::io::{Read, BufReader};
120    ///
121    /// let mut bytes = &b"oh, hello"[..];
122    /// let mut rdr = BufReader::with_capacity(6, &mut bytes);
123    /// assert_eq!(rdr.peek(2).unwrap(), b"oh");
124    /// let mut buf = [0; 4];
125    /// rdr.read(&mut buf[..]).unwrap();
126    /// assert_eq!(&buf, b"oh, ");
127    /// assert_eq!(rdr.peek(2).unwrap(), b"he");
128    /// let mut s = String::new();
129    /// rdr.read_to_string(&mut s).unwrap();
130    /// assert_eq!(&s, "hello");
131    /// assert_eq!(rdr.peek(1).unwrap().len(), 0);
132    /// ```
133    #[unstable(feature = "bufreader_peek", issue = "128405")]
134    pub fn peek(&mut self, n: usize) -> io::Result<&[u8]> {
135        assert!(n <= self.capacity());
136        while n > self.buf.buffer().len() {
137            if self.buf.pos() > 0 {
138                self.buf.backshift();
139            }
140            let new = self.buf.read_more(&mut self.inner)?;
141            if new == 0 {
142                // end of file, no more bytes to read
143                return Ok(&self.buf.buffer()[..]);
144            }
145            debug_assert_eq!(self.buf.pos(), 0);
146        }
147        Ok(&self.buf.buffer()[..n])
148    }
149}
150
151impl<R: ?Sized> BufReader<R> {
152    /// Gets a reference to the underlying reader.
153    ///
154    /// It is inadvisable to directly read from the underlying reader.
155    ///
156    /// # Examples
157    ///
158    /// ```no_run
159    /// use std::io::BufReader;
160    /// use std::fs::File;
161    ///
162    /// fn main() -> std::io::Result<()> {
163    ///     let f1 = File::open("log.txt")?;
164    ///     let reader = BufReader::new(f1);
165    ///
166    ///     let f2 = reader.get_ref();
167    ///     Ok(())
168    /// }
169    /// ```
170    #[stable(feature = "rust1", since = "1.0.0")]
171    pub fn get_ref(&self) -> &R {
172        &self.inner
173    }
174
175    /// Gets a mutable reference to the underlying reader.
176    ///
177    /// It is inadvisable to directly read from the underlying reader.
178    ///
179    /// # Examples
180    ///
181    /// ```no_run
182    /// use std::io::BufReader;
183    /// use std::fs::File;
184    ///
185    /// fn main() -> std::io::Result<()> {
186    ///     let f1 = File::open("log.txt")?;
187    ///     let mut reader = BufReader::new(f1);
188    ///
189    ///     let f2 = reader.get_mut();
190    ///     Ok(())
191    /// }
192    /// ```
193    #[stable(feature = "rust1", since = "1.0.0")]
194    pub fn get_mut(&mut self) -> &mut R {
195        &mut self.inner
196    }
197
198    /// Returns a reference to the internally buffered data.
199    ///
200    /// Unlike [`fill_buf`], this will not attempt to fill the buffer if it is empty.
201    ///
202    /// [`fill_buf`]: BufRead::fill_buf
203    ///
204    /// # Examples
205    ///
206    /// ```no_run
207    /// use std::io::{BufReader, BufRead};
208    /// use std::fs::File;
209    ///
210    /// fn main() -> std::io::Result<()> {
211    ///     let f = File::open("log.txt")?;
212    ///     let mut reader = BufReader::new(f);
213    ///     assert!(reader.buffer().is_empty());
214    ///
215    ///     if reader.fill_buf()?.len() > 0 {
216    ///         assert!(!reader.buffer().is_empty());
217    ///     }
218    ///     Ok(())
219    /// }
220    /// ```
221    #[stable(feature = "bufreader_buffer", since = "1.37.0")]
222    pub fn buffer(&self) -> &[u8] {
223        self.buf.buffer()
224    }
225
226    /// Returns the number of bytes the internal buffer can hold at once.
227    ///
228    /// # Examples
229    ///
230    /// ```no_run
231    /// use std::io::{BufReader, BufRead};
232    /// use std::fs::File;
233    ///
234    /// fn main() -> std::io::Result<()> {
235    ///     let f = File::open("log.txt")?;
236    ///     let mut reader = BufReader::new(f);
237    ///
238    ///     let capacity = reader.capacity();
239    ///     let buffer = reader.fill_buf()?;
240    ///     assert!(buffer.len() <= capacity);
241    ///     Ok(())
242    /// }
243    /// ```
244    #[stable(feature = "buffered_io_capacity", since = "1.46.0")]
245    pub fn capacity(&self) -> usize {
246        self.buf.capacity()
247    }
248
249    /// Unwraps this `BufReader<R>`, returning the underlying reader.
250    ///
251    /// Note that any leftover data in the internal buffer is lost. Therefore,
252    /// a following read from the underlying reader may lead to data loss.
253    ///
254    /// # Examples
255    ///
256    /// ```no_run
257    /// use std::io::BufReader;
258    /// use std::fs::File;
259    ///
260    /// fn main() -> std::io::Result<()> {
261    ///     let f1 = File::open("log.txt")?;
262    ///     let reader = BufReader::new(f1);
263    ///
264    ///     let f2 = reader.into_inner();
265    ///     Ok(())
266    /// }
267    /// ```
268    #[stable(feature = "rust1", since = "1.0.0")]
269    pub fn into_inner(self) -> R
270    where
271        R: Sized,
272    {
273        self.inner
274    }
275
276    /// Invalidates all data in the internal buffer.
277    #[inline]
278    pub(in crate::io) fn discard_buffer(&mut self) {
279        self.buf.discard_buffer()
280    }
281}
282
283// This is only used by a test which asserts that the initialization-tracking is correct.
284#[cfg(test)]
285impl<R: ?Sized> BufReader<R> {
286    #[allow(missing_docs)]
287    pub fn initialized(&self) -> usize {
288        self.buf.initialized()
289    }
290}
291
292impl<R: ?Sized + Seek> BufReader<R> {
293    /// Seeks relative to the current position. If the new position lies within the buffer,
294    /// the buffer will not be flushed, allowing for more efficient seeks.
295    /// This method does not return the location of the underlying reader, so the caller
296    /// must track this information themselves if it is required.
297    #[stable(feature = "bufreader_seek_relative", since = "1.53.0")]
298    pub fn seek_relative(&mut self, offset: i64) -> io::Result<()> {
299        let pos = self.buf.pos() as u64;
300        if offset < 0 {
301            if let Some(_) = pos.checked_sub((-offset) as u64) {
302                self.buf.unconsume((-offset) as usize);
303                return Ok(());
304            }
305        } else if let Some(new_pos) = pos.checked_add(offset as u64) {
306            if new_pos <= self.buf.filled() as u64 {
307                self.buf.consume(offset as usize);
308                return Ok(());
309            }
310        }
311
312        self.seek(SeekFrom::Current(offset)).map(drop)
313    }
314}
315
316impl<R> SpecReadByte for BufReader<R>
317where
318    Self: Read,
319{
320    #[inline]
321    fn spec_read_byte(&mut self) -> Option<io::Result<u8>> {
322        let mut byte = 0;
323        if self.buf.consume_with(1, |claimed| byte = claimed[0]) {
324            return Some(Ok(byte));
325        }
326
327        // Fallback case, only reached once per buffer refill.
328        uninlined_slow_read_byte(self)
329    }
330}
331
332#[stable(feature = "rust1", since = "1.0.0")]
333impl<R: ?Sized + Read> Read for BufReader<R> {
334    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
335        // If we don't have any buffered data and we're doing a massive read
336        // (larger than our internal buffer), bypass our internal buffer
337        // entirely.
338        if self.buf.pos() == self.buf.filled() && buf.len() >= self.capacity() {
339            self.discard_buffer();
340            return self.inner.read(buf);
341        }
342        let mut rem = self.fill_buf()?;
343        let nread = rem.read(buf)?;
344        self.consume(nread);
345        Ok(nread)
346    }
347
348    fn read_buf(&mut self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> {
349        // If we don't have any buffered data and we're doing a massive read
350        // (larger than our internal buffer), bypass our internal buffer
351        // entirely.
352        if self.buf.pos() == self.buf.filled() && cursor.capacity() >= self.capacity() {
353            self.discard_buffer();
354            return self.inner.read_buf(cursor);
355        }
356
357        let prev = cursor.written();
358
359        let mut rem = self.fill_buf()?;
360        rem.read_buf(cursor.reborrow())?; // actually never fails
361
362        self.consume(cursor.written() - prev); //slice impl of read_buf known to never unfill buf
363
364        Ok(())
365    }
366
367    // Small read_exacts from a BufReader are extremely common when used with a deserializer.
368    // The default implementation calls read in a loop, which results in surprisingly poor code
369    // generation for the common path where the buffer has enough bytes to fill the passed-in
370    // buffer.
371    fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
372        if self.buf.consume_with(buf.len(), |claimed| buf.copy_from_slice(claimed)) {
373            return Ok(());
374        }
375
376        crate::io::default_read_exact(self, buf)
377    }
378
379    fn read_buf_exact(&mut self, mut cursor: BorrowedCursor<'_>) -> io::Result<()> {
380        if self.buf.consume_with(cursor.capacity(), |claimed| cursor.append(claimed)) {
381            return Ok(());
382        }
383
384        crate::io::default_read_buf_exact(self, cursor)
385    }
386
387    fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result<usize> {
388        let total_len = bufs.iter().map(|b| b.len()).sum::<usize>();
389        if self.buf.pos() == self.buf.filled() && total_len >= self.capacity() {
390            self.discard_buffer();
391            return self.inner.read_vectored(bufs);
392        }
393        let mut rem = self.fill_buf()?;
394        let nread = rem.read_vectored(bufs)?;
395
396        self.consume(nread);
397        Ok(nread)
398    }
399
400    fn is_read_vectored(&self) -> bool {
401        self.inner.is_read_vectored()
402    }
403
404    // The inner reader might have an optimized `read_to_end`. Drain our buffer and then
405    // delegate to the inner implementation.
406    fn read_to_end(&mut self, buf: &mut Vec<u8>) -> io::Result<usize> {
407        let inner_buf = self.buffer();
408        buf.try_reserve(inner_buf.len())?;
409        buf.extend_from_slice(inner_buf);
410        let nread = inner_buf.len();
411        self.discard_buffer();
412        Ok(nread + self.inner.read_to_end(buf)?)
413    }
414
415    // The inner reader might have an optimized `read_to_end`. Drain our buffer and then
416    // delegate to the inner implementation.
417    fn read_to_string(&mut self, buf: &mut String) -> io::Result<usize> {
418        // In the general `else` case below we must read bytes into a side buffer, check
419        // that they are valid UTF-8, and then append them to `buf`. This requires a
420        // potentially large memcpy.
421        //
422        // If `buf` is empty--the most common case--we can leverage `append_to_string`
423        // to read directly into `buf`'s internal byte buffer, saving an allocation and
424        // a memcpy.
425        if buf.is_empty() {
426            // `append_to_string`'s safety relies on the buffer only being appended to since
427            // it only checks the UTF-8 validity of new data. If there were existing content in
428            // `buf` then an untrustworthy reader (i.e. `self.inner`) could not only append
429            // bytes but also modify existing bytes and render them invalid. On the other hand,
430            // if `buf` is empty then by definition any writes must be appends and
431            // `append_to_string` will validate all of the new bytes.
432            unsafe { crate::io::append_to_string(buf, |b| self.read_to_end(b)) }
433        } else {
434            // We cannot append our byte buffer directly onto the `buf` String as there could
435            // be an incomplete UTF-8 sequence that has only been partially read. We must read
436            // everything into a side buffer first and then call `from_utf8` on the complete
437            // buffer.
438            let mut bytes = Vec::new();
439            self.read_to_end(&mut bytes)?;
440            let string = crate::str::from_utf8(&bytes).map_err(|_| io::Error::INVALID_UTF8)?;
441            *buf += string;
442            Ok(string.len())
443        }
444    }
445}
446
447#[stable(feature = "rust1", since = "1.0.0")]
448impl<R: ?Sized + Read> BufRead for BufReader<R> {
449    fn fill_buf(&mut self) -> io::Result<&[u8]> {
450        self.buf.fill_buf(&mut self.inner)
451    }
452
453    fn consume(&mut self, amt: usize) {
454        self.buf.consume(amt)
455    }
456}
457
458#[stable(feature = "rust1", since = "1.0.0")]
459impl<R> fmt::Debug for BufReader<R>
460where
461    R: ?Sized + fmt::Debug,
462{
463    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
464        fmt.debug_struct("BufReader")
465            .field("reader", &&self.inner)
466            .field(
467                "buffer",
468                &format_args!("{}/{}", self.buf.filled() - self.buf.pos(), self.capacity()),
469            )
470            .finish()
471    }
472}
473
474#[stable(feature = "rust1", since = "1.0.0")]
475impl<R: ?Sized + Seek> Seek for BufReader<R> {
476    /// Seek to an offset, in bytes, in the underlying reader.
477    ///
478    /// The position used for seeking with <code>[SeekFrom::Current]\(_)</code> is the
479    /// position the underlying reader would be at if the `BufReader<R>` had no
480    /// internal buffer.
481    ///
482    /// Seeking always discards the internal buffer, even if the seek position
483    /// would otherwise fall within it. This guarantees that calling
484    /// [`BufReader::into_inner()`] immediately after a seek yields the underlying reader
485    /// at the same position.
486    ///
487    /// To seek without discarding the internal buffer, use [`BufReader::seek_relative`].
488    ///
489    /// See [`std::io::Seek`] for more details.
490    ///
491    /// Note: In the edge case where you're seeking with <code>[SeekFrom::Current]\(n)</code>
492    /// where `n` minus the internal buffer length overflows an `i64`, two
493    /// seeks will be performed instead of one. If the second seek returns
494    /// [`Err`], the underlying reader will be left at the same position it would
495    /// have if you called `seek` with <code>[SeekFrom::Current]\(0)</code>.
496    ///
497    /// [`std::io::Seek`]: Seek
498    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
499        let result: u64;
500        if let SeekFrom::Current(n) = pos {
501            let remainder = (self.buf.filled() - self.buf.pos()) as i64;
502            // it should be safe to assume that remainder fits within an i64 as the alternative
503            // means we managed to allocate 8 exbibytes and that's absurd.
504            // But it's not out of the realm of possibility for some weird underlying reader to
505            // support seeking by i64::MIN so we need to handle underflow when subtracting
506            // remainder.
507            if let Some(offset) = n.checked_sub(remainder) {
508                result = self.inner.seek(SeekFrom::Current(offset))?;
509            } else {
510                // seek backwards by our remainder, and then by the offset
511                self.inner.seek(SeekFrom::Current(-remainder))?;
512                self.discard_buffer();
513                result = self.inner.seek(SeekFrom::Current(n))?;
514            }
515        } else {
516            // Seeking with Start/End doesn't care about our buffer length.
517            result = self.inner.seek(pos)?;
518        }
519        self.discard_buffer();
520        Ok(result)
521    }
522
523    /// Returns the current seek position from the start of the stream.
524    ///
525    /// The value returned is equivalent to `self.seek(SeekFrom::Current(0))`
526    /// but does not flush the internal buffer. Due to this optimization the
527    /// function does not guarantee that calling `.into_inner()` immediately
528    /// afterwards will yield the underlying reader at the same position. Use
529    /// [`BufReader::seek`] instead if you require that guarantee.
530    ///
531    /// # Panics
532    ///
533    /// This function will panic if the position of the inner reader is smaller
534    /// than the amount of buffered data. That can happen if the inner reader
535    /// has an incorrect implementation of [`Seek::stream_position`], or if the
536    /// position has gone out of sync due to calling [`Seek::seek`] directly on
537    /// the underlying reader.
538    ///
539    /// # Example
540    ///
541    /// ```no_run
542    /// use std::{
543    ///     io::{self, BufRead, BufReader, Seek},
544    ///     fs::File,
545    /// };
546    ///
547    /// fn main() -> io::Result<()> {
548    ///     let mut f = BufReader::new(File::open("foo.txt")?);
549    ///
550    ///     let before = f.stream_position()?;
551    ///     f.read_line(&mut String::new())?;
552    ///     let after = f.stream_position()?;
553    ///
554    ///     println!("The first line was {} bytes long", after - before);
555    ///     Ok(())
556    /// }
557    /// ```
558    fn stream_position(&mut self) -> io::Result<u64> {
559        let remainder = (self.buf.filled() - self.buf.pos()) as u64;
560        self.inner.stream_position().map(|pos| {
561            pos.checked_sub(remainder).expect(
562                "overflow when subtracting remaining buffer size from inner stream position",
563            )
564        })
565    }
566
567    /// Seeks relative to the current position.
568    ///
569    /// If the new position lies within the buffer, the buffer will not be
570    /// flushed, allowing for more efficient seeks. This method does not return
571    /// the location of the underlying reader, so the caller must track this
572    /// information themselves if it is required.
573    fn seek_relative(&mut self, offset: i64) -> io::Result<()> {
574        self.seek_relative(offset)
575    }
576}
577
578impl<T: ?Sized> SizeHint for BufReader<T> {
579    #[inline]
580    fn lower_bound(&self) -> usize {
581        SizeHint::lower_bound(self.get_ref()) + self.buffer().len()
582    }
583
584    #[inline]
585    fn upper_bound(&self) -> Option<usize> {
586        SizeHint::upper_bound(self.get_ref()).and_then(|up| self.buffer().len().checked_add(up))
587    }
588}