1#[cfg(not(any(all(target_os = "linux", target_env = "gnu"), target_os = "hurd")))]
46use libc::sendfile as sendfile64;
47#[cfg(any(all(target_os = "linux", target_env = "gnu"), target_os = "hurd"))]
48use libc::sendfile64;
49use libc::{EBADF, EINVAL, ENOSYS, EOPNOTSUPP, EOVERFLOW, EPERM, EXDEV};
50
51use crate::cmp::min;
52use crate::fs::{File, Metadata};
53use crate::io::copy::generic_copy;
54use crate::io::{
55 BufRead, BufReader, BufWriter, Error, PipeReader, PipeWriter, Read, Result, StderrLock,
56 StdinLock, StdoutLock, Take, Write,
57};
58use crate::mem::ManuallyDrop;
59use crate::net::TcpStream;
60use crate::os::unix::fs::FileTypeExt;
61use crate::os::unix::io::{AsRawFd, FromRawFd, RawFd};
62use crate::os::unix::net::UnixStream;
63use crate::process::{ChildStderr, ChildStdin, ChildStdout};
64use crate::ptr;
65use crate::sync::atomic::{AtomicBool, AtomicU8, Ordering};
66use crate::sys::cvt;
67use crate::sys::fs::CachedFileMetadata;
68use crate::sys::weak::syscall;
69
70#[cfg(test)]
71mod tests;
72
73pub(crate) fn copy_spec<R: Read + ?Sized, W: Write + ?Sized>(
74 read: &mut R,
75 write: &mut W,
76) -> Result<u64> {
77 let copier = Copier { read, write };
78 SpecCopy::copy(copier)
79}
80
81enum FdMeta {
87 Metadata(Metadata),
88 Socket,
89 Pipe,
90 NoneObtained,
92}
93
94#[derive(PartialEq)]
95enum FdHandle {
96 Input,
97 Output,
98}
99
100impl FdMeta {
101 fn maybe_fifo(&self) -> bool {
102 match self {
103 FdMeta::Metadata(meta) => meta.file_type().is_fifo(),
104 FdMeta::Socket => false,
105 FdMeta::Pipe => true,
106 FdMeta::NoneObtained => true,
107 }
108 }
109
110 fn potential_sendfile_source(&self) -> bool {
111 match self {
112 FdMeta::Metadata(meta)
116 if meta.file_type().is_file() && meta.len() > 0
117 || meta.file_type().is_block_device() =>
118 {
119 true
120 }
121 _ => false,
122 }
123 }
124
125 fn copy_file_range_candidate(&self, f: FdHandle) -> bool {
126 match self {
127 FdMeta::Metadata(meta) if f == FdHandle::Input && meta.is_file() && meta.len() > 0 => {
130 true
131 }
132 FdMeta::Metadata(meta) if f == FdHandle::Output && meta.is_file() => true,
133 _ => false,
134 }
135 }
136}
137
138fn safe_kernel_copy(source: &FdMeta, sink: &FdMeta) -> bool {
148 match (source, sink) {
149 (FdMeta::Socket, _) => true,
154 (FdMeta::Pipe, _) => true,
155 (FdMeta::Metadata(meta), _)
156 if meta.file_type().is_fifo() || meta.file_type().is_socket() =>
157 {
158 true
159 }
160 (_, FdMeta::Metadata(meta))
163 if !meta.file_type().is_fifo() && !meta.file_type().is_socket() =>
164 {
165 true
166 }
167 _ => false,
168 }
169}
170
171struct CopyParams(FdMeta, Option<RawFd>);
172
173struct Copier<'a, 'b, R: Read + ?Sized, W: Write + ?Sized> {
174 read: &'a mut R,
175 write: &'b mut W,
176}
177
178trait SpecCopy {
179 fn copy(self) -> Result<u64>;
180}
181
182impl<R: Read + ?Sized, W: Write + ?Sized> SpecCopy for Copier<'_, '_, R, W> {
183 default fn copy(self) -> Result<u64> {
184 generic_copy(self.read, self.write)
185 }
186}
187
188impl<R: CopyRead, W: CopyWrite> SpecCopy for Copier<'_, '_, R, W> {
189 fn copy(self) -> Result<u64> {
190 let (reader, writer) = (self.read, self.write);
191 let r_cfg = reader.properties();
192 let w_cfg = writer.properties();
193
194 let mut flush = || -> Result<u64> {
196 let bytes = reader.drain_to(writer, u64::MAX)?;
197 writer.flush()?;
199 Ok(bytes)
200 };
201
202 let mut written = 0u64;
203
204 if let (CopyParams(input_meta, Some(readfd)), CopyParams(output_meta, Some(writefd))) =
205 (r_cfg, w_cfg)
206 {
207 written += flush()?;
208 let max_write = reader.min_limit();
209
210 if input_meta.copy_file_range_candidate(FdHandle::Input)
211 && output_meta.copy_file_range_candidate(FdHandle::Output)
212 {
213 let result = copy_regular_files(readfd, writefd, max_write);
214 result.update_take(reader);
215
216 match result {
217 CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written),
218 CopyResult::Error(e, _) => return Err(e),
219 CopyResult::Fallback(bytes) => written += bytes,
220 }
221 }
222
223 if input_meta.potential_sendfile_source() && safe_kernel_copy(&input_meta, &output_meta)
229 {
230 let result = sendfile_splice(SpliceMode::Sendfile, readfd, writefd, max_write);
231 result.update_take(reader);
232
233 match result {
234 CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written),
235 CopyResult::Error(e, _) => return Err(e),
236 CopyResult::Fallback(bytes) => written += bytes,
237 }
238 }
239
240 if (input_meta.maybe_fifo() || output_meta.maybe_fifo())
241 && safe_kernel_copy(&input_meta, &output_meta)
242 {
243 let result = sendfile_splice(SpliceMode::Splice, readfd, writefd, max_write);
244 result.update_take(reader);
245
246 match result {
247 CopyResult::Ended(bytes_copied) => return Ok(bytes_copied + written),
248 CopyResult::Error(e, _) => return Err(e),
249 CopyResult::Fallback(0) => { }
250 CopyResult::Fallback(_) => {
251 unreachable!("splice should not return > 0 bytes on the fallback path")
252 }
253 }
254 }
255 }
256
257 match generic_copy(reader, writer) {
259 Ok(bytes) => Ok(bytes + written),
260 err => err,
261 }
262 }
263}
264
265#[rustc_specialization_trait]
266trait CopyRead: Read {
267 fn drain_to<W: Write>(&mut self, _writer: &mut W, _limit: u64) -> Result<u64> {
275 Ok(0)
276 }
277
278 fn taken(&mut self, _bytes: u64) {}
280
281 fn min_limit(&self) -> u64 {
286 u64::MAX
287 }
288
289 fn properties(&self) -> CopyParams;
291}
292
293#[rustc_specialization_trait]
294trait CopyWrite: Write {
295 fn properties(&self) -> CopyParams;
297}
298
299impl<T> CopyRead for &mut T
300where
301 T: CopyRead,
302{
303 fn drain_to<W: Write>(&mut self, writer: &mut W, limit: u64) -> Result<u64> {
304 (**self).drain_to(writer, limit)
305 }
306
307 fn taken(&mut self, bytes: u64) {
308 (**self).taken(bytes);
309 }
310
311 fn min_limit(&self) -> u64 {
312 (**self).min_limit()
313 }
314
315 fn properties(&self) -> CopyParams {
316 (**self).properties()
317 }
318}
319
320impl<T> CopyWrite for &mut T
321where
322 T: CopyWrite,
323{
324 fn properties(&self) -> CopyParams {
325 (**self).properties()
326 }
327}
328
329impl CopyRead for File {
330 fn properties(&self) -> CopyParams {
331 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
332 }
333}
334
335impl CopyRead for &File {
336 fn properties(&self) -> CopyParams {
337 CopyParams(fd_to_meta(*self), Some(self.as_raw_fd()))
338 }
339}
340
341impl CopyWrite for File {
342 fn properties(&self) -> CopyParams {
343 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
344 }
345}
346
347impl CopyWrite for &File {
348 fn properties(&self) -> CopyParams {
349 CopyParams(fd_to_meta(*self), Some(self.as_raw_fd()))
350 }
351}
352
353impl CopyRead for TcpStream {
354 fn properties(&self) -> CopyParams {
355 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
357 }
358}
359
360impl CopyRead for &TcpStream {
361 fn properties(&self) -> CopyParams {
362 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
364 }
365}
366
367impl CopyWrite for TcpStream {
368 fn properties(&self) -> CopyParams {
369 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
371 }
372}
373
374impl CopyWrite for &TcpStream {
375 fn properties(&self) -> CopyParams {
376 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
378 }
379}
380
381impl CopyRead for UnixStream {
382 fn properties(&self) -> CopyParams {
383 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
385 }
386}
387
388impl CopyRead for &UnixStream {
389 fn properties(&self) -> CopyParams {
390 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
392 }
393}
394
395impl CopyWrite for UnixStream {
396 fn properties(&self) -> CopyParams {
397 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
399 }
400}
401
402impl CopyWrite for &UnixStream {
403 fn properties(&self) -> CopyParams {
404 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
406 }
407}
408
409impl CopyRead for PipeReader {
410 fn properties(&self) -> CopyParams {
411 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
412 }
413}
414
415impl CopyRead for &PipeReader {
416 fn properties(&self) -> CopyParams {
417 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
418 }
419}
420
421impl CopyWrite for PipeWriter {
422 fn properties(&self) -> CopyParams {
423 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
424 }
425}
426
427impl CopyWrite for &PipeWriter {
428 fn properties(&self) -> CopyParams {
429 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
430 }
431}
432
433impl CopyWrite for ChildStdin {
434 fn properties(&self) -> CopyParams {
435 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
436 }
437}
438
439impl CopyRead for ChildStdout {
440 fn properties(&self) -> CopyParams {
441 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
442 }
443}
444
445impl CopyRead for ChildStderr {
446 fn properties(&self) -> CopyParams {
447 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
448 }
449}
450
451impl CopyRead for StdinLock<'_> {
452 fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
453 let buf_reader = self.as_mut_buf();
454 let buf = buf_reader.buffer();
455 let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))];
456 let bytes_drained = buf.len();
457 writer.write_all(buf)?;
458 buf_reader.consume(bytes_drained);
459
460 Ok(bytes_drained as u64)
461 }
462
463 fn properties(&self) -> CopyParams {
464 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
465 }
466}
467
468impl CopyWrite for StdoutLock<'_> {
469 fn properties(&self) -> CopyParams {
470 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
471 }
472}
473
474impl CopyWrite for StderrLock<'_> {
475 fn properties(&self) -> CopyParams {
476 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
477 }
478}
479
480impl<T: CopyRead> CopyRead for Take<T> {
481 fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
482 let local_limit = self.limit();
483 let combined_limit = min(outer_limit, local_limit);
484 let bytes_drained = self.get_mut().drain_to(writer, combined_limit)?;
485 self.set_limit(local_limit - bytes_drained);
487
488 Ok(bytes_drained)
489 }
490
491 fn taken(&mut self, bytes: u64) {
492 self.set_limit(self.limit() - bytes);
493 self.get_mut().taken(bytes);
494 }
495
496 fn min_limit(&self) -> u64 {
497 min(Take::limit(self), self.get_ref().min_limit())
498 }
499
500 fn properties(&self) -> CopyParams {
501 self.get_ref().properties()
502 }
503}
504
505impl<T: ?Sized + CopyRead> CopyRead for BufReader<T> {
506 fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
507 let buf = self.buffer();
508 let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))];
509 let bytes = buf.len();
510 writer.write_all(buf)?;
511 self.consume(bytes);
512
513 let remaining = outer_limit - bytes as u64;
514
515 let inner_bytes = self.get_mut().drain_to(writer, remaining)?;
517
518 Ok(bytes as u64 + inner_bytes)
519 }
520
521 fn taken(&mut self, bytes: u64) {
522 self.get_mut().taken(bytes);
523 }
524
525 fn min_limit(&self) -> u64 {
526 self.get_ref().min_limit()
527 }
528
529 fn properties(&self) -> CopyParams {
530 self.get_ref().properties()
531 }
532}
533
534impl<T: ?Sized + CopyWrite> CopyWrite for BufWriter<T> {
535 fn properties(&self) -> CopyParams {
536 self.get_ref().properties()
537 }
538}
539
540impl CopyRead for CachedFileMetadata {
541 fn properties(&self) -> CopyParams {
542 CopyParams(FdMeta::Metadata(self.1.clone()), Some(self.0.as_raw_fd()))
543 }
544}
545
546impl CopyWrite for CachedFileMetadata {
547 fn properties(&self) -> CopyParams {
548 CopyParams(FdMeta::Metadata(self.1.clone()), Some(self.0.as_raw_fd()))
549 }
550}
551
552fn fd_to_meta<T: AsRawFd>(fd: &T) -> FdMeta {
553 let fd = fd.as_raw_fd();
554 let file: ManuallyDrop<File> = ManuallyDrop::new(unsafe { File::from_raw_fd(fd) });
555 match file.metadata() {
556 Ok(meta) => FdMeta::Metadata(meta),
557 Err(_) => FdMeta::NoneObtained,
558 }
559}
560
561pub(super) enum CopyResult {
562 Ended(u64),
563 Error(Error, u64),
564 Fallback(u64),
565}
566
567impl CopyResult {
568 fn update_take(&self, reader: &mut impl CopyRead) {
569 match *self {
570 CopyResult::Fallback(bytes)
571 | CopyResult::Ended(bytes)
572 | CopyResult::Error(_, bytes) => reader.taken(bytes),
573 }
574 }
575}
576
577const INVALID_FD: RawFd = -1;
583
584pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> CopyResult {
591 use crate::cmp;
592
593 const NOT_PROBED: u8 = 0;
594 const UNAVAILABLE: u8 = 1;
595 const AVAILABLE: u8 = 2;
596
597 static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);
600
601 let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
602 NOT_PROBED => false,
603 UNAVAILABLE => return CopyResult::Fallback(0),
604 _ => true,
605 };
606
607 syscall! {
608 fn copy_file_range(
609 fd_in: libc::c_int,
610 off_in: *mut libc::loff_t,
611 fd_out: libc::c_int,
612 off_out: *mut libc::loff_t,
613 len: libc::size_t,
614 flags: libc::c_uint
615 ) -> libc::ssize_t
616 }
617
618 fn probe_copy_file_range_support() -> u8 {
619 match unsafe {
623 cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
624 .map_err(|e| e.raw_os_error())
625 } {
626 Err(Some(EPERM | ENOSYS)) => UNAVAILABLE,
627 Err(Some(EBADF)) => AVAILABLE,
628 Ok(_) => panic!("unexpected copy_file_range probe success"),
629 Err(_) => UNAVAILABLE,
632 }
633 }
634
635 let mut written = 0u64;
636 while written < max_len {
637 let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);
638 let bytes_to_copy = cmp::min(bytes_to_copy as usize, 0x4000_0000usize);
642 let copy_result = unsafe {
643 cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
646 };
647
648 if !have_probed && copy_result.is_ok() {
649 have_probed = true;
650 HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
651 }
652
653 match copy_result {
654 Ok(0) if written == 0 => {
655 return CopyResult::Fallback(0);
661 }
662 Ok(0) => return CopyResult::Ended(written), Ok(ret) => written += ret as u64,
664 Err(err) => {
665 return match err.raw_os_error() {
666 Some(EOVERFLOW) => CopyResult::Fallback(written),
668 Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF))
669 if written == 0 =>
670 {
671 if !have_probed {
672 let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) {
673 probe_copy_file_range_support()
684 } else {
685 AVAILABLE
686 };
687 HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed);
688 }
689
690 CopyResult::Fallback(0)
706 }
707 _ => CopyResult::Error(err, written),
708 };
709 }
710 }
711 }
712 CopyResult::Ended(written)
713}
714
715#[derive(PartialEq)]
716enum SpliceMode {
717 Sendfile,
718 Splice,
719}
720
721fn sendfile_splice(mode: SpliceMode, reader: RawFd, writer: RawFd, len: u64) -> CopyResult {
724 static HAS_SENDFILE: AtomicBool = AtomicBool::new(true);
725 static HAS_SPLICE: AtomicBool = AtomicBool::new(true);
726
727 #[cfg(target_os = "android")]
730 syscall! {
731 fn splice(
732 srcfd: libc::c_int,
733 src_offset: *const i64,
734 dstfd: libc::c_int,
735 dst_offset: *const i64,
736 len: libc::size_t,
737 flags: libc::c_int
738 ) -> libc::ssize_t
739 }
740
741 #[cfg(target_os = "linux")]
742 use libc::splice;
743
744 match mode {
745 SpliceMode::Sendfile if !HAS_SENDFILE.load(Ordering::Relaxed) => {
746 return CopyResult::Fallback(0);
747 }
748 SpliceMode::Splice if !HAS_SPLICE.load(Ordering::Relaxed) => {
749 return CopyResult::Fallback(0);
750 }
751 _ => (),
752 }
753
754 let mut written = 0u64;
755 while written < len {
756 let chunk_size = crate::cmp::min(len - written, 0x7ffff000_u64) as usize;
758
759 let result = match mode {
760 SpliceMode::Sendfile => {
761 cvt(unsafe { sendfile64(writer, reader, ptr::null_mut(), chunk_size) })
762 }
763 SpliceMode::Splice => cvt(unsafe {
764 splice(reader, ptr::null_mut(), writer, ptr::null_mut(), chunk_size, 0)
765 }),
766 };
767
768 match result {
769 Ok(0) => break, Ok(ret) => written += ret as u64,
771 Err(err) => {
772 return match err.raw_os_error() {
773 Some(ENOSYS | EPERM) => {
774 match mode {
777 SpliceMode::Sendfile => HAS_SENDFILE.store(false, Ordering::Relaxed),
778 SpliceMode::Splice => HAS_SPLICE.store(false, Ordering::Relaxed),
779 }
780 assert_eq!(written, 0);
781 CopyResult::Fallback(0)
782 }
783 Some(EINVAL) => {
784 assert_eq!(written, 0);
786 CopyResult::Fallback(0)
787 }
788 Some(os_err) if mode == SpliceMode::Sendfile && os_err == EOVERFLOW => {
789 CopyResult::Fallback(written)
790 }
791 _ => CopyResult::Error(err, written),
792 };
793 }
794 }
795 }
796 CopyResult::Ended(written)
797}