1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

use crate::alloc::{GlobalAlloc, Layout, System};
use crate::ptr;
use crate::sys::common::alloc::{realloc_fallback, MIN_ALIGN};

#[stable(feature = "alloc_system_type", since = "1.28.0")]
unsafe impl GlobalAlloc for System {
    #[inline]
    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
        // jemalloc provides alignment less than MIN_ALIGN for small allocations.
        // So only rely on MIN_ALIGN if size >= align.
        // Also see <https://github.com/rust-lang/rust/issues/45955> and
        // <https://github.com/rust-lang/rust/issues/62251#issuecomment-507580914>.
        if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() {
            libc::malloc(layout.size()) as *mut u8
        } else {
            // `posix_memalign` returns a non-aligned value if supplied a very
            // large alignment on older versions of Apple's platforms (unknown
            // exactly which version range, but the issue is definitely
            // present in macOS 10.14 and iOS 13.3).
            //
            // <https://github.com/rust-lang/rust/issues/30170>
            #[cfg(target_vendor = "apple")]
            {
                if layout.align() > (1 << 31) {
                    return ptr::null_mut();
                }
            }
            aligned_malloc(&layout)
        }
    }

    #[inline]
    unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
        // See the comment above in `alloc` for why this check looks the way it does.
        if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() {
            libc::calloc(layout.size(), 1) as *mut u8
        } else {
            let ptr = self.alloc(layout);
            if !ptr.is_null() {
                ptr::write_bytes(ptr, 0, layout.size());
            }
            ptr
        }
    }

    #[inline]
    unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
        libc::free(ptr as *mut libc::c_void)
    }

    #[inline]
    unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
        if layout.align() <= MIN_ALIGN && layout.align() <= new_size {
            libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8
        } else {
            realloc_fallback(self, ptr, layout, new_size)
        }
    }
}

cfg_if::cfg_if! {
    // We use posix_memalign wherever possible, but not all targets have that function.
    if #[cfg(any(
        target_os = "redox",
        target_os = "espidf",
        target_os = "horizon",
        target_os = "vita",
    ))] {
        #[inline]
        unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
            libc::memalign(layout.align(), layout.size()) as *mut u8
        }
    } else {
        #[inline]
        unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 {
            let mut out = ptr::null_mut();
            // We prefer posix_memalign over aligned_malloc since with aligned_malloc,
            // implementations are making almost arbitrary choices for which alignments are
            // "supported", making it hard to use. For instance, some implementations require the
            // size to be a multiple of the alignment (wasi emmalloc), while others require the
            // alignment to be at least the pointer size (Illumos, macOS) -- which may or may not be
            // standards-compliant, but that does not help us.
            // posix_memalign only has one, clear requirement: that the alignment be a multiple of
            // `sizeof(void*)`. Since these are all powers of 2, we can just use max.
            let align = layout.align().max(crate::mem::size_of::<usize>());
            let ret = libc::posix_memalign(&mut out, align, layout.size());
            if ret != 0 { ptr::null_mut() } else { out as *mut u8 }
        }
    }
}