1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185

// Reference: Section 7.4 "Hints" of ACLE

// CP15 instruction
#[cfg(not(any(
    // v8
    target_arch = "aarch64",
    target_arch = "arm64ec",
    // v7
    target_feature = "v7",
    // v6-M
    target_feature = "mclass"
)))]
mod cp15;

#[cfg(not(any(
    target_arch = "aarch64",
    target_arch = "arm64ec",
    target_feature = "v7",
    target_feature = "mclass"
)))]
#[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
pub use self::cp15::*;

// Dedicated instructions
#[cfg(any(
    target_arch = "aarch64",
    target_arch = "arm64ec",
    target_feature = "v7",
    target_feature = "mclass"
))]
macro_rules! dmb_dsb {
    ($A:ident) => {
        #[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
        impl super::super::sealed::Dmb for $A {
            #[inline(always)]
            unsafe fn __dmb(&self) {
                super::dmb(super::arg::$A)
            }
        }

        #[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
        impl super::super::sealed::Dsb for $A {
            #[inline(always)]
            unsafe fn __dsb(&self) {
                super::dsb(super::arg::$A)
            }
        }
    };
}

#[cfg(any(
    target_arch = "aarch64",
    target_arch = "arm64ec",
    target_feature = "v7",
    target_feature = "mclass"
))]
mod common;

#[cfg(any(
    target_arch = "aarch64",
    target_arch = "arm64ec",
    target_feature = "v7",
    target_feature = "mclass"
))]
#[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
pub use self::common::*;

#[cfg(any(
    target_arch = "aarch64",
    target_arch = "arm64ec",
    target_feature = "v7",
))]
mod not_mclass;

#[cfg(any(
    target_arch = "aarch64",
    target_arch = "arm64ec",
    target_feature = "v7",
))]
#[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
pub use self::not_mclass::*;

#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
mod v8;

#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
#[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
pub use self::v8::*;

/// Generates a DMB (data memory barrier) instruction or equivalent CP15 instruction.
///
/// DMB ensures the observed ordering of memory accesses. Memory accesses of the specified type
/// issued before the DMB are guaranteed to be observed (in the specified scope) before memory
/// accesses issued after the DMB.
///
/// For example, DMB should be used between storing data, and updating a flag variable that makes
/// that data available to another core.
///
/// The __dmb() intrinsic also acts as a compiler memory barrier of the appropriate type.
#[inline(always)]
#[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
pub unsafe fn __dmb<A>(arg: A)
where
    A: super::sealed::Dmb,
{
    arg.__dmb()
}

/// Generates a DSB (data synchronization barrier) instruction or equivalent CP15 instruction.
///
/// DSB ensures the completion of memory accesses. A DSB behaves as the equivalent DMB and has
/// additional properties. After a DSB instruction completes, all memory accesses of the specified
/// type issued before the DSB are guaranteed to have completed.
///
/// The __dsb() intrinsic also acts as a compiler memory barrier of the appropriate type.
#[inline(always)]
#[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
pub unsafe fn __dsb<A>(arg: A)
where
    A: super::sealed::Dsb,
{
    arg.__dsb()
}

/// Generates an ISB (instruction synchronization barrier) instruction or equivalent CP15
/// instruction.
///
/// This instruction flushes the processor pipeline fetch buffers, so that following instructions
/// are fetched from cache or memory.
///
/// An ISB is needed after some system maintenance operations. An ISB is also needed before
/// transferring control to code that has been loaded or modified in memory, for example by an
/// overlay mechanism or just-in-time code generator.  (Note that if instruction and data caches are
/// separate, privileged cache maintenance operations would be needed in order to unify the caches.)
///
/// The only supported argument for the __isb() intrinsic is 15, corresponding to the SY (full
/// system) scope of the ISB instruction.
#[inline(always)]
#[unstable(feature = "stdarch_arm_barrier", issue = "117219")]
pub unsafe fn __isb<A>(arg: A)
where
    A: super::sealed::Isb,
{
    arg.__isb()
}

extern "unadjusted" {
    #[cfg_attr(
        any(target_arch = "aarch64", target_arch = "arm64ec"),
        link_name = "llvm.aarch64.dmb"
    )]
    #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.dmb")]
    fn dmb(_: i32);

    #[cfg_attr(
        any(target_arch = "aarch64", target_arch = "arm64ec"),
        link_name = "llvm.aarch64.dsb"
    )]
    #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.dsb")]
    fn dsb(_: i32);

    #[cfg_attr(
        any(target_arch = "aarch64", target_arch = "arm64ec"),
        link_name = "llvm.aarch64.isb"
    )]
    #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.isb")]
    fn isb(_: i32);
}

// we put these in a module to prevent weirdness with glob re-exports
mod arg {
    // See Section 7.3  Memory barriers of ACLE
    pub const SY: i32 = 15;
    pub const ST: i32 = 14;
    pub const LD: i32 = 13;
    pub const ISH: i32 = 11;
    pub const ISHST: i32 = 10;
    pub const ISHLD: i32 = 9;
    pub const NSH: i32 = 7;
    pub const NSHST: i32 = 6;
    pub const NSHLD: i32 = 5;
    pub const OSH: i32 = 3;
    pub const OSHST: i32 = 2;
    pub const OSHLD: i32 = 1;
}