1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
//! ARM's Transactional Memory Extensions (TME).
//!
//! This CPU feature is available on Aarch64 - A architecture profile.
//! This feature is in the non-neon feature set. TME specific vendor documentation can
//! be found [TME Intrinsics Introduction][tme_intrinsics_intro].
//!
//! The reference is [ACLE Q4 2019][acle_q4_2019_ref].
//!
//! ACLE has a section for TME extensions and state masks for aborts and failure codes.
//! [ARM A64 Architecture Register Datasheet][a_profile_future] also describes possible failure code scenarios.
//!
//! [acle_q4_2019_ref]: https://static.docs.arm.com/101028/0010/ACLE_2019Q4_release-0010.pdf
//! [tme_intrinsics_intro]: https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics
//! [llvm_aarch64_int]: https://github.com/llvm/llvm-project/commit/a36d31478c182903523e04eb271bbf102bfab2cc#diff-ff24e1c35f4d54f1110ce5d90c709319R626-R646
//! [a_profile_future]: https://static.docs.arm.com/ddi0601/a/SysReg_xml_futureA-2019-04.pdf?_ga=2.116560387.441514988.1590524918-1110153136.1588469296

#[cfg(test)]
use stdarch_test::assert_instr;

extern "unadjusted" {
    #[link_name = "llvm.aarch64.tstart"]
    fn aarch64_tstart() -> u64;
    #[link_name = "llvm.aarch64.tcommit"]
    fn aarch64_tcommit();
    #[link_name = "llvm.aarch64.tcancel"]
    fn aarch64_tcancel(imm0: u64);
    #[link_name = "llvm.aarch64.ttest"]
    fn aarch64_ttest() -> u64;
}

/// Transaction successfully started.
pub const _TMSTART_SUCCESS: u64 = 0x00_u64;

/// Extraction mask for failure reason
pub const _TMFAILURE_REASON: u64 = 0x00007FFF_u64;

/// Transaction retry is possible.
pub const _TMFAILURE_RTRY: u64 = 1 << 15;

/// Transaction executed a TCANCEL instruction
pub const _TMFAILURE_CNCL: u64 = 1 << 16;

/// Transaction aborted because a conflict occurred
pub const _TMFAILURE_MEM: u64 = 1 << 17;

/// Fallback error type for any other reason
pub const _TMFAILURE_IMP: u64 = 1 << 18;

/// Transaction aborted because a non-permissible operation was attempted
pub const _TMFAILURE_ERR: u64 = 1 << 19;

/// Transaction aborted due to read or write set limit was exceeded
pub const _TMFAILURE_SIZE: u64 = 1 << 20;

/// Transaction aborted due to transactional nesting level was exceeded
pub const _TMFAILURE_NEST: u64 = 1 << 21;

/// Transaction aborted due to a debug trap.
pub const _TMFAILURE_DBG: u64 = 1 << 22;

/// Transaction failed from interrupt
pub const _TMFAILURE_INT: u64 = 1 << 23;

/// Indicates a TRIVIAL version of TM is available
pub const _TMFAILURE_TRIVIAL: u64 = 1 << 24;

/// Starts a new transaction. When the transaction starts successfully the return value is 0.
/// If the transaction fails, all state modifications are discarded and a cause of the failure
/// is encoded in the return value.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(tstart))]
pub unsafe fn __tstart() -> u64 {
    aarch64_tstart()
}

/// Commits the current transaction. For a nested transaction, the only effect is that the
/// transactional nesting depth is decreased. For an outer transaction, the state modifications
/// performed transactionally are committed to the architectural state.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(tcommit))]
pub unsafe fn __tcommit() {
    aarch64_tcommit()
}

/// Cancels the current transaction and discards all state modifications that were performed transactionally.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(tcancel, IMM16 = 0x0))]
#[rustc_legacy_const_generics(0)]
pub unsafe fn __tcancel<const IMM16: u64>() {
    static_assert!(IMM16 <= 65535);
    aarch64_tcancel(IMM16);
}

/// Tests if executing inside a transaction. If no transaction is currently executing,
/// the return value is 0. Otherwise, this intrinsic returns the depth of the transaction.
///
/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics).
#[inline]
#[target_feature(enable = "tme")]
#[cfg_attr(test, assert_instr(ttest))]
pub unsafe fn __ttest() -> u64 {
    aarch64_ttest()
}

#[cfg(test)]
mod tests {
    use stdarch_test::simd_test;

    use crate::core_arch::aarch64::*;

    const CANCEL_CODE: u64 = (0 | (0x123 & _TMFAILURE_REASON) as u64) as u64;

    #[simd_test(enable = "tme")]
    unsafe fn test_tstart() {
        let mut x = 0;
        for i in 0..10 {
            let code = tme::__tstart();
            if code == _TMSTART_SUCCESS {
                x += 1;
                assert_eq!(x, i + 1);
                break;
            }
            assert_eq!(x, 0);
        }
    }

    #[simd_test(enable = "tme")]
    unsafe fn test_tcommit() {
        let mut x = 0;
        for i in 0..10 {
            let code = tme::__tstart();
            if code == _TMSTART_SUCCESS {
                x += 1;
                assert_eq!(x, i + 1);
                tme::__tcommit();
            }
            assert_eq!(x, i + 1);
        }
    }

    #[simd_test(enable = "tme")]
    unsafe fn test_tcancel() {
        let mut x = 0;

        for i in 0..10 {
            let code = tme::__tstart();
            if code == _TMSTART_SUCCESS {
                x += 1;
                assert_eq!(x, i + 1);
                tme::__tcancel::<CANCEL_CODE>();
                break;
            }
        }

        assert_eq!(x, 0);
    }

    #[simd_test(enable = "tme")]
    unsafe fn test_ttest() {
        for _ in 0..10 {
            let code = tme::__tstart();
            if code == _TMSTART_SUCCESS {
                if tme::__ttest() == 2 {
                    tme::__tcancel::<CANCEL_CODE>();
                    break;
                }
            }
        }
    }
}