core/stdarch/crates/core_arch/src/x86/
vaes.rs

1//! Vectorized AES Instructions (VAES)
2//!
3//! The intrinsics here correspond to those in the `immintrin.h` C header.
4//!
5//! The reference is [Intel 64 and IA-32 Architectures Software Developer's
6//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref].
7//!
8//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
9
10use crate::core_arch::x86::__m256i;
11use crate::core_arch::x86::__m512i;
12
13#[cfg(test)]
14use stdarch_test::assert_instr;
15
16#[allow(improper_ctypes)]
17extern "C" {
18    #[link_name = "llvm.x86.aesni.aesenc.256"]
19    fn aesenc_256(a: __m256i, round_key: __m256i) -> __m256i;
20    #[link_name = "llvm.x86.aesni.aesenclast.256"]
21    fn aesenclast_256(a: __m256i, round_key: __m256i) -> __m256i;
22    #[link_name = "llvm.x86.aesni.aesdec.256"]
23    fn aesdec_256(a: __m256i, round_key: __m256i) -> __m256i;
24    #[link_name = "llvm.x86.aesni.aesdeclast.256"]
25    fn aesdeclast_256(a: __m256i, round_key: __m256i) -> __m256i;
26    #[link_name = "llvm.x86.aesni.aesenc.512"]
27    fn aesenc_512(a: __m512i, round_key: __m512i) -> __m512i;
28    #[link_name = "llvm.x86.aesni.aesenclast.512"]
29    fn aesenclast_512(a: __m512i, round_key: __m512i) -> __m512i;
30    #[link_name = "llvm.x86.aesni.aesdec.512"]
31    fn aesdec_512(a: __m512i, round_key: __m512i) -> __m512i;
32    #[link_name = "llvm.x86.aesni.aesdeclast.512"]
33    fn aesdeclast_512(a: __m512i, round_key: __m512i) -> __m512i;
34}
35
36/// Performs one round of an AES encryption flow on each 128-bit word (state) in `a` using
37/// the corresponding 128-bit word (key) in `round_key`.
38///
39/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesenc_epi128)
40#[inline]
41#[target_feature(enable = "vaes")]
42#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
43#[cfg_attr(test, assert_instr(vaesenc))]
44pub unsafe fn _mm256_aesenc_epi128(a: __m256i, round_key: __m256i) -> __m256i {
45    aesenc_256(a, round_key)
46}
47
48/// Performs the last round of an AES encryption flow on each 128-bit word (state) in `a` using
49/// the corresponding 128-bit word (key) in `round_key`.
50///
51/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesenclast_epi128)
52#[inline]
53#[target_feature(enable = "vaes")]
54#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
55#[cfg_attr(test, assert_instr(vaesenclast))]
56pub unsafe fn _mm256_aesenclast_epi128(a: __m256i, round_key: __m256i) -> __m256i {
57    aesenclast_256(a, round_key)
58}
59
60/// Performs one round of an AES decryption flow on each 128-bit word (state) in `a` using
61/// the corresponding 128-bit word (key) in `round_key`.
62///
63/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesdec_epi128)
64#[inline]
65#[target_feature(enable = "vaes")]
66#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
67#[cfg_attr(test, assert_instr(vaesdec))]
68pub unsafe fn _mm256_aesdec_epi128(a: __m256i, round_key: __m256i) -> __m256i {
69    aesdec_256(a, round_key)
70}
71
72/// Performs the last round of an AES decryption flow on each 128-bit word (state) in `a` using
73/// the corresponding 128-bit word (key) in `round_key`.
74///
75/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesdeclast_epi128)
76#[inline]
77#[target_feature(enable = "vaes")]
78#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
79#[cfg_attr(test, assert_instr(vaesdeclast))]
80pub unsafe fn _mm256_aesdeclast_epi128(a: __m256i, round_key: __m256i) -> __m256i {
81    aesdeclast_256(a, round_key)
82}
83
84/// Performs one round of an AES encryption flow on each 128-bit word (state) in `a` using
85/// the corresponding 128-bit word (key) in `round_key`.
86///
87/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesenc_epi128)
88#[inline]
89#[target_feature(enable = "vaes,avx512f")]
90#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
91#[cfg_attr(test, assert_instr(vaesenc))]
92pub unsafe fn _mm512_aesenc_epi128(a: __m512i, round_key: __m512i) -> __m512i {
93    aesenc_512(a, round_key)
94}
95
96/// Performs the last round of an AES encryption flow on each 128-bit word (state) in `a` using
97/// the corresponding 128-bit word (key) in `round_key`.
98///
99/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesenclast_epi128)
100#[inline]
101#[target_feature(enable = "vaes,avx512f")]
102#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
103#[cfg_attr(test, assert_instr(vaesenclast))]
104pub unsafe fn _mm512_aesenclast_epi128(a: __m512i, round_key: __m512i) -> __m512i {
105    aesenclast_512(a, round_key)
106}
107
108/// Performs one round of an AES decryption flow on each 128-bit word (state) in `a` using
109/// the corresponding 128-bit word (key) in `round_key`.
110///
111/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesdec_epi128)
112#[inline]
113#[target_feature(enable = "vaes,avx512f")]
114#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
115#[cfg_attr(test, assert_instr(vaesdec))]
116pub unsafe fn _mm512_aesdec_epi128(a: __m512i, round_key: __m512i) -> __m512i {
117    aesdec_512(a, round_key)
118}
119
120/// Performs the last round of an AES decryption flow on each 128-bit word (state) in `a` using
121/// the corresponding 128-bit word (key) in `round_key`.
122///
123/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesdeclast_epi128)
124#[inline]
125#[target_feature(enable = "vaes,avx512f")]
126#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
127#[cfg_attr(test, assert_instr(vaesdeclast))]
128pub unsafe fn _mm512_aesdeclast_epi128(a: __m512i, round_key: __m512i) -> __m512i {
129    aesdeclast_512(a, round_key)
130}
131
132#[cfg(test)]
133mod tests {
134    // The constants in the tests below are just bit patterns. They should not
135    // be interpreted as integers; signedness does not make sense for them, but
136    // __mXXXi happens to be defined in terms of signed integers.
137    #![allow(overflowing_literals)]
138
139    use stdarch_test::simd_test;
140
141    use crate::core_arch::x86::*;
142
143    // the first parts of these tests are straight ports from the AES-NI tests
144    // the second parts directly compare the two, for inputs that are different across lanes
145    // and "more random" than the standard test vectors
146    // ideally we'd be using quickcheck here instead
147
148    #[target_feature(enable = "avx2")]
149    unsafe fn helper_for_256_vaes(
150        linear: unsafe fn(__m128i, __m128i) -> __m128i,
151        vectorized: unsafe fn(__m256i, __m256i) -> __m256i,
152    ) {
153        let a = _mm256_set_epi64x(
154            0xDCB4DB3657BF0B7D,
155            0x18DB0601068EDD9F,
156            0xB76B908233200DC5,
157            0xE478235FA8E22D5E,
158        );
159        let k = _mm256_set_epi64x(
160            0x672F6F105A94CEA7,
161            0x8298B8FFCA5F829C,
162            0xA3927047B3FB61D8,
163            0x978093862CDE7187,
164        );
165        let mut a_decomp = [_mm_setzero_si128(); 2];
166        a_decomp[0] = _mm256_extracti128_si256::<0>(a);
167        a_decomp[1] = _mm256_extracti128_si256::<1>(a);
168        let mut k_decomp = [_mm_setzero_si128(); 2];
169        k_decomp[0] = _mm256_extracti128_si256::<0>(k);
170        k_decomp[1] = _mm256_extracti128_si256::<1>(k);
171        let r = vectorized(a, k);
172        let mut e_decomp = [_mm_setzero_si128(); 2];
173        for i in 0..2 {
174            e_decomp[i] = linear(a_decomp[i], k_decomp[i]);
175        }
176        assert_eq_m128i(_mm256_extracti128_si256::<0>(r), e_decomp[0]);
177        assert_eq_m128i(_mm256_extracti128_si256::<1>(r), e_decomp[1]);
178    }
179
180    #[target_feature(enable = "sse2")]
181    unsafe fn setup_state_key<T>(broadcast: unsafe fn(__m128i) -> T) -> (T, T) {
182        // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx.
183        let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff);
184        let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee);
185        (broadcast(a), broadcast(k))
186    }
187
188    #[target_feature(enable = "avx2")]
189    unsafe fn setup_state_key_256() -> (__m256i, __m256i) {
190        setup_state_key(_mm256_broadcastsi128_si256)
191    }
192
193    #[target_feature(enable = "avx512f")]
194    unsafe fn setup_state_key_512() -> (__m512i, __m512i) {
195        setup_state_key(_mm512_broadcast_i32x4)
196    }
197
198    #[simd_test(enable = "vaes,avx512vl")]
199    unsafe fn test_mm256_aesdec_epi128() {
200        // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx.
201        let (a, k) = setup_state_key_256();
202        let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee);
203        let e = _mm256_broadcastsi128_si256(e);
204        let r = _mm256_aesdec_epi128(a, k);
205        assert_eq_m256i(r, e);
206
207        helper_for_256_vaes(_mm_aesdec_si128, _mm256_aesdec_epi128);
208    }
209
210    #[simd_test(enable = "vaes,avx512vl")]
211    unsafe fn test_mm256_aesdeclast_epi128() {
212        // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx.
213        let (a, k) = setup_state_key_256();
214        let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493);
215        let e = _mm256_broadcastsi128_si256(e);
216        let r = _mm256_aesdeclast_epi128(a, k);
217        assert_eq_m256i(r, e);
218
219        helper_for_256_vaes(_mm_aesdeclast_si128, _mm256_aesdeclast_epi128);
220    }
221
222    #[simd_test(enable = "vaes,avx512vl")]
223    unsafe fn test_mm256_aesenc_epi128() {
224        // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx.
225        // they are repeated appropriately
226        let (a, k) = setup_state_key_256();
227        let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333);
228        let e = _mm256_broadcastsi128_si256(e);
229        let r = _mm256_aesenc_epi128(a, k);
230        assert_eq_m256i(r, e);
231
232        helper_for_256_vaes(_mm_aesenc_si128, _mm256_aesenc_epi128);
233    }
234
235    #[simd_test(enable = "vaes,avx512vl")]
236    unsafe fn test_mm256_aesenclast_epi128() {
237        // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx.
238        let (a, k) = setup_state_key_256();
239        let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8);
240        let e = _mm256_broadcastsi128_si256(e);
241        let r = _mm256_aesenclast_epi128(a, k);
242        assert_eq_m256i(r, e);
243
244        helper_for_256_vaes(_mm_aesenclast_si128, _mm256_aesenclast_epi128);
245    }
246
247    #[target_feature(enable = "avx512f")]
248    unsafe fn helper_for_512_vaes(
249        linear: unsafe fn(__m128i, __m128i) -> __m128i,
250        vectorized: unsafe fn(__m512i, __m512i) -> __m512i,
251    ) {
252        let a = _mm512_set_epi64(
253            0xDCB4DB3657BF0B7D,
254            0x18DB0601068EDD9F,
255            0xB76B908233200DC5,
256            0xE478235FA8E22D5E,
257            0xAB05CFFA2621154C,
258            0x1171B47A186174C9,
259            0x8C6B6C0E7595CEC9,
260            0xBE3E7D4934E961BD,
261        );
262        let k = _mm512_set_epi64(
263            0x672F6F105A94CEA7,
264            0x8298B8FFCA5F829C,
265            0xA3927047B3FB61D8,
266            0x978093862CDE7187,
267            0xB1927AB22F31D0EC,
268            0xA9A5DA619BE4D7AF,
269            0xCA2590F56884FDC6,
270            0x19BE9F660038BDB5,
271        );
272        let mut a_decomp = [_mm_setzero_si128(); 4];
273        a_decomp[0] = _mm512_extracti32x4_epi32::<0>(a);
274        a_decomp[1] = _mm512_extracti32x4_epi32::<1>(a);
275        a_decomp[2] = _mm512_extracti32x4_epi32::<2>(a);
276        a_decomp[3] = _mm512_extracti32x4_epi32::<3>(a);
277        let mut k_decomp = [_mm_setzero_si128(); 4];
278        k_decomp[0] = _mm512_extracti32x4_epi32::<0>(k);
279        k_decomp[1] = _mm512_extracti32x4_epi32::<1>(k);
280        k_decomp[2] = _mm512_extracti32x4_epi32::<2>(k);
281        k_decomp[3] = _mm512_extracti32x4_epi32::<3>(k);
282        let r = vectorized(a, k);
283        let mut e_decomp = [_mm_setzero_si128(); 4];
284        for i in 0..4 {
285            e_decomp[i] = linear(a_decomp[i], k_decomp[i]);
286        }
287        assert_eq_m128i(_mm512_extracti32x4_epi32::<0>(r), e_decomp[0]);
288        assert_eq_m128i(_mm512_extracti32x4_epi32::<1>(r), e_decomp[1]);
289        assert_eq_m128i(_mm512_extracti32x4_epi32::<2>(r), e_decomp[2]);
290        assert_eq_m128i(_mm512_extracti32x4_epi32::<3>(r), e_decomp[3]);
291    }
292
293    #[simd_test(enable = "vaes,avx512f")]
294    unsafe fn test_mm512_aesdec_epi128() {
295        // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx.
296        let (a, k) = setup_state_key_512();
297        let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee);
298        let e = _mm512_broadcast_i32x4(e);
299        let r = _mm512_aesdec_epi128(a, k);
300        assert_eq_m512i(r, e);
301
302        helper_for_512_vaes(_mm_aesdec_si128, _mm512_aesdec_epi128);
303    }
304
305    #[simd_test(enable = "vaes,avx512f")]
306    unsafe fn test_mm512_aesdeclast_epi128() {
307        // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx.
308        let (a, k) = setup_state_key_512();
309        let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493);
310        let e = _mm512_broadcast_i32x4(e);
311        let r = _mm512_aesdeclast_epi128(a, k);
312        assert_eq_m512i(r, e);
313
314        helper_for_512_vaes(_mm_aesdeclast_si128, _mm512_aesdeclast_epi128);
315    }
316
317    #[simd_test(enable = "vaes,avx512f")]
318    unsafe fn test_mm512_aesenc_epi128() {
319        // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx.
320        let (a, k) = setup_state_key_512();
321        let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333);
322        let e = _mm512_broadcast_i32x4(e);
323        let r = _mm512_aesenc_epi128(a, k);
324        assert_eq_m512i(r, e);
325
326        helper_for_512_vaes(_mm_aesenc_si128, _mm512_aesenc_epi128);
327    }
328
329    #[simd_test(enable = "vaes,avx512f")]
330    unsafe fn test_mm512_aesenclast_epi128() {
331        // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx.
332        let (a, k) = setup_state_key_512();
333        let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8);
334        let e = _mm512_broadcast_i32x4(e);
335        let r = _mm512_aesenclast_epi128(a, k);
336        assert_eq_m512i(r, e);
337
338        helper_for_512_vaes(_mm_aesenclast_si128, _mm512_aesenclast_epi128);
339    }
340}