1use crate::core_arch::simd::*;
11use crate::core_arch::x86::__m128i;
12use crate::core_arch::x86::__m256i;
13use crate::core_arch::x86::__m512i;
14use crate::core_arch::x86::__mmask8;
15use crate::core_arch::x86::__mmask16;
16use crate::intrinsics::simd::{simd_ctpop, simd_select_bitmask};
17use crate::mem::transmute;
18
19#[cfg(test)]
20use stdarch_test::assert_instr;
21
22#[inline]
26#[target_feature(enable = "avx512vpopcntdq")]
27#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
28#[cfg_attr(test, assert_instr(vpopcntd))]
29pub fn _mm512_popcnt_epi32(a: __m512i) -> __m512i {
30 unsafe { transmute(simd_ctpop(a.as_i32x16())) }
31}
32
33#[inline]
40#[target_feature(enable = "avx512vpopcntdq")]
41#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
42#[cfg_attr(test, assert_instr(vpopcntd))]
43pub fn _mm512_maskz_popcnt_epi32(k: __mmask16, a: __m512i) -> __m512i {
44 unsafe {
45 transmute(simd_select_bitmask(
46 k,
47 simd_ctpop(a.as_i32x16()),
48 i32x16::ZERO,
49 ))
50 }
51}
52
53#[inline]
60#[target_feature(enable = "avx512vpopcntdq")]
61#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
62#[cfg_attr(test, assert_instr(vpopcntd))]
63pub fn _mm512_mask_popcnt_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i {
64 unsafe {
65 transmute(simd_select_bitmask(
66 k,
67 simd_ctpop(a.as_i32x16()),
68 src.as_i32x16(),
69 ))
70 }
71}
72
73#[inline]
77#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
78#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
79#[cfg_attr(test, assert_instr(vpopcntd))]
80pub fn _mm256_popcnt_epi32(a: __m256i) -> __m256i {
81 unsafe { transmute(simd_ctpop(a.as_i32x8())) }
82}
83
84#[inline]
91#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
92#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
93#[cfg_attr(test, assert_instr(vpopcntd))]
94pub fn _mm256_maskz_popcnt_epi32(k: __mmask8, a: __m256i) -> __m256i {
95 unsafe {
96 transmute(simd_select_bitmask(
97 k,
98 simd_ctpop(a.as_i32x8()),
99 i32x8::ZERO,
100 ))
101 }
102}
103
104#[inline]
111#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
112#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
113#[cfg_attr(test, assert_instr(vpopcntd))]
114pub fn _mm256_mask_popcnt_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i {
115 unsafe {
116 transmute(simd_select_bitmask(
117 k,
118 simd_ctpop(a.as_i32x8()),
119 src.as_i32x8(),
120 ))
121 }
122}
123
124#[inline]
128#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
129#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
130#[cfg_attr(test, assert_instr(vpopcntd))]
131pub fn _mm_popcnt_epi32(a: __m128i) -> __m128i {
132 unsafe { transmute(simd_ctpop(a.as_i32x4())) }
133}
134
135#[inline]
142#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
143#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
144#[cfg_attr(test, assert_instr(vpopcntd))]
145pub fn _mm_maskz_popcnt_epi32(k: __mmask8, a: __m128i) -> __m128i {
146 unsafe {
147 transmute(simd_select_bitmask(
148 k,
149 simd_ctpop(a.as_i32x4()),
150 i32x4::ZERO,
151 ))
152 }
153}
154
155#[inline]
162#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
163#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
164#[cfg_attr(test, assert_instr(vpopcntd))]
165pub fn _mm_mask_popcnt_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
166 unsafe {
167 transmute(simd_select_bitmask(
168 k,
169 simd_ctpop(a.as_i32x4()),
170 src.as_i32x4(),
171 ))
172 }
173}
174
175#[inline]
179#[target_feature(enable = "avx512vpopcntdq")]
180#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
181#[cfg_attr(test, assert_instr(vpopcntq))]
182pub fn _mm512_popcnt_epi64(a: __m512i) -> __m512i {
183 unsafe { transmute(simd_ctpop(a.as_i64x8())) }
184}
185
186#[inline]
193#[target_feature(enable = "avx512vpopcntdq")]
194#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
195#[cfg_attr(test, assert_instr(vpopcntq))]
196pub fn _mm512_maskz_popcnt_epi64(k: __mmask8, a: __m512i) -> __m512i {
197 unsafe {
198 transmute(simd_select_bitmask(
199 k,
200 simd_ctpop(a.as_i64x8()),
201 i64x8::ZERO,
202 ))
203 }
204}
205
206#[inline]
213#[target_feature(enable = "avx512vpopcntdq")]
214#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
215#[cfg_attr(test, assert_instr(vpopcntq))]
216pub fn _mm512_mask_popcnt_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i {
217 unsafe {
218 transmute(simd_select_bitmask(
219 k,
220 simd_ctpop(a.as_i64x8()),
221 src.as_i64x8(),
222 ))
223 }
224}
225
226#[inline]
230#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
231#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
232#[cfg_attr(test, assert_instr(vpopcntq))]
233pub fn _mm256_popcnt_epi64(a: __m256i) -> __m256i {
234 unsafe { transmute(simd_ctpop(a.as_i64x4())) }
235}
236
237#[inline]
244#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
245#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
246#[cfg_attr(test, assert_instr(vpopcntq))]
247pub fn _mm256_maskz_popcnt_epi64(k: __mmask8, a: __m256i) -> __m256i {
248 unsafe {
249 transmute(simd_select_bitmask(
250 k,
251 simd_ctpop(a.as_i64x4()),
252 i64x4::ZERO,
253 ))
254 }
255}
256
257#[inline]
264#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
265#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
266#[cfg_attr(test, assert_instr(vpopcntq))]
267pub fn _mm256_mask_popcnt_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i {
268 unsafe {
269 transmute(simd_select_bitmask(
270 k,
271 simd_ctpop(a.as_i64x4()),
272 src.as_i64x4(),
273 ))
274 }
275}
276
277#[inline]
281#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
282#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
283#[cfg_attr(test, assert_instr(vpopcntq))]
284pub fn _mm_popcnt_epi64(a: __m128i) -> __m128i {
285 unsafe { transmute(simd_ctpop(a.as_i64x2())) }
286}
287
288#[inline]
295#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
296#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
297#[cfg_attr(test, assert_instr(vpopcntq))]
298pub fn _mm_maskz_popcnt_epi64(k: __mmask8, a: __m128i) -> __m128i {
299 unsafe {
300 transmute(simd_select_bitmask(
301 k,
302 simd_ctpop(a.as_i64x2()),
303 i64x2::ZERO,
304 ))
305 }
306}
307
308#[inline]
315#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
316#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
317#[cfg_attr(test, assert_instr(vpopcntq))]
318pub fn _mm_mask_popcnt_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
319 unsafe {
320 transmute(simd_select_bitmask(
321 k,
322 simd_ctpop(a.as_i64x2()),
323 src.as_i64x2(),
324 ))
325 }
326}
327
328#[cfg(test)]
329mod tests {
330 use stdarch_test::simd_test;
331
332 use crate::core_arch::x86::*;
333
334 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
335 unsafe fn test_mm512_popcnt_epi32() {
336 let test_data = _mm512_set_epi32(
337 0,
338 1,
339 -1,
340 2,
341 7,
342 0xFF_FE,
343 0x7F_FF_FF_FF,
344 -100,
345 0x40_00_00_00,
346 103,
347 371,
348 552,
349 432_948,
350 818_826_998,
351 255,
352 256,
353 );
354 let actual_result = _mm512_popcnt_epi32(test_data);
355 let reference_result =
356 _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 1, 5, 6, 3, 10, 17, 8, 1);
357 assert_eq_m512i(actual_result, reference_result);
358 }
359
360 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
361 unsafe fn test_mm512_mask_popcnt_epi32() {
362 let test_data = _mm512_set_epi32(
363 0,
364 1,
365 -1,
366 2,
367 7,
368 0xFF_FE,
369 0x7F_FF_FF_FF,
370 -100,
371 0x40_00_00_00,
372 103,
373 371,
374 552,
375 432_948,
376 818_826_998,
377 255,
378 256,
379 );
380 let mask = 0xFF_00;
381 let actual_result = _mm512_mask_popcnt_epi32(test_data, mask, test_data);
382 let reference_result = _mm512_set_epi32(
383 0,
384 1,
385 32,
386 1,
387 3,
388 15,
389 31,
390 28,
391 0x40_00_00_00,
392 103,
393 371,
394 552,
395 432_948,
396 818_826_998,
397 255,
398 256,
399 );
400 assert_eq_m512i(actual_result, reference_result);
401 }
402
403 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
404 unsafe fn test_mm512_maskz_popcnt_epi32() {
405 let test_data = _mm512_set_epi32(
406 0,
407 1,
408 -1,
409 2,
410 7,
411 0xFF_FE,
412 0x7F_FF_FF_FF,
413 -100,
414 0x40_00_00_00,
415 103,
416 371,
417 552,
418 432_948,
419 818_826_998,
420 255,
421 256,
422 );
423 let mask = 0xFF_00;
424 let actual_result = _mm512_maskz_popcnt_epi32(mask, test_data);
425 let reference_result = _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0);
426 assert_eq_m512i(actual_result, reference_result);
427 }
428
429 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
430 unsafe fn test_mm256_popcnt_epi32() {
431 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
432 let actual_result = _mm256_popcnt_epi32(test_data);
433 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 3, 15, 31, 28);
434 assert_eq_m256i(actual_result, reference_result);
435 }
436
437 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
438 unsafe fn test_mm256_mask_popcnt_epi32() {
439 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
440 let mask = 0xF0;
441 let actual_result = _mm256_mask_popcnt_epi32(test_data, mask, test_data);
442 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
443 assert_eq_m256i(actual_result, reference_result);
444 }
445
446 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
447 unsafe fn test_mm256_maskz_popcnt_epi32() {
448 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
449 let mask = 0xF0;
450 let actual_result = _mm256_maskz_popcnt_epi32(mask, test_data);
451 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 0, 0, 0, 0);
452 assert_eq_m256i(actual_result, reference_result);
453 }
454
455 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
456 unsafe fn test_mm_popcnt_epi32() {
457 let test_data = _mm_set_epi32(0, 1, -1, -100);
458 let actual_result = _mm_popcnt_epi32(test_data);
459 let reference_result = _mm_set_epi32(0, 1, 32, 28);
460 assert_eq_m128i(actual_result, reference_result);
461 }
462
463 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
464 unsafe fn test_mm_mask_popcnt_epi32() {
465 let test_data = _mm_set_epi32(0, 1, -1, -100);
466 let mask = 0xE;
467 let actual_result = _mm_mask_popcnt_epi32(test_data, mask, test_data);
468 let reference_result = _mm_set_epi32(0, 1, 32, -100);
469 assert_eq_m128i(actual_result, reference_result);
470 }
471
472 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
473 unsafe fn test_mm_maskz_popcnt_epi32() {
474 let test_data = _mm_set_epi32(0, 1, -1, -100);
475 let mask = 0xE;
476 let actual_result = _mm_maskz_popcnt_epi32(mask, test_data);
477 let reference_result = _mm_set_epi32(0, 1, 32, 0);
478 assert_eq_m128i(actual_result, reference_result);
479 }
480
481 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
482 unsafe fn test_mm512_popcnt_epi64() {
483 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
484 let actual_result = _mm512_popcnt_epi64(test_data);
485 let reference_result = _mm512_set_epi64(0, 1, 64, 1, 3, 15, 63, 60);
486 assert_eq_m512i(actual_result, reference_result);
487 }
488
489 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
490 unsafe fn test_mm512_mask_popcnt_epi64() {
491 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
492 let mask = 0xF0;
493 let actual_result = _mm512_mask_popcnt_epi64(test_data, mask, test_data);
494 let reference_result =
495 _mm512_set_epi64(0, 1, 64, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
496 assert_eq_m512i(actual_result, reference_result);
497 }
498
499 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
500 unsafe fn test_mm512_maskz_popcnt_epi64() {
501 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
502 let mask = 0xF0;
503 let actual_result = _mm512_maskz_popcnt_epi64(mask, test_data);
504 let reference_result = _mm512_set_epi64(0, 1, 64, 1, 0, 0, 0, 0);
505 assert_eq_m512i(actual_result, reference_result);
506 }
507
508 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
509 unsafe fn test_mm256_popcnt_epi64() {
510 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
511 let actual_result = _mm256_popcnt_epi64(test_data);
512 let reference_result = _mm256_set_epi64x(0, 1, 64, 60);
513 assert_eq_m256i(actual_result, reference_result);
514 }
515
516 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
517 unsafe fn test_mm256_mask_popcnt_epi64() {
518 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
519 let mask = 0xE;
520 let actual_result = _mm256_mask_popcnt_epi64(test_data, mask, test_data);
521 let reference_result = _mm256_set_epi64x(0, 1, 64, -100);
522 assert_eq_m256i(actual_result, reference_result);
523 }
524
525 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
526 unsafe fn test_mm256_maskz_popcnt_epi64() {
527 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
528 let mask = 0xE;
529 let actual_result = _mm256_maskz_popcnt_epi64(mask, test_data);
530 let reference_result = _mm256_set_epi64x(0, 1, 64, 0);
531 assert_eq_m256i(actual_result, reference_result);
532 }
533
534 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
535 unsafe fn test_mm_popcnt_epi64() {
536 let test_data = _mm_set_epi64x(0, 1);
537 let actual_result = _mm_popcnt_epi64(test_data);
538 let reference_result = _mm_set_epi64x(0, 1);
539 assert_eq_m128i(actual_result, reference_result);
540 let test_data = _mm_set_epi64x(-1, -100);
541 let actual_result = _mm_popcnt_epi64(test_data);
542 let reference_result = _mm_set_epi64x(64, 60);
543 assert_eq_m128i(actual_result, reference_result);
544 }
545
546 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
547 unsafe fn test_mm_mask_popcnt_epi64() {
548 let test_data = _mm_set_epi64x(0, -100);
549 let mask = 0x2;
550 let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data);
551 let reference_result = _mm_set_epi64x(0, -100);
552 assert_eq_m128i(actual_result, reference_result);
553 let test_data = _mm_set_epi64x(-1, 1);
554 let mask = 0x2;
555 let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data);
556 let reference_result = _mm_set_epi64x(64, 1);
557 assert_eq_m128i(actual_result, reference_result);
558 }
559
560 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
561 unsafe fn test_mm_maskz_popcnt_epi64() {
562 let test_data = _mm_set_epi64x(0, 1);
563 let mask = 0x2;
564 let actual_result = _mm_maskz_popcnt_epi64(mask, test_data);
565 let reference_result = _mm_set_epi64x(0, 0);
566 assert_eq_m128i(actual_result, reference_result);
567 let test_data = _mm_set_epi64x(-1, -100);
568 let mask = 0x2;
569 let actual_result = _mm_maskz_popcnt_epi64(mask, test_data);
570 let reference_result = _mm_set_epi64x(64, 0);
571 assert_eq_m128i(actual_result, reference_result);
572 }
573}