1use crate::core_arch::simd::*;
11use crate::core_arch::x86::__m128i;
12use crate::core_arch::x86::__m256i;
13use crate::core_arch::x86::__m512i;
14use crate::core_arch::x86::__mmask8;
15use crate::core_arch::x86::__mmask16;
16use crate::intrinsics::simd::{simd_ctpop, simd_select_bitmask};
17use crate::mem::transmute;
18
19#[cfg(test)]
20use stdarch_test::assert_instr;
21
22#[inline]
26#[target_feature(enable = "avx512vpopcntdq")]
27#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
28#[cfg_attr(test, assert_instr(vpopcntd))]
29#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
30pub const fn _mm512_popcnt_epi32(a: __m512i) -> __m512i {
31 unsafe { transmute(simd_ctpop(a.as_i32x16())) }
32}
33
34#[inline]
41#[target_feature(enable = "avx512vpopcntdq")]
42#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
43#[cfg_attr(test, assert_instr(vpopcntd))]
44#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
45pub const fn _mm512_maskz_popcnt_epi32(k: __mmask16, a: __m512i) -> __m512i {
46 unsafe {
47 transmute(simd_select_bitmask(
48 k,
49 simd_ctpop(a.as_i32x16()),
50 i32x16::ZERO,
51 ))
52 }
53}
54
55#[inline]
62#[target_feature(enable = "avx512vpopcntdq")]
63#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
64#[cfg_attr(test, assert_instr(vpopcntd))]
65#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
66pub const fn _mm512_mask_popcnt_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i {
67 unsafe {
68 transmute(simd_select_bitmask(
69 k,
70 simd_ctpop(a.as_i32x16()),
71 src.as_i32x16(),
72 ))
73 }
74}
75
76#[inline]
80#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
81#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
82#[cfg_attr(test, assert_instr(vpopcntd))]
83#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
84pub const fn _mm256_popcnt_epi32(a: __m256i) -> __m256i {
85 unsafe { transmute(simd_ctpop(a.as_i32x8())) }
86}
87
88#[inline]
95#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
96#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
97#[cfg_attr(test, assert_instr(vpopcntd))]
98#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
99pub const fn _mm256_maskz_popcnt_epi32(k: __mmask8, a: __m256i) -> __m256i {
100 unsafe {
101 transmute(simd_select_bitmask(
102 k,
103 simd_ctpop(a.as_i32x8()),
104 i32x8::ZERO,
105 ))
106 }
107}
108
109#[inline]
116#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
117#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
118#[cfg_attr(test, assert_instr(vpopcntd))]
119#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
120pub const fn _mm256_mask_popcnt_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i {
121 unsafe {
122 transmute(simd_select_bitmask(
123 k,
124 simd_ctpop(a.as_i32x8()),
125 src.as_i32x8(),
126 ))
127 }
128}
129
130#[inline]
134#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
135#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
136#[cfg_attr(test, assert_instr(vpopcntd))]
137#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
138pub const fn _mm_popcnt_epi32(a: __m128i) -> __m128i {
139 unsafe { transmute(simd_ctpop(a.as_i32x4())) }
140}
141
142#[inline]
149#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
150#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
151#[cfg_attr(test, assert_instr(vpopcntd))]
152#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
153pub const fn _mm_maskz_popcnt_epi32(k: __mmask8, a: __m128i) -> __m128i {
154 unsafe {
155 transmute(simd_select_bitmask(
156 k,
157 simd_ctpop(a.as_i32x4()),
158 i32x4::ZERO,
159 ))
160 }
161}
162
163#[inline]
170#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
171#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
172#[cfg_attr(test, assert_instr(vpopcntd))]
173#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
174pub const fn _mm_mask_popcnt_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
175 unsafe {
176 transmute(simd_select_bitmask(
177 k,
178 simd_ctpop(a.as_i32x4()),
179 src.as_i32x4(),
180 ))
181 }
182}
183
184#[inline]
188#[target_feature(enable = "avx512vpopcntdq")]
189#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
190#[cfg_attr(test, assert_instr(vpopcntq))]
191#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
192pub const fn _mm512_popcnt_epi64(a: __m512i) -> __m512i {
193 unsafe { transmute(simd_ctpop(a.as_i64x8())) }
194}
195
196#[inline]
203#[target_feature(enable = "avx512vpopcntdq")]
204#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
205#[cfg_attr(test, assert_instr(vpopcntq))]
206#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
207pub const fn _mm512_maskz_popcnt_epi64(k: __mmask8, a: __m512i) -> __m512i {
208 unsafe {
209 transmute(simd_select_bitmask(
210 k,
211 simd_ctpop(a.as_i64x8()),
212 i64x8::ZERO,
213 ))
214 }
215}
216
217#[inline]
224#[target_feature(enable = "avx512vpopcntdq")]
225#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
226#[cfg_attr(test, assert_instr(vpopcntq))]
227#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
228pub const fn _mm512_mask_popcnt_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i {
229 unsafe {
230 transmute(simd_select_bitmask(
231 k,
232 simd_ctpop(a.as_i64x8()),
233 src.as_i64x8(),
234 ))
235 }
236}
237
238#[inline]
242#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
243#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
244#[cfg_attr(test, assert_instr(vpopcntq))]
245#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
246pub const fn _mm256_popcnt_epi64(a: __m256i) -> __m256i {
247 unsafe { transmute(simd_ctpop(a.as_i64x4())) }
248}
249
250#[inline]
257#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
258#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
259#[cfg_attr(test, assert_instr(vpopcntq))]
260#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
261pub const fn _mm256_maskz_popcnt_epi64(k: __mmask8, a: __m256i) -> __m256i {
262 unsafe {
263 transmute(simd_select_bitmask(
264 k,
265 simd_ctpop(a.as_i64x4()),
266 i64x4::ZERO,
267 ))
268 }
269}
270
271#[inline]
278#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
279#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
280#[cfg_attr(test, assert_instr(vpopcntq))]
281#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
282pub const fn _mm256_mask_popcnt_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i {
283 unsafe {
284 transmute(simd_select_bitmask(
285 k,
286 simd_ctpop(a.as_i64x4()),
287 src.as_i64x4(),
288 ))
289 }
290}
291
292#[inline]
296#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
297#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
298#[cfg_attr(test, assert_instr(vpopcntq))]
299#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
300pub const fn _mm_popcnt_epi64(a: __m128i) -> __m128i {
301 unsafe { transmute(simd_ctpop(a.as_i64x2())) }
302}
303
304#[inline]
311#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
312#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
313#[cfg_attr(test, assert_instr(vpopcntq))]
314#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
315pub const fn _mm_maskz_popcnt_epi64(k: __mmask8, a: __m128i) -> __m128i {
316 unsafe {
317 transmute(simd_select_bitmask(
318 k,
319 simd_ctpop(a.as_i64x2()),
320 i64x2::ZERO,
321 ))
322 }
323}
324
325#[inline]
332#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
333#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
334#[cfg_attr(test, assert_instr(vpopcntq))]
335#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
336pub const fn _mm_mask_popcnt_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
337 unsafe {
338 transmute(simd_select_bitmask(
339 k,
340 simd_ctpop(a.as_i64x2()),
341 src.as_i64x2(),
342 ))
343 }
344}
345
346#[cfg(test)]
347mod tests {
348 use crate::core_arch::assert_eq_const as assert_eq;
349 use stdarch_test::simd_test;
350
351 use crate::core_arch::x86::*;
352
353 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
354 const fn test_mm512_popcnt_epi32() {
355 let test_data = _mm512_set_epi32(
356 0,
357 1,
358 -1,
359 2,
360 7,
361 0xFF_FE,
362 0x7F_FF_FF_FF,
363 -100,
364 0x40_00_00_00,
365 103,
366 371,
367 552,
368 432_948,
369 818_826_998,
370 255,
371 256,
372 );
373 let actual_result = _mm512_popcnt_epi32(test_data);
374 let reference_result =
375 _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 1, 5, 6, 3, 10, 17, 8, 1);
376 assert_eq_m512i(actual_result, reference_result);
377 }
378
379 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
380 const fn test_mm512_mask_popcnt_epi32() {
381 let test_data = _mm512_set_epi32(
382 0,
383 1,
384 -1,
385 2,
386 7,
387 0xFF_FE,
388 0x7F_FF_FF_FF,
389 -100,
390 0x40_00_00_00,
391 103,
392 371,
393 552,
394 432_948,
395 818_826_998,
396 255,
397 256,
398 );
399 let mask = 0xFF_00;
400 let actual_result = _mm512_mask_popcnt_epi32(test_data, mask, test_data);
401 let reference_result = _mm512_set_epi32(
402 0,
403 1,
404 32,
405 1,
406 3,
407 15,
408 31,
409 28,
410 0x40_00_00_00,
411 103,
412 371,
413 552,
414 432_948,
415 818_826_998,
416 255,
417 256,
418 );
419 assert_eq_m512i(actual_result, reference_result);
420 }
421
422 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
423 const fn test_mm512_maskz_popcnt_epi32() {
424 let test_data = _mm512_set_epi32(
425 0,
426 1,
427 -1,
428 2,
429 7,
430 0xFF_FE,
431 0x7F_FF_FF_FF,
432 -100,
433 0x40_00_00_00,
434 103,
435 371,
436 552,
437 432_948,
438 818_826_998,
439 255,
440 256,
441 );
442 let mask = 0xFF_00;
443 let actual_result = _mm512_maskz_popcnt_epi32(mask, test_data);
444 let reference_result = _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0);
445 assert_eq_m512i(actual_result, reference_result);
446 }
447
448 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
449 const fn test_mm256_popcnt_epi32() {
450 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
451 let actual_result = _mm256_popcnt_epi32(test_data);
452 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 3, 15, 31, 28);
453 assert_eq_m256i(actual_result, reference_result);
454 }
455
456 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
457 const fn test_mm256_mask_popcnt_epi32() {
458 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
459 let mask = 0xF0;
460 let actual_result = _mm256_mask_popcnt_epi32(test_data, mask, test_data);
461 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
462 assert_eq_m256i(actual_result, reference_result);
463 }
464
465 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
466 const fn test_mm256_maskz_popcnt_epi32() {
467 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
468 let mask = 0xF0;
469 let actual_result = _mm256_maskz_popcnt_epi32(mask, test_data);
470 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 0, 0, 0, 0);
471 assert_eq_m256i(actual_result, reference_result);
472 }
473
474 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
475 const fn test_mm_popcnt_epi32() {
476 let test_data = _mm_set_epi32(0, 1, -1, -100);
477 let actual_result = _mm_popcnt_epi32(test_data);
478 let reference_result = _mm_set_epi32(0, 1, 32, 28);
479 assert_eq_m128i(actual_result, reference_result);
480 }
481
482 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
483 const fn test_mm_mask_popcnt_epi32() {
484 let test_data = _mm_set_epi32(0, 1, -1, -100);
485 let mask = 0xE;
486 let actual_result = _mm_mask_popcnt_epi32(test_data, mask, test_data);
487 let reference_result = _mm_set_epi32(0, 1, 32, -100);
488 assert_eq_m128i(actual_result, reference_result);
489 }
490
491 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
492 const fn test_mm_maskz_popcnt_epi32() {
493 let test_data = _mm_set_epi32(0, 1, -1, -100);
494 let mask = 0xE;
495 let actual_result = _mm_maskz_popcnt_epi32(mask, test_data);
496 let reference_result = _mm_set_epi32(0, 1, 32, 0);
497 assert_eq_m128i(actual_result, reference_result);
498 }
499
500 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
501 const fn test_mm512_popcnt_epi64() {
502 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
503 let actual_result = _mm512_popcnt_epi64(test_data);
504 let reference_result = _mm512_set_epi64(0, 1, 64, 1, 3, 15, 63, 60);
505 assert_eq_m512i(actual_result, reference_result);
506 }
507
508 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
509 const fn test_mm512_mask_popcnt_epi64() {
510 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
511 let mask = 0xF0;
512 let actual_result = _mm512_mask_popcnt_epi64(test_data, mask, test_data);
513 let reference_result =
514 _mm512_set_epi64(0, 1, 64, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
515 assert_eq_m512i(actual_result, reference_result);
516 }
517
518 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
519 const fn test_mm512_maskz_popcnt_epi64() {
520 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
521 let mask = 0xF0;
522 let actual_result = _mm512_maskz_popcnt_epi64(mask, test_data);
523 let reference_result = _mm512_set_epi64(0, 1, 64, 1, 0, 0, 0, 0);
524 assert_eq_m512i(actual_result, reference_result);
525 }
526
527 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
528 const fn test_mm256_popcnt_epi64() {
529 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
530 let actual_result = _mm256_popcnt_epi64(test_data);
531 let reference_result = _mm256_set_epi64x(0, 1, 64, 60);
532 assert_eq_m256i(actual_result, reference_result);
533 }
534
535 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
536 const fn test_mm256_mask_popcnt_epi64() {
537 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
538 let mask = 0xE;
539 let actual_result = _mm256_mask_popcnt_epi64(test_data, mask, test_data);
540 let reference_result = _mm256_set_epi64x(0, 1, 64, -100);
541 assert_eq_m256i(actual_result, reference_result);
542 }
543
544 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
545 const fn test_mm256_maskz_popcnt_epi64() {
546 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
547 let mask = 0xE;
548 let actual_result = _mm256_maskz_popcnt_epi64(mask, test_data);
549 let reference_result = _mm256_set_epi64x(0, 1, 64, 0);
550 assert_eq_m256i(actual_result, reference_result);
551 }
552
553 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
554 const fn test_mm_popcnt_epi64() {
555 let test_data = _mm_set_epi64x(0, 1);
556 let actual_result = _mm_popcnt_epi64(test_data);
557 let reference_result = _mm_set_epi64x(0, 1);
558 assert_eq_m128i(actual_result, reference_result);
559 let test_data = _mm_set_epi64x(-1, -100);
560 let actual_result = _mm_popcnt_epi64(test_data);
561 let reference_result = _mm_set_epi64x(64, 60);
562 assert_eq_m128i(actual_result, reference_result);
563 }
564
565 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
566 const fn test_mm_mask_popcnt_epi64() {
567 let test_data = _mm_set_epi64x(0, -100);
568 let mask = 0x2;
569 let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data);
570 let reference_result = _mm_set_epi64x(0, -100);
571 assert_eq_m128i(actual_result, reference_result);
572 let test_data = _mm_set_epi64x(-1, 1);
573 let mask = 0x2;
574 let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data);
575 let reference_result = _mm_set_epi64x(64, 1);
576 assert_eq_m128i(actual_result, reference_result);
577 }
578
579 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
580 const fn test_mm_maskz_popcnt_epi64() {
581 let test_data = _mm_set_epi64x(0, 1);
582 let mask = 0x2;
583 let actual_result = _mm_maskz_popcnt_epi64(mask, test_data);
584 let reference_result = _mm_set_epi64x(0, 0);
585 assert_eq_m128i(actual_result, reference_result);
586 let test_data = _mm_set_epi64x(-1, -100);
587 let mask = 0x2;
588 let actual_result = _mm_maskz_popcnt_epi64(mask, test_data);
589 let reference_result = _mm_set_epi64x(64, 0);
590 assert_eq_m128i(actual_result, reference_result);
591 }
592}