rustc_const_eval/interpret/intrinsics/
simd.rs

1use either::Either;
2use rustc_abi::{BackendRepr, Endian};
3use rustc_apfloat::ieee::{Double, Half, Quad, Single};
4use rustc_apfloat::{Float, Round};
5use rustc_middle::mir::interpret::{InterpErrorKind, Pointer, UndefinedBehaviorInfo};
6use rustc_middle::ty::{FloatTy, SimdAlign};
7use rustc_middle::{bug, err_ub_format, mir, span_bug, throw_unsup_format, ty};
8use rustc_span::{Symbol, sym};
9use tracing::trace;
10
11use super::{
12    ImmTy, InterpCx, InterpResult, Machine, MinMax, MulAddType, OpTy, PlaceTy, Provenance, Scalar,
13    Size, TyAndLayout, assert_matches, interp_ok, throw_ub_format,
14};
15use crate::interpret::Writeable;
16
17impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
18    /// Returns `true` if emulation happened.
19    /// Here we implement the intrinsics that are common to all CTFE instances; individual machines can add their own
20    /// intrinsic handling.
21    pub fn eval_simd_intrinsic(
22        &mut self,
23        intrinsic_name: Symbol,
24        generic_args: ty::GenericArgsRef<'tcx>,
25        args: &[OpTy<'tcx, M::Provenance>],
26        dest: &PlaceTy<'tcx, M::Provenance>,
27        ret: Option<mir::BasicBlock>,
28    ) -> InterpResult<'tcx, bool> {
29        let dest = dest.force_mplace(self)?;
30
31        match intrinsic_name {
32            sym::simd_insert => {
33                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
34                let elem = &args[2];
35                let (input, input_len) = self.project_to_simd(&args[0])?;
36                let (dest, dest_len) = self.project_to_simd(&dest)?;
37                assert_eq!(input_len, dest_len, "Return vector length must match input length");
38                // Bounds are not checked by typeck so we have to do it ourselves.
39                if index >= input_len {
40                    throw_ub_format!(
41                        "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}"
42                    );
43                }
44
45                for i in 0..dest_len {
46                    let place = self.project_index(&dest, i)?;
47                    let value =
48                        if i == index { elem.clone() } else { self.project_index(&input, i)? };
49                    self.copy_op(&value, &place)?;
50                }
51            }
52            sym::simd_extract => {
53                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
54                let (input, input_len) = self.project_to_simd(&args[0])?;
55                // Bounds are not checked by typeck so we have to do it ourselves.
56                if index >= input_len {
57                    throw_ub_format!(
58                        "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}"
59                    );
60                }
61                self.copy_op(&self.project_index(&input, index)?, &dest)?;
62            }
63            sym::simd_neg
64            | sym::simd_fabs
65            | sym::simd_ceil
66            | sym::simd_floor
67            | sym::simd_round
68            | sym::simd_round_ties_even
69            | sym::simd_trunc
70            | sym::simd_ctlz
71            | sym::simd_ctpop
72            | sym::simd_cttz
73            | sym::simd_bswap
74            | sym::simd_bitreverse => {
75                let (op, op_len) = self.project_to_simd(&args[0])?;
76                let (dest, dest_len) = self.project_to_simd(&dest)?;
77
78                assert_eq!(dest_len, op_len);
79
80                #[derive(Copy, Clone)]
81                enum Op {
82                    MirOp(mir::UnOp),
83                    Abs,
84                    Round(rustc_apfloat::Round),
85                    Numeric(Symbol),
86                }
87                let which = match intrinsic_name {
88                    sym::simd_neg => Op::MirOp(mir::UnOp::Neg),
89                    sym::simd_fabs => Op::Abs,
90                    sym::simd_ceil => Op::Round(rustc_apfloat::Round::TowardPositive),
91                    sym::simd_floor => Op::Round(rustc_apfloat::Round::TowardNegative),
92                    sym::simd_round => Op::Round(rustc_apfloat::Round::NearestTiesToAway),
93                    sym::simd_round_ties_even => Op::Round(rustc_apfloat::Round::NearestTiesToEven),
94                    sym::simd_trunc => Op::Round(rustc_apfloat::Round::TowardZero),
95                    sym::simd_ctlz => Op::Numeric(sym::ctlz),
96                    sym::simd_ctpop => Op::Numeric(sym::ctpop),
97                    sym::simd_cttz => Op::Numeric(sym::cttz),
98                    sym::simd_bswap => Op::Numeric(sym::bswap),
99                    sym::simd_bitreverse => Op::Numeric(sym::bitreverse),
100                    _ => unreachable!(),
101                };
102
103                for i in 0..dest_len {
104                    let op = self.read_immediate(&self.project_index(&op, i)?)?;
105                    let dest = self.project_index(&dest, i)?;
106                    let val = match which {
107                        Op::MirOp(mir_op) => {
108                            // this already does NaN adjustments
109                            self.unary_op(mir_op, &op)?.to_scalar()
110                        }
111                        Op::Abs => {
112                            // Works for f32 and f64.
113                            let ty::Float(float_ty) = op.layout.ty.kind() else {
114                                span_bug!(
115                                    self.cur_span(),
116                                    "{} operand is not a float",
117                                    intrinsic_name
118                                )
119                            };
120                            let op = op.to_scalar();
121                            // "Bitwise" operation, no NaN adjustments
122                            match float_ty {
123                                FloatTy::F16 => Scalar::from_f16(op.to_f16()?.abs()),
124                                FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()),
125                                FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()),
126                                FloatTy::F128 => Scalar::from_f128(op.to_f128()?.abs()),
127                            }
128                        }
129                        Op::Round(rounding) => {
130                            let ty::Float(float_ty) = op.layout.ty.kind() else {
131                                span_bug!(
132                                    self.cur_span(),
133                                    "{} operand is not a float",
134                                    intrinsic_name
135                                )
136                            };
137                            let op = op.to_scalar();
138                            match float_ty {
139                                FloatTy::F16 => self.float_round::<Half>(op, rounding)?,
140                                FloatTy::F32 => self.float_round::<Single>(op, rounding)?,
141                                FloatTy::F64 => self.float_round::<Double>(op, rounding)?,
142                                FloatTy::F128 => self.float_round::<Quad>(op, rounding)?,
143                            }
144                        }
145                        Op::Numeric(name) => {
146                            self.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)?
147                        }
148                    };
149                    self.write_scalar(val, &dest)?;
150                }
151            }
152            sym::simd_add
153            | sym::simd_sub
154            | sym::simd_mul
155            | sym::simd_div
156            | sym::simd_rem
157            | sym::simd_shl
158            | sym::simd_shr
159            | sym::simd_and
160            | sym::simd_or
161            | sym::simd_xor
162            | sym::simd_eq
163            | sym::simd_ne
164            | sym::simd_lt
165            | sym::simd_le
166            | sym::simd_gt
167            | sym::simd_ge
168            | sym::simd_fmax
169            | sym::simd_fmin
170            | sym::simd_saturating_add
171            | sym::simd_saturating_sub
172            | sym::simd_arith_offset => {
173                use mir::BinOp;
174
175                let (left, left_len) = self.project_to_simd(&args[0])?;
176                let (right, right_len) = self.project_to_simd(&args[1])?;
177                let (dest, dest_len) = self.project_to_simd(&dest)?;
178
179                assert_eq!(dest_len, left_len);
180                assert_eq!(dest_len, right_len);
181
182                enum Op {
183                    MirOp(BinOp),
184                    SaturatingOp(BinOp),
185                    FMinMax(MinMax),
186                    WrappingOffset,
187                }
188                let which = match intrinsic_name {
189                    sym::simd_add => Op::MirOp(BinOp::Add),
190                    sym::simd_sub => Op::MirOp(BinOp::Sub),
191                    sym::simd_mul => Op::MirOp(BinOp::Mul),
192                    sym::simd_div => Op::MirOp(BinOp::Div),
193                    sym::simd_rem => Op::MirOp(BinOp::Rem),
194                    sym::simd_shl => Op::MirOp(BinOp::ShlUnchecked),
195                    sym::simd_shr => Op::MirOp(BinOp::ShrUnchecked),
196                    sym::simd_and => Op::MirOp(BinOp::BitAnd),
197                    sym::simd_or => Op::MirOp(BinOp::BitOr),
198                    sym::simd_xor => Op::MirOp(BinOp::BitXor),
199                    sym::simd_eq => Op::MirOp(BinOp::Eq),
200                    sym::simd_ne => Op::MirOp(BinOp::Ne),
201                    sym::simd_lt => Op::MirOp(BinOp::Lt),
202                    sym::simd_le => Op::MirOp(BinOp::Le),
203                    sym::simd_gt => Op::MirOp(BinOp::Gt),
204                    sym::simd_ge => Op::MirOp(BinOp::Ge),
205                    sym::simd_fmax => Op::FMinMax(MinMax::MaxNum),
206                    sym::simd_fmin => Op::FMinMax(MinMax::MinNum),
207                    sym::simd_saturating_add => Op::SaturatingOp(BinOp::Add),
208                    sym::simd_saturating_sub => Op::SaturatingOp(BinOp::Sub),
209                    sym::simd_arith_offset => Op::WrappingOffset,
210                    _ => unreachable!(),
211                };
212
213                for i in 0..dest_len {
214                    let left = self.read_immediate(&self.project_index(&left, i)?)?;
215                    let right = self.read_immediate(&self.project_index(&right, i)?)?;
216                    let dest = self.project_index(&dest, i)?;
217                    let val = match which {
218                        Op::MirOp(mir_op) => {
219                            // this does NaN adjustments.
220                            let val = self.binary_op(mir_op, &left, &right).map_err_kind(|kind| {
221                                match kind {
222                                    InterpErrorKind::UndefinedBehavior(UndefinedBehaviorInfo::ShiftOverflow { shift_amount, .. }) => {
223                                        // this resets the interpreter backtrace, but it's not worth avoiding that.
224                                        let shift_amount = match shift_amount {
225                                            Either::Left(v) => v.to_string(),
226                                            Either::Right(v) => v.to_string(),
227                                        };
228                                        err_ub_format!("overflowing shift by {shift_amount} in `{intrinsic_name}` in lane {i}")
229                                    }
230                                    kind => kind
231                                }
232                            })?;
233                            if matches!(
234                                mir_op,
235                                BinOp::Eq
236                                    | BinOp::Ne
237                                    | BinOp::Lt
238                                    | BinOp::Le
239                                    | BinOp::Gt
240                                    | BinOp::Ge
241                            ) {
242                                // Special handling for boolean-returning operations
243                                assert_eq!(val.layout.ty, self.tcx.types.bool);
244                                let val = val.to_scalar().to_bool().unwrap();
245                                bool_to_simd_element(val, dest.layout.size)
246                            } else {
247                                assert_ne!(val.layout.ty, self.tcx.types.bool);
248                                assert_eq!(val.layout.ty, dest.layout.ty);
249                                val.to_scalar()
250                            }
251                        }
252                        Op::SaturatingOp(mir_op) => self.saturating_arith(mir_op, &left, &right)?,
253                        Op::WrappingOffset => {
254                            let ptr = left.to_scalar().to_pointer(self)?;
255                            let offset_count = right.to_scalar().to_target_isize(self)?;
256                            let pointee_ty = left.layout.ty.builtin_deref(true).unwrap();
257
258                            let pointee_size =
259                                i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
260                            let offset_bytes = offset_count.wrapping_mul(pointee_size);
261                            let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self);
262                            Scalar::from_maybe_pointer(offset_ptr, self)
263                        }
264                        Op::FMinMax(op) => self.fminmax_op(op, &left, &right)?,
265                    };
266                    self.write_scalar(val, &dest)?;
267                }
268            }
269            sym::simd_reduce_and
270            | sym::simd_reduce_or
271            | sym::simd_reduce_xor
272            | sym::simd_reduce_any
273            | sym::simd_reduce_all
274            | sym::simd_reduce_max
275            | sym::simd_reduce_min => {
276                use mir::BinOp;
277
278                let (op, op_len) = self.project_to_simd(&args[0])?;
279
280                let imm_from_bool = |b| {
281                    ImmTy::from_scalar(
282                        Scalar::from_bool(b),
283                        self.layout_of(self.tcx.types.bool).unwrap(),
284                    )
285                };
286
287                enum Op {
288                    MirOp(BinOp),
289                    MirOpBool(BinOp),
290                    MinMax(MinMax),
291                }
292                let which = match intrinsic_name {
293                    sym::simd_reduce_and => Op::MirOp(BinOp::BitAnd),
294                    sym::simd_reduce_or => Op::MirOp(BinOp::BitOr),
295                    sym::simd_reduce_xor => Op::MirOp(BinOp::BitXor),
296                    sym::simd_reduce_any => Op::MirOpBool(BinOp::BitOr),
297                    sym::simd_reduce_all => Op::MirOpBool(BinOp::BitAnd),
298                    sym::simd_reduce_max => Op::MinMax(MinMax::MaxNum),
299                    sym::simd_reduce_min => Op::MinMax(MinMax::MinNum),
300                    _ => unreachable!(),
301                };
302
303                // Initialize with first lane, then proceed with the rest.
304                let mut res = self.read_immediate(&self.project_index(&op, 0)?)?;
305                if matches!(which, Op::MirOpBool(_)) {
306                    // Convert to `bool` scalar.
307                    res = imm_from_bool(simd_element_to_bool(res)?);
308                }
309                for i in 1..op_len {
310                    let op = self.read_immediate(&self.project_index(&op, i)?)?;
311                    res = match which {
312                        Op::MirOp(mir_op) => self.binary_op(mir_op, &res, &op)?,
313                        Op::MirOpBool(mir_op) => {
314                            let op = imm_from_bool(simd_element_to_bool(op)?);
315                            self.binary_op(mir_op, &res, &op)?
316                        }
317                        Op::MinMax(mmop) => {
318                            if matches!(res.layout.ty.kind(), ty::Float(_)) {
319                                ImmTy::from_scalar(self.fminmax_op(mmop, &res, &op)?, res.layout)
320                            } else {
321                                // Just boring integers, no NaNs to worry about.
322                                let mirop = match mmop {
323                                    MinMax::MinNum | MinMax::Minimum => BinOp::Le,
324                                    MinMax::MaxNum | MinMax::Maximum => BinOp::Ge,
325                                };
326                                if self.binary_op(mirop, &res, &op)?.to_scalar().to_bool()? {
327                                    res
328                                } else {
329                                    op
330                                }
331                            }
332                        }
333                    };
334                }
335                self.write_immediate(*res, &dest)?;
336            }
337            sym::simd_reduce_add_ordered | sym::simd_reduce_mul_ordered => {
338                use mir::BinOp;
339
340                let (op, op_len) = self.project_to_simd(&args[0])?;
341                let init = self.read_immediate(&args[1])?;
342
343                let mir_op = match intrinsic_name {
344                    sym::simd_reduce_add_ordered => BinOp::Add,
345                    sym::simd_reduce_mul_ordered => BinOp::Mul,
346                    _ => unreachable!(),
347                };
348
349                let mut res = init;
350                for i in 0..op_len {
351                    let op = self.read_immediate(&self.project_index(&op, i)?)?;
352                    res = self.binary_op(mir_op, &res, &op)?;
353                }
354                self.write_immediate(*res, &dest)?;
355            }
356            sym::simd_select => {
357                let (mask, mask_len) = self.project_to_simd(&args[0])?;
358                let (yes, yes_len) = self.project_to_simd(&args[1])?;
359                let (no, no_len) = self.project_to_simd(&args[2])?;
360                let (dest, dest_len) = self.project_to_simd(&dest)?;
361
362                assert_eq!(dest_len, mask_len);
363                assert_eq!(dest_len, yes_len);
364                assert_eq!(dest_len, no_len);
365
366                for i in 0..dest_len {
367                    let mask = self.read_immediate(&self.project_index(&mask, i)?)?;
368                    let yes = self.read_immediate(&self.project_index(&yes, i)?)?;
369                    let no = self.read_immediate(&self.project_index(&no, i)?)?;
370                    let dest = self.project_index(&dest, i)?;
371
372                    let val = if simd_element_to_bool(mask)? { yes } else { no };
373                    self.write_immediate(*val, &dest)?;
374                }
375            }
376            // Variant of `select` that takes a bitmask rather than a "vector of bool".
377            sym::simd_select_bitmask => {
378                let mask = &args[0];
379                let (yes, yes_len) = self.project_to_simd(&args[1])?;
380                let (no, no_len) = self.project_to_simd(&args[2])?;
381                let (dest, dest_len) = self.project_to_simd(&dest)?;
382                let bitmask_len = dest_len.next_multiple_of(8);
383                if bitmask_len > 64 {
384                    throw_unsup_format!(
385                        "simd_select_bitmask: vectors larger than 64 elements are currently not supported"
386                    );
387                }
388
389                assert_eq!(dest_len, yes_len);
390                assert_eq!(dest_len, no_len);
391
392                // Read the mask, either as an integer or as an array.
393                let mask: u64 = match mask.layout.ty.kind() {
394                    ty::Uint(_) => {
395                        // Any larger integer type is fine.
396                        assert!(mask.layout.size.bits() >= bitmask_len);
397                        self.read_scalar(mask)?.to_bits(mask.layout.size)?.try_into().unwrap()
398                    }
399                    ty::Array(elem, _len) if elem == &self.tcx.types.u8 => {
400                        // The array must have exactly the right size.
401                        assert_eq!(mask.layout.size.bits(), bitmask_len);
402                        // Read the raw bytes.
403                        let mask = mask.assert_mem_place(); // arrays cannot be immediate
404                        let mask_bytes =
405                            self.read_bytes_ptr_strip_provenance(mask.ptr(), mask.layout.size)?;
406                        // Turn them into a `u64` in the right way.
407                        let mask_size = mask.layout.size.bytes_usize();
408                        let mut mask_arr = [0u8; 8];
409                        match self.tcx.data_layout.endian {
410                            Endian::Little => {
411                                // Fill the first N bytes.
412                                mask_arr[..mask_size].copy_from_slice(mask_bytes);
413                                u64::from_le_bytes(mask_arr)
414                            }
415                            Endian::Big => {
416                                // Fill the last N bytes.
417                                let i = mask_arr.len().strict_sub(mask_size);
418                                mask_arr[i..].copy_from_slice(mask_bytes);
419                                u64::from_be_bytes(mask_arr)
420                            }
421                        }
422                    }
423                    _ => bug!("simd_select_bitmask: invalid mask type {}", mask.layout.ty),
424                };
425
426                let dest_len = u32::try_from(dest_len).unwrap();
427                for i in 0..dest_len {
428                    let bit_i = simd_bitmask_index(i, dest_len, self.tcx.data_layout.endian);
429                    let mask = mask & 1u64.strict_shl(bit_i);
430                    let yes = self.read_immediate(&self.project_index(&yes, i.into())?)?;
431                    let no = self.read_immediate(&self.project_index(&no, i.into())?)?;
432                    let dest = self.project_index(&dest, i.into())?;
433
434                    let val = if mask != 0 { yes } else { no };
435                    self.write_immediate(*val, &dest)?;
436                }
437                // The remaining bits of the mask are ignored.
438            }
439            // Converts a "vector of bool" into a bitmask.
440            sym::simd_bitmask => {
441                let (op, op_len) = self.project_to_simd(&args[0])?;
442                let bitmask_len = op_len.next_multiple_of(8);
443                if bitmask_len > 64 {
444                    throw_unsup_format!(
445                        "simd_bitmask: vectors larger than 64 elements are currently not supported"
446                    );
447                }
448
449                let op_len = u32::try_from(op_len).unwrap();
450                let mut res = 0u64;
451                for i in 0..op_len {
452                    let op = self.read_immediate(&self.project_index(&op, i.into())?)?;
453                    if simd_element_to_bool(op)? {
454                        let bit_i = simd_bitmask_index(i, op_len, self.tcx.data_layout.endian);
455                        res |= 1u64.strict_shl(bit_i);
456                    }
457                }
458                // Write the result, depending on the `dest` type.
459                // Returns either an unsigned integer or array of `u8`.
460                match dest.layout.ty.kind() {
461                    ty::Uint(_) => {
462                        // Any larger integer type is fine, it will be zero-extended.
463                        assert!(dest.layout.size.bits() >= bitmask_len);
464                        self.write_scalar(Scalar::from_uint(res, dest.layout.size), &dest)?;
465                    }
466                    ty::Array(elem, _len) if elem == &self.tcx.types.u8 => {
467                        // The array must have exactly the right size.
468                        assert_eq!(dest.layout.size.bits(), bitmask_len);
469                        // We have to write the result byte-for-byte.
470                        let res_size = dest.layout.size.bytes_usize();
471                        let res_bytes;
472                        let res_bytes_slice = match self.tcx.data_layout.endian {
473                            Endian::Little => {
474                                res_bytes = res.to_le_bytes();
475                                &res_bytes[..res_size] // take the first N bytes
476                            }
477                            Endian::Big => {
478                                res_bytes = res.to_be_bytes();
479                                &res_bytes[res_bytes.len().strict_sub(res_size)..] // take the last N bytes
480                            }
481                        };
482                        self.write_bytes_ptr(dest.ptr(), res_bytes_slice.iter().cloned())?;
483                    }
484                    _ => bug!("simd_bitmask: invalid return type {}", dest.layout.ty),
485                }
486            }
487            sym::simd_cast
488            | sym::simd_as
489            | sym::simd_cast_ptr
490            | sym::simd_with_exposed_provenance => {
491                let (op, op_len) = self.project_to_simd(&args[0])?;
492                let (dest, dest_len) = self.project_to_simd(&dest)?;
493
494                assert_eq!(dest_len, op_len);
495
496                let unsafe_cast = intrinsic_name == sym::simd_cast;
497                let safe_cast = intrinsic_name == sym::simd_as;
498                let ptr_cast = intrinsic_name == sym::simd_cast_ptr;
499                let from_exposed_cast = intrinsic_name == sym::simd_with_exposed_provenance;
500
501                for i in 0..dest_len {
502                    let op = self.read_immediate(&self.project_index(&op, i)?)?;
503                    let dest = self.project_index(&dest, i)?;
504
505                    let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) {
506                        // Int-to-(int|float): always safe
507                        (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_))
508                            if safe_cast || unsafe_cast =>
509                            self.int_to_int_or_float(&op, dest.layout)?,
510                        // Float-to-float: always safe
511                        (ty::Float(_), ty::Float(_)) if safe_cast || unsafe_cast =>
512                            self.float_to_float_or_int(&op, dest.layout)?,
513                        // Float-to-int in safe mode
514                        (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast =>
515                            self.float_to_float_or_int(&op, dest.layout)?,
516                        // Float-to-int in unchecked mode
517                        (ty::Float(_), ty::Int(_) | ty::Uint(_)) if unsafe_cast => {
518                            self.float_to_int_checked(&op, dest.layout, Round::TowardZero)?
519                                .ok_or_else(|| {
520                                    err_ub_format!(
521                                        "`simd_cast` intrinsic called on {op} which cannot be represented in target type `{:?}`",
522                                        dest.layout.ty
523                                    )
524                                })?
525                        }
526                        // Ptr-to-ptr cast
527                        (ty::RawPtr(..), ty::RawPtr(..)) if ptr_cast =>
528                            self.ptr_to_ptr(&op, dest.layout)?,
529                        // Int->Ptr casts
530                        (ty::Int(_) | ty::Uint(_), ty::RawPtr(..)) if from_exposed_cast =>
531                            self.pointer_with_exposed_provenance_cast(&op, dest.layout)?,
532                        // Error otherwise
533                        _ =>
534                            throw_unsup_format!(
535                                "Unsupported SIMD cast from element type {from_ty} to {to_ty}",
536                                from_ty = op.layout.ty,
537                                to_ty = dest.layout.ty,
538                            ),
539                    };
540                    self.write_immediate(*val, &dest)?;
541                }
542            }
543            sym::simd_shuffle_const_generic => {
544                let (left, left_len) = self.project_to_simd(&args[0])?;
545                let (right, right_len) = self.project_to_simd(&args[1])?;
546                let (dest, dest_len) = self.project_to_simd(&dest)?;
547
548                let index = generic_args[2].expect_const().to_value().valtree.unwrap_branch();
549                let index_len = index.len();
550
551                assert_eq!(left_len, right_len);
552                assert_eq!(u64::try_from(index_len).unwrap(), dest_len);
553
554                for i in 0..dest_len {
555                    let src_index: u64 =
556                        index[usize::try_from(i).unwrap()].unwrap_leaf().to_u32().into();
557                    let dest = self.project_index(&dest, i)?;
558
559                    let val = if src_index < left_len {
560                        self.read_immediate(&self.project_index(&left, src_index)?)?
561                    } else if src_index < left_len.strict_add(right_len) {
562                        let right_idx = src_index.strict_sub(left_len);
563                        self.read_immediate(&self.project_index(&right, right_idx)?)?
564                    } else {
565                        throw_ub_format!(
566                            "`simd_shuffle_const_generic` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}"
567                        );
568                    };
569                    self.write_immediate(*val, &dest)?;
570                }
571            }
572            sym::simd_shuffle => {
573                let (left, left_len) = self.project_to_simd(&args[0])?;
574                let (right, right_len) = self.project_to_simd(&args[1])?;
575                let (index, index_len) = self.project_to_simd(&args[2])?;
576                let (dest, dest_len) = self.project_to_simd(&dest)?;
577
578                assert_eq!(left_len, right_len);
579                assert_eq!(index_len, dest_len);
580
581                for i in 0..dest_len {
582                    let src_index: u64 = self
583                        .read_immediate(&self.project_index(&index, i)?)?
584                        .to_scalar()
585                        .to_u32()?
586                        .into();
587                    let dest = self.project_index(&dest, i)?;
588
589                    let val = if src_index < left_len {
590                        self.read_immediate(&self.project_index(&left, src_index)?)?
591                    } else if src_index < left_len.strict_add(right_len) {
592                        let right_idx = src_index.strict_sub(left_len);
593                        self.read_immediate(&self.project_index(&right, right_idx)?)?
594                    } else {
595                        throw_ub_format!(
596                            "`simd_shuffle` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}"
597                        );
598                    };
599                    self.write_immediate(*val, &dest)?;
600                }
601            }
602            sym::simd_gather => {
603                let (passthru, passthru_len) = self.project_to_simd(&args[0])?;
604                let (ptrs, ptrs_len) = self.project_to_simd(&args[1])?;
605                let (mask, mask_len) = self.project_to_simd(&args[2])?;
606                let (dest, dest_len) = self.project_to_simd(&dest)?;
607
608                assert_eq!(dest_len, passthru_len);
609                assert_eq!(dest_len, ptrs_len);
610                assert_eq!(dest_len, mask_len);
611
612                for i in 0..dest_len {
613                    let passthru = self.read_immediate(&self.project_index(&passthru, i)?)?;
614                    let ptr = self.read_immediate(&self.project_index(&ptrs, i)?)?;
615                    let mask = self.read_immediate(&self.project_index(&mask, i)?)?;
616                    let dest = self.project_index(&dest, i)?;
617
618                    let val = if simd_element_to_bool(mask)? {
619                        let place = self.deref_pointer(&ptr)?;
620                        self.read_immediate(&place)?
621                    } else {
622                        passthru
623                    };
624                    self.write_immediate(*val, &dest)?;
625                }
626            }
627            sym::simd_scatter => {
628                let (value, value_len) = self.project_to_simd(&args[0])?;
629                let (ptrs, ptrs_len) = self.project_to_simd(&args[1])?;
630                let (mask, mask_len) = self.project_to_simd(&args[2])?;
631
632                assert_eq!(ptrs_len, value_len);
633                assert_eq!(ptrs_len, mask_len);
634
635                for i in 0..ptrs_len {
636                    let value = self.read_immediate(&self.project_index(&value, i)?)?;
637                    let ptr = self.read_immediate(&self.project_index(&ptrs, i)?)?;
638                    let mask = self.read_immediate(&self.project_index(&mask, i)?)?;
639
640                    if simd_element_to_bool(mask)? {
641                        let place = self.deref_pointer(&ptr)?;
642                        self.write_immediate(*value, &place)?;
643                    }
644                }
645            }
646            sym::simd_masked_load => {
647                let dest_layout = dest.layout;
648
649                let (mask, mask_len) = self.project_to_simd(&args[0])?;
650                let ptr = self.read_pointer(&args[1])?;
651                let (default, default_len) = self.project_to_simd(&args[2])?;
652                let (dest, dest_len) = self.project_to_simd(&dest)?;
653
654                assert_eq!(dest_len, mask_len);
655                assert_eq!(dest_len, default_len);
656
657                self.check_simd_ptr_alignment(
658                    ptr,
659                    dest_layout,
660                    generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
661                        .unwrap_leaf()
662                        .to_simd_alignment(),
663                )?;
664
665                for i in 0..dest_len {
666                    let mask = self.read_immediate(&self.project_index(&mask, i)?)?;
667                    let default = self.read_immediate(&self.project_index(&default, i)?)?;
668                    let dest = self.project_index(&dest, i)?;
669
670                    let val = if simd_element_to_bool(mask)? {
671                        // Size * u64 is implemented as always checked
672                        let ptr = ptr.wrapping_offset(dest.layout.size * i, self);
673                        // we have already checked the alignment requirements
674                        let place = self.ptr_to_mplace_unaligned(ptr, dest.layout);
675                        self.read_immediate(&place)?
676                    } else {
677                        default
678                    };
679                    self.write_immediate(*val, &dest)?;
680                }
681            }
682            sym::simd_masked_store => {
683                let (mask, mask_len) = self.project_to_simd(&args[0])?;
684                let ptr = self.read_pointer(&args[1])?;
685                let (vals, vals_len) = self.project_to_simd(&args[2])?;
686
687                assert_eq!(mask_len, vals_len);
688
689                self.check_simd_ptr_alignment(
690                    ptr,
691                    args[2].layout,
692                    generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0]
693                        .unwrap_leaf()
694                        .to_simd_alignment(),
695                )?;
696
697                for i in 0..vals_len {
698                    let mask = self.read_immediate(&self.project_index(&mask, i)?)?;
699                    let val = self.read_immediate(&self.project_index(&vals, i)?)?;
700
701                    if simd_element_to_bool(mask)? {
702                        // Size * u64 is implemented as always checked
703                        let ptr = ptr.wrapping_offset(val.layout.size * i, self);
704                        // we have already checked the alignment requirements
705                        let place = self.ptr_to_mplace_unaligned(ptr, val.layout);
706                        self.write_immediate(*val, &place)?
707                    };
708                }
709            }
710            sym::simd_fma | sym::simd_relaxed_fma => {
711                // `simd_fma` should always deterministically use `mul_add`, whereas `relaxed_fma`
712                // is non-deterministic, and can use either `mul_add` or `a * b + c`
713                let typ = match intrinsic_name {
714                    sym::simd_fma => MulAddType::Fused,
715                    sym::simd_relaxed_fma => MulAddType::Nondeterministic,
716                    _ => unreachable!(),
717                };
718
719                let (a, a_len) = self.project_to_simd(&args[0])?;
720                let (b, b_len) = self.project_to_simd(&args[1])?;
721                let (c, c_len) = self.project_to_simd(&args[2])?;
722                let (dest, dest_len) = self.project_to_simd(&dest)?;
723
724                assert_eq!(dest_len, a_len);
725                assert_eq!(dest_len, b_len);
726                assert_eq!(dest_len, c_len);
727
728                for i in 0..dest_len {
729                    let a = self.read_scalar(&self.project_index(&a, i)?)?;
730                    let b = self.read_scalar(&self.project_index(&b, i)?)?;
731                    let c = self.read_scalar(&self.project_index(&c, i)?)?;
732                    let dest = self.project_index(&dest, i)?;
733
734                    let ty::Float(float_ty) = dest.layout.ty.kind() else {
735                        span_bug!(self.cur_span(), "{} operand is not a float", intrinsic_name)
736                    };
737
738                    let val = match float_ty {
739                        FloatTy::F16 => self.float_muladd::<Half>(a, b, c, typ)?,
740                        FloatTy::F32 => self.float_muladd::<Single>(a, b, c, typ)?,
741                        FloatTy::F64 => self.float_muladd::<Double>(a, b, c, typ)?,
742                        FloatTy::F128 => self.float_muladd::<Quad>(a, b, c, typ)?,
743                    };
744                    self.write_scalar(val, &dest)?;
745                }
746            }
747
748            // Unsupported intrinsic: skip the return_to_block below.
749            _ => return interp_ok(false),
750        }
751
752        trace!("{:?}", self.dump_place(&dest.clone().into()));
753        self.return_to_block(ret)?;
754        interp_ok(true)
755    }
756
757    fn fminmax_op(
758        &self,
759        op: MinMax,
760        left: &ImmTy<'tcx, M::Provenance>,
761        right: &ImmTy<'tcx, M::Provenance>,
762    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
763        assert_eq!(left.layout.ty, right.layout.ty);
764        let ty::Float(float_ty) = left.layout.ty.kind() else {
765            bug!("fmax operand is not a float")
766        };
767        let left = left.to_scalar();
768        let right = right.to_scalar();
769        interp_ok(match float_ty {
770            FloatTy::F16 => self.float_minmax::<Half>(left, right, op)?,
771            FloatTy::F32 => self.float_minmax::<Single>(left, right, op)?,
772            FloatTy::F64 => self.float_minmax::<Double>(left, right, op)?,
773            FloatTy::F128 => self.float_minmax::<Quad>(left, right, op)?,
774        })
775    }
776
777    fn check_simd_ptr_alignment(
778        &self,
779        ptr: Pointer<Option<M::Provenance>>,
780        vector_layout: TyAndLayout<'tcx>,
781        alignment: SimdAlign,
782    ) -> InterpResult<'tcx> {
783        assert_matches!(vector_layout.backend_repr, BackendRepr::SimdVector { .. });
784
785        let align = match alignment {
786            ty::SimdAlign::Unaligned => {
787                // The pointer is supposed to be unaligned, so no check is required.
788                return interp_ok(());
789            }
790            ty::SimdAlign::Element => {
791                // Take the alignment of the only field, which is an array and therefore has the same
792                // alignment as the element type.
793                vector_layout.field(self, 0).align.abi
794            }
795            ty::SimdAlign::Vector => vector_layout.align.abi,
796        };
797
798        self.check_ptr_align(ptr, align)
799    }
800}
801
802fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 {
803    assert!(idx < vec_len);
804    match endianness {
805        Endian::Little => idx,
806        #[expect(clippy::arithmetic_side_effects)] // idx < vec_len
807        Endian::Big => vec_len - 1 - idx, // reverse order of bits
808    }
809}
810
811fn bool_to_simd_element<Prov: Provenance>(b: bool, size: Size) -> Scalar<Prov> {
812    // SIMD uses all-1 as pattern for "true". In two's complement,
813    // -1 has all its bits set to one and `from_int` will truncate or
814    // sign-extend it to `size` as required.
815    let val = if b { -1 } else { 0 };
816    Scalar::from_int(val, size)
817}
818
819fn simd_element_to_bool<Prov: Provenance>(elem: ImmTy<'_, Prov>) -> InterpResult<'_, bool> {
820    assert!(
821        matches!(elem.layout.ty.kind(), ty::Int(_) | ty::Uint(_)),
822        "SIMD mask element type must be an integer, but this is `{}`",
823        elem.layout.ty
824    );
825    let val = elem.to_scalar().to_int(elem.layout.size)?;
826    interp_ok(match val {
827        0 => false,
828        -1 => true,
829        _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"),
830    })
831}
rustc_const_eval/interpret/intrinsics/simd.rs

rustc_const_eval/interpret/intrinsics/
simd.rs
