rustc_const_eval/interpret/

operator.rs

use either::Either;
use rustc_abi::Size;
use rustc_apfloat::{Float, FloatConvert};
use rustc_middle::mir::interpret::{InterpResult, PointerArithmetic, Scalar};
use rustc_middle::ty::layout::TyAndLayout;
use rustc_middle::ty::{self, FloatTy, ScalarInt};
use rustc_middle::{bug, mir, span_bug};
use rustc_span::sym;
use tracing::trace;

use super::{ImmTy, InterpCx, Machine, MemPlaceMeta, interp_ok, throw_ub};

/// Describes an atomic RMW operation.
pub enum AtomicRmwOp {
    MirOp {
        op: mir::BinOp,
        /// Indicates whether the result of the operation should be negated (`UnOp::Not`, must be a
        /// boolean/integer-typed operation).
        neg: bool,
    },
    Max,
    Min,
    Swap,
}

impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
    fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> ImmTy<'tcx, M::Provenance> {
        let res = Ord::cmp(&lhs, &rhs);
        return ImmTy::from_ordering(res, *self.tcx);
    }

    fn binary_char_op(&self, bin_op: mir::BinOp, l: char, r: char) -> ImmTy<'tcx, M::Provenance> {
        use rustc_middle::mir::BinOp::*;

        if bin_op == Cmp {
            return self.three_way_compare(l, r);
        }

        let res = match bin_op {
            Eq => l == r,
            Ne => l != r,
            Lt => l < r,
            Le => l <= r,
            Gt => l > r,
            Ge => l >= r,
            _ => span_bug!(self.cur_span(), "Invalid operation on char: {:?}", bin_op),
        };
        ImmTy::from_bool(res, *self.tcx)
    }

    fn binary_bool_op(&self, bin_op: mir::BinOp, l: bool, r: bool) -> ImmTy<'tcx, M::Provenance> {
        use rustc_middle::mir::BinOp::*;

        let res = match bin_op {
            Eq => l == r,
            Ne => l != r,
            Lt => l < r,
            Le => l <= r,
            Gt => l > r,
            Ge => l >= r,
            BitAnd => l & r,
            BitOr => l | r,
            BitXor => l ^ r,
            _ => span_bug!(self.cur_span(), "Invalid operation on bool: {:?}", bin_op),
        };
        ImmTy::from_bool(res, *self.tcx)
    }

    fn binary_float_op<F: Float + FloatConvert<F> + Into<Scalar<M::Provenance>>>(
        &self,
        bin_op: mir::BinOp,
        layout: TyAndLayout<'tcx>,
        l: F,
        r: F,
    ) -> ImmTy<'tcx, M::Provenance> {
        use rustc_middle::mir::BinOp::*;

        // Performs appropriate non-deterministic adjustments of NaN results.
        let adjust_nan = |f: F| -> F { self.adjust_nan(f, &[l, r]) };

        match bin_op {
            Eq => ImmTy::from_bool(l == r, *self.tcx),
            Ne => ImmTy::from_bool(l != r, *self.tcx),
            Lt => ImmTy::from_bool(l < r, *self.tcx),
            Le => ImmTy::from_bool(l <= r, *self.tcx),
            Gt => ImmTy::from_bool(l > r, *self.tcx),
            Ge => ImmTy::from_bool(l >= r, *self.tcx),
            Add => ImmTy::from_scalar(adjust_nan((l + r).value).into(), layout),
            Sub => ImmTy::from_scalar(adjust_nan((l - r).value).into(), layout),
            Mul => ImmTy::from_scalar(adjust_nan((l * r).value).into(), layout),
            Div => ImmTy::from_scalar(adjust_nan((l / r).value).into(), layout),
            Rem => ImmTy::from_scalar(adjust_nan((l % r).value).into(), layout),
            _ => span_bug!(self.cur_span(), "invalid float op: `{:?}`", bin_op),
        }
    }

    fn binary_int_op(
        &self,
        bin_op: mir::BinOp,
        left: &ImmTy<'tcx, M::Provenance>,
        right: &ImmTy<'tcx, M::Provenance>,
    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
        use rustc_middle::mir::BinOp::*;

        // This checks the size, so that we can just assert it below.
        let l = left.to_scalar_int()?;
        let r = right.to_scalar_int()?;
        // Prepare to convert the values to signed or unsigned form.
        let l_signed = || l.to_int(left.layout.size);
        let l_unsigned = || l.to_uint(left.layout.size);
        let r_signed = || r.to_int(right.layout.size);
        let r_unsigned = || r.to_uint(right.layout.size);

        let throw_ub_on_overflow = match bin_op {
            AddUnchecked => Some(sym::unchecked_add),
            SubUnchecked => Some(sym::unchecked_sub),
            MulUnchecked => Some(sym::unchecked_mul),
            ShlUnchecked => Some(sym::unchecked_shl),
            ShrUnchecked => Some(sym::unchecked_shr),
            _ => None,
        };
        let with_overflow = bin_op.is_overflowing();

        // Shift ops can have an RHS with a different numeric type.
        if matches!(bin_op, Shl | ShlUnchecked | Shr | ShrUnchecked) {
            let l_bits = left.layout.size.bits();
            // Compute the equivalent shift modulo `size` that is in the range `0..size`. (This is
            // the one MIR operator that does *not* directly map to a single LLVM operation.)
            let (shift_amount, overflow) = if right.layout.backend_repr.is_signed() {
                let shift_amount = r_signed();
                let rem = shift_amount.rem_euclid(l_bits.into());
                // `rem` is guaranteed positive, so the `unwrap` cannot fail
                (u128::try_from(rem).unwrap(), rem != shift_amount)
            } else {
                let shift_amount = r_unsigned();
                let rem = shift_amount.rem_euclid(l_bits.into());
                (rem, rem != shift_amount)
            };
            let shift_amount = u32::try_from(shift_amount).unwrap(); // we brought this in the range `0..size` so this will always fit
            // Compute the shifted result.
            let result = if left.layout.backend_repr.is_signed() {
                let l = l_signed();
                let result = match bin_op {
                    Shl | ShlUnchecked => l.checked_shl(shift_amount).unwrap(),
                    Shr | ShrUnchecked => l.checked_shr(shift_amount).unwrap(),
                    _ => bug!(),
                };
                ScalarInt::truncate_from_int(result, left.layout.size).0
            } else {
                let l = l_unsigned();
                let result = match bin_op {
                    Shl | ShlUnchecked => l.checked_shl(shift_amount).unwrap(),
                    Shr | ShrUnchecked => l.checked_shr(shift_amount).unwrap(),
                    _ => bug!(),
                };
                ScalarInt::truncate_from_uint(result, left.layout.size).0
            };

            if overflow && let Some(intrinsic) = throw_ub_on_overflow {
                throw_ub!(ShiftOverflow {
                    intrinsic,
                    shift_amount: if right.layout.backend_repr.is_signed() {
                        Either::Right(r_signed())
                    } else {
                        Either::Left(r_unsigned())
                    }
                });
            }

            return interp_ok(ImmTy::from_scalar_int(result, left.layout));
        }

        // For the remaining ops, the types must be the same on both sides
        if left.layout.ty != right.layout.ty {
            span_bug!(
                self.cur_span(),
                "invalid asymmetric binary op {bin_op:?}: {l:?} ({l_ty}), {r:?} ({r_ty})",
                l_ty = left.layout.ty,
                r_ty = right.layout.ty,
            )
        }

        let size = left.layout.size;

        // Operations that need special treatment for signed integers
        if left.layout.backend_repr.is_signed() {
            let op: Option<fn(&i128, &i128) -> bool> = match bin_op {
                Lt => Some(i128::lt),
                Le => Some(i128::le),
                Gt => Some(i128::gt),
                Ge => Some(i128::ge),
                _ => None,
            };
            if let Some(op) = op {
                return interp_ok(ImmTy::from_bool(op(&l_signed(), &r_signed()), *self.tcx));
            }
            if bin_op == Cmp {
                return interp_ok(self.three_way_compare(l_signed(), r_signed()));
            }
            let op: Option<fn(i128, i128) -> (i128, bool)> = match bin_op {
                Div if r.is_null() => throw_ub!(DivisionByZero),
                Rem if r.is_null() => throw_ub!(RemainderByZero),
                Div => Some(i128::overflowing_div),
                Rem => Some(i128::overflowing_rem),
                Add | AddUnchecked | AddWithOverflow => Some(i128::overflowing_add),
                Sub | SubUnchecked | SubWithOverflow => Some(i128::overflowing_sub),
                Mul | MulUnchecked | MulWithOverflow => Some(i128::overflowing_mul),
                _ => None,
            };
            if let Some(op) = op {
                let l = l_signed();
                let r = r_signed();

                // We need a special check for overflowing Rem and Div since they are *UB*
                // on overflow, which can happen with "int_min $OP -1".
                if matches!(bin_op, Rem | Div) {
                    if l == size.signed_int_min() && r == -1 {
                        if bin_op == Rem {
                            throw_ub!(RemainderOverflow)
                        } else {
                            throw_ub!(DivisionOverflow)
                        }
                    }
                }

                let (result, oflo) = op(l, r);
                // This may be out-of-bounds for the result type, so we have to truncate.
                // If that truncation loses any information, we have an overflow.
                let (result, lossy) = ScalarInt::truncate_from_int(result, left.layout.size);
                let overflow = oflo || lossy;
                if overflow && let Some(intrinsic) = throw_ub_on_overflow {
                    throw_ub!(ArithOverflow { intrinsic });
                }
                let res = ImmTy::from_scalar_int(result, left.layout);
                return interp_ok(if with_overflow {
                    let overflow = ImmTy::from_bool(overflow, *self.tcx);
                    ImmTy::from_pair(res, overflow, self)
                } else {
                    res
                });
            }
        }
        // From here on it's okay to treat everything as unsigned.
        let l = l_unsigned();
        let r = r_unsigned();

        if bin_op == Cmp {
            return interp_ok(self.three_way_compare(l, r));
        }

        interp_ok(match bin_op {
            Eq => ImmTy::from_bool(l == r, *self.tcx),
            Ne => ImmTy::from_bool(l != r, *self.tcx),

            Lt => ImmTy::from_bool(l < r, *self.tcx),
            Le => ImmTy::from_bool(l <= r, *self.tcx),
            Gt => ImmTy::from_bool(l > r, *self.tcx),
            Ge => ImmTy::from_bool(l >= r, *self.tcx),

            BitOr => ImmTy::from_uint(l | r, left.layout),
            BitAnd => ImmTy::from_uint(l & r, left.layout),
            BitXor => ImmTy::from_uint(l ^ r, left.layout),

            _ => {
                assert!(!left.layout.backend_repr.is_signed());
                let op: fn(u128, u128) -> (u128, bool) = match bin_op {
                    Add | AddUnchecked | AddWithOverflow => u128::overflowing_add,
                    Sub | SubUnchecked | SubWithOverflow => u128::overflowing_sub,
                    Mul | MulUnchecked | MulWithOverflow => u128::overflowing_mul,
                    Div if r == 0 => throw_ub!(DivisionByZero),
                    Rem if r == 0 => throw_ub!(RemainderByZero),
                    Div => u128::overflowing_div,
                    Rem => u128::overflowing_rem,
                    _ => span_bug!(
                        self.cur_span(),
                        "invalid binary op {:?}: {:?}, {:?} (both {})",
                        bin_op,
                        left,
                        right,
                        right.layout.ty,
                    ),
                };
                let (result, oflo) = op(l, r);
                // Truncate to target type.
                // If that truncation loses any information, we have an overflow.
                let (result, lossy) = ScalarInt::truncate_from_uint(result, left.layout.size);
                let overflow = oflo || lossy;
                if overflow && let Some(intrinsic) = throw_ub_on_overflow {
                    throw_ub!(ArithOverflow { intrinsic });
                }
                let res = ImmTy::from_scalar_int(result, left.layout);
                if with_overflow {
                    let overflow = ImmTy::from_bool(overflow, *self.tcx);
                    ImmTy::from_pair(res, overflow, self)
                } else {
                    res
                }
            }
        })
    }

    /// Computes the total size of this access, `count * elem_size`,
    /// checking for overflow beyond isize::MAX.
    pub fn compute_size_in_bytes(&self, elem_size: Size, count: u64) -> Option<Size> {
        // `checked_mul` applies `u64` limits independent of the target pointer size... but the
        // subsequent check for `max_size_of_val` means we also handle 32bit targets correctly.
        // (We cannot use `Size::checked_mul` as that enforces `obj_size_bound` as the limit, which
        // would be wrong here.)
        elem_size
            .bytes()
            .checked_mul(count)
            .map(Size::from_bytes)
            .filter(|&total| total <= self.max_size_of_val())
    }

    fn binary_ptr_op(
        &self,
        bin_op: mir::BinOp,
        left: &ImmTy<'tcx, M::Provenance>,
        right: &ImmTy<'tcx, M::Provenance>,
    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
        use rustc_middle::mir::BinOp::*;

        match bin_op {
            // Pointer ops that are always supported.
            Offset => {
                let ptr = left.to_scalar().to_pointer(self);
                let pointee_ty = left.layout.ty.builtin_deref(true).unwrap();
                let pointee_layout = self.layout_of(pointee_ty)?;
                assert!(pointee_layout.is_sized());

                // The size always fits in `i64` as it can be at most `isize::MAX`.
                let pointee_size = i64::try_from(pointee_layout.size.bytes()).unwrap();
                // This uses the same type as `right`, which can be `isize` or `usize`.
                // `pointee_size` is guaranteed to fit into both types.
                let pointee_size = ImmTy::from_int(pointee_size, right.layout);
                // Multiply element size and element count.
                let (val, overflowed) = self
                    .binary_op(mir::BinOp::MulWithOverflow, right, &pointee_size)?
                    .to_scalar_pair();
                // This must not overflow.
                if overflowed.to_bool()? {
                    throw_ub!(PointerArithOverflow)
                }

                let offset_bytes = val.to_target_isize(self)?;
                if !right.layout.backend_repr.is_signed() && offset_bytes < 0 {
                    // We were supposed to do an unsigned offset but the result is negative -- this
                    // can only mean that the cast wrapped around.
                    throw_ub!(PointerArithOverflow)
                }
                let offset_ptr = self.ptr_offset_inbounds(ptr, offset_bytes)?;
                interp_ok(ImmTy::from_scalar(
                    Scalar::from_maybe_pointer(offset_ptr, self),
                    left.layout,
                ))
            }

            // Fall back to machine hook so Miri can support more pointer ops.
            _ => M::binary_ptr_op(self, bin_op, left, right),
        }
    }

    /// Returns the result of the specified operation.
    ///
    /// Whether this produces a scalar or a pair depends on the specific `bin_op`.
    pub fn binary_op(
        &self,
        bin_op: mir::BinOp,
        left: &ImmTy<'tcx, M::Provenance>,
        right: &ImmTy<'tcx, M::Provenance>,
    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
        trace!(
            "Running binary op {:?}: {:?} ({}), {:?} ({})",
            bin_op, *left, left.layout.ty, *right, right.layout.ty
        );

        match left.layout.ty.kind() {
            ty::Char => {
                assert_eq!(left.layout.ty, right.layout.ty);
                let left = left.to_scalar();
                let right = right.to_scalar();
                interp_ok(self.binary_char_op(bin_op, left.to_char()?, right.to_char()?))
            }
            ty::Bool => {
                assert_eq!(left.layout.ty, right.layout.ty);
                let left = left.to_scalar();
                let right = right.to_scalar();
                interp_ok(self.binary_bool_op(bin_op, left.to_bool()?, right.to_bool()?))
            }
            ty::Float(fty) => {
                assert_eq!(left.layout.ty, right.layout.ty);
                let layout = left.layout;
                let left = left.to_scalar();
                let right = right.to_scalar();
                interp_ok(match fty {
                    FloatTy::F16 => {
                        self.binary_float_op(bin_op, layout, left.to_f16()?, right.to_f16()?)
                    }
                    FloatTy::F32 => {
                        self.binary_float_op(bin_op, layout, left.to_f32()?, right.to_f32()?)
                    }
                    FloatTy::F64 => {
                        self.binary_float_op(bin_op, layout, left.to_f64()?, right.to_f64()?)
                    }
                    FloatTy::F128 => {
                        self.binary_float_op(bin_op, layout, left.to_f128()?, right.to_f128()?)
                    }
                })
            }
            _ if left.layout.ty.is_integral() => {
                // the RHS type can be different, e.g. for shifts -- but it has to be integral, too
                assert!(
                    right.layout.ty.is_integral(),
                    "Unexpected types for BinOp: {} {:?} {}",
                    left.layout.ty,
                    bin_op,
                    right.layout.ty
                );

                self.binary_int_op(bin_op, left, right)
            }
            _ if left.layout.ty.is_any_ptr() => {
                // The RHS type must be a `pointer` *or an integer type* (for `Offset`).
                // (Even when both sides are pointers, their type might differ, see issue #91636)
                assert!(
                    right.layout.ty.is_any_ptr() || right.layout.ty.is_integral(),
                    "Unexpected types for BinOp: {} {:?} {}",
                    left.layout.ty,
                    bin_op,
                    right.layout.ty
                );

                self.binary_ptr_op(bin_op, left, right)
            }
            _ => span_bug!(
                self.cur_span(),
                "Invalid MIR: bad LHS type for binop: {}",
                left.layout.ty
            ),
        }
    }

    /// Returns the result of the specified operation.
    pub fn unary_op(
        &self,
        un_op: mir::UnOp,
        val: &ImmTy<'tcx, M::Provenance>,
    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
        use rustc_middle::mir::UnOp::*;

        let layout = val.layout;
        trace!("Running unary op {:?}: {:?} ({})", un_op, val, layout.ty);

        match layout.ty.kind() {
            ty::Bool => {
                let val = val.to_scalar();
                let val = val.to_bool()?;
                let res = match un_op {
                    Not => !val,
                    _ => span_bug!(self.cur_span(), "Invalid bool op {:?}", un_op),
                };
                interp_ok(ImmTy::from_bool(res, *self.tcx))
            }
            ty::Float(fty) => {
                let val = val.to_scalar();
                if un_op != Neg {
                    span_bug!(self.cur_span(), "Invalid float op {:?}", un_op);
                }

                // No NaN adjustment here, `-` is a bitwise operation!
                let res = match fty {
                    FloatTy::F16 => Scalar::from_f16(-val.to_f16()?),
                    FloatTy::F32 => Scalar::from_f32(-val.to_f32()?),
                    FloatTy::F64 => Scalar::from_f64(-val.to_f64()?),
                    FloatTy::F128 => Scalar::from_f128(-val.to_f128()?),
                };
                interp_ok(ImmTy::from_scalar(res, layout))
            }
            ty::Int(..) => {
                let val = val.to_scalar().to_int(layout.size)?;
                let res = match un_op {
                    Not => !val,
                    Neg => val.wrapping_neg(),
                    _ => span_bug!(self.cur_span(), "Invalid integer op {:?}", un_op),
                };
                let res = ScalarInt::truncate_from_int(res, layout.size).0;
                interp_ok(ImmTy::from_scalar(res.into(), layout))
            }
            ty::Uint(..) => {
                let val = val.to_scalar().to_uint(layout.size)?;
                let res = match un_op {
                    Not => !val,
                    _ => span_bug!(self.cur_span(), "Invalid unsigned integer op {:?}", un_op),
                };
                let res = ScalarInt::truncate_from_uint(res, layout.size).0;
                interp_ok(ImmTy::from_scalar(res.into(), layout))
            }
            ty::RawPtr(..) | ty::Ref(..) => {
                assert_eq!(un_op, PtrMetadata);
                self.deref_pointer(val)?; // validity check
                let (_, meta) = val.to_scalar_and_meta();
                interp_ok(match meta {
                    MemPlaceMeta::Meta(scalar) => {
                        let ty = un_op.ty(*self.tcx, val.layout.ty);
                        let layout = self.layout_of(ty)?;
                        ImmTy::from_scalar(scalar, layout)
                    }
                    MemPlaceMeta::None => {
                        let unit_layout = self.layout_of(self.tcx.types.unit)?;
                        ImmTy::uninit(unit_layout)
                    }
                })
            }
            _ => {
                bug!("Unexpected unary op argument {val:?}")
            }
        }
    }

    pub fn atomic_rmw_op(
        &self,
        op: AtomicRmwOp,
        left: &ImmTy<'tcx, M::Provenance>,
        right: &ImmTy<'tcx, M::Provenance>,
    ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
        interp_ok(match op {
            AtomicRmwOp::MirOp { op, neg } => {
                let val = self.binary_op(op, &left, right)?;
                if neg { self.unary_op(mir::UnOp::Not, &val)? } else { val }
            }
            AtomicRmwOp::Max => {
                let lt = self.binary_op(mir::BinOp::Lt, &left, right)?.to_scalar().to_bool()?;
                if lt { right } else { &left }.clone()
            }
            AtomicRmwOp::Min => {
                let lt = self.binary_op(mir::BinOp::Lt, &left, right)?.to_scalar().to_bool()?;
                if lt { &left } else { right }.clone()
            }
            AtomicRmwOp::Swap => right.clone(),
        })
    }
}