miri/intrinsics/x86/
gfni.rs

1use rustc_span::Symbol;
2
3use crate::*;
4
5impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
6pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
7    fn emulate_x86_gfni_intrinsic(
8        &mut self,
9        link_name: Symbol,
10        args: &[OpTy<'tcx>],
11        dest: &MPlaceTy<'tcx>,
12    ) -> InterpResult<'tcx, EmulateItemResult> {
13        let this = self.eval_context_mut();
14
15        // Prefix should have already been checked.
16        let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.").unwrap();
17
18        this.expect_target_feature_for_intrinsic(link_name, "gfni")?;
19        if unprefixed_name.ends_with(".256") {
20            this.expect_target_feature_for_intrinsic(link_name, "avx")?;
21        } else if unprefixed_name.ends_with(".512") {
22            this.expect_target_feature_for_intrinsic(link_name, "avx512f")?;
23        }
24
25        match unprefixed_name {
26            // Used to implement the `_mm{, 256, 512}_gf2p8affine_epi64_epi8` functions.
27            // See `affine_transform` for details.
28            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8affine_
29            "vgf2p8affineqb.128" | "vgf2p8affineqb.256" | "vgf2p8affineqb.512" => {
30                let [left, right, imm8] = this.check_shim_sig_unadjusted(link_name, args)?;
31                affine_transform(this, left, right, imm8, dest, /* inverse */ false)?;
32            }
33            // Used to implement the `_mm{, 256, 512}_gf2p8affineinv_epi64_epi8` functions.
34            // See `affine_transform` for details.
35            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8affineinv
36            "vgf2p8affineinvqb.128" | "vgf2p8affineinvqb.256" | "vgf2p8affineinvqb.512" => {
37                let [left, right, imm8] = this.check_shim_sig_unadjusted(link_name, args)?;
38                affine_transform(this, left, right, imm8, dest, /* inverse */ true)?;
39            }
40            // Used to implement the `_mm{, 256, 512}_gf2p8mul_epi8` functions.
41            // Multiplies packed 8-bit integers in `left` and `right` in the finite field GF(2^8)
42            // and store the results in `dst`. The field GF(2^8) is represented in
43            // polynomial representation with the reduction polynomial x^8 + x^4 + x^3 + x + 1.
44            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8mul
45            "vgf2p8mulb.128" | "vgf2p8mulb.256" | "vgf2p8mulb.512" => {
46                let [left, right] = this.check_shim_sig_unadjusted(link_name, args)?;
47                let (left, left_len) = this.project_to_simd(left)?;
48                let (right, right_len) = this.project_to_simd(right)?;
49                let (dest, dest_len) = this.project_to_simd(dest)?;
50
51                assert_eq!(left_len, right_len);
52                assert_eq!(dest_len, right_len);
53
54                for i in 0..dest_len {
55                    let left = this.read_scalar(&this.project_index(&left, i)?)?.to_u8()?;
56                    let right = this.read_scalar(&this.project_index(&right, i)?)?.to_u8()?;
57                    let dest = this.project_index(&dest, i)?;
58                    this.write_scalar(Scalar::from_u8(gf2p8_mul(left, right)), &dest)?;
59                }
60            }
61            _ => return interp_ok(EmulateItemResult::NotSupported),
62        }
63        interp_ok(EmulateItemResult::NeedsReturn)
64    }
65}
66
67/// Calculates the affine transformation `right * left + imm8` inside the finite field GF(2^8).
68/// `right` is an 8x8 bit matrix, `left` and `imm8` are bit vectors.
69/// If `inverse` is set, then the inverse transformation with respect to the reduction polynomial
70/// x^8 + x^4 + x^3 + x + 1 is performed instead.
71fn affine_transform<'tcx>(
72    ecx: &mut MiriInterpCx<'tcx>,
73    left: &OpTy<'tcx>,
74    right: &OpTy<'tcx>,
75    imm8: &OpTy<'tcx>,
76    dest: &MPlaceTy<'tcx>,
77    inverse: bool,
78) -> InterpResult<'tcx, ()> {
79    let (left, left_len) = ecx.project_to_simd(left)?;
80    let (right, right_len) = ecx.project_to_simd(right)?;
81    let (dest, dest_len) = ecx.project_to_simd(dest)?;
82
83    assert_eq!(dest_len, right_len);
84    assert_eq!(dest_len, left_len);
85
86    let imm8 = ecx.read_scalar(imm8)?.to_u8()?;
87
88    // Each 8x8 bit matrix gets multiplied with eight bit vectors.
89    // Therefore, the iteration is done in chunks of eight.
90    for i in (0..dest_len).step_by(8) {
91        // Get the bit matrix.
92        let mut matrix = [0u8; 8];
93        for j in 0..8 {
94            matrix[usize::try_from(j).unwrap()] =
95                ecx.read_scalar(&ecx.project_index(&right, i.wrapping_add(j))?)?.to_u8()?;
96        }
97
98        // Multiply the matrix with the vector and perform the addition.
99        for j in 0..8 {
100            let index = i.wrapping_add(j);
101            let left = ecx.read_scalar(&ecx.project_index(&left, index)?)?.to_u8()?;
102            let left = if inverse { TABLE[usize::from(left)] } else { left };
103
104            let mut res = 0;
105
106            // Do the matrix multiplication.
107            for bit in 0u8..8 {
108                let mut b = matrix[usize::from(bit)] & left;
109
110                // Calculate the parity bit.
111                b = (b & 0b1111) ^ (b >> 4);
112                b = (b & 0b11) ^ (b >> 2);
113                b = (b & 0b1) ^ (b >> 1);
114
115                res |= b << 7u8.wrapping_sub(bit);
116            }
117
118            // Perform the addition.
119            res ^= imm8;
120
121            let dest = ecx.project_index(&dest, index)?;
122            ecx.write_scalar(Scalar::from_u8(res), &dest)?;
123        }
124    }
125
126    interp_ok(())
127}
128
129/// A lookup table for computing the inverse byte for the inverse affine transformation.
130// This is a evaluated at compile time. Trait based conversion is not available.
131/// See <https://www.corsix.org/content/galois-field-instructions-2021-cpus> for the
132/// definition of `gf_inv` which was used for the creation of this table.
133static TABLE: [u8; 256] = {
134    let mut array = [0; 256];
135
136    let mut i = 1;
137    while i < 256 {
138        #[expect(clippy::as_conversions)] // no `try_from` in const...
139        let mut x = i as u8;
140        let mut y = gf2p8_mul(x, x);
141        x = y;
142        let mut j = 2;
143        while j < 8 {
144            x = gf2p8_mul(x, x);
145            y = gf2p8_mul(x, y);
146            j += 1;
147        }
148        array[i] = y;
149        i += 1;
150    }
151
152    array
153};
154
155/// Multiplies packed 8-bit integers in `left` and `right` in the finite field GF(2^8)
156/// and store the results in `dst`. The field GF(2^8) is represented in
157/// polynomial representation with the reduction polynomial x^8 + x^4 + x^3 + x + 1.
158/// See <https://www.corsix.org/content/galois-field-instructions-2021-cpus> for details.
159// This is a const function. Trait based conversion is not available.
160#[expect(clippy::as_conversions)]
161const fn gf2p8_mul(left: u8, right: u8) -> u8 {
162    // This implementation is based on the `gf2p8mul_byte` definition found inside the Intel intrinsics guide.
163    // See https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=gf2p8mul
164    // for more information.
165
166    const POLYNOMIAL: u32 = 0x11b;
167
168    let left = left as u32;
169    let right = right as u32;
170
171    let mut result = 0u32;
172
173    let mut i = 0u32;
174    while i < 8 {
175        if left & (1 << i) != 0 {
176            result ^= right << i;
177        }
178        i = i.wrapping_add(1);
179    }
180
181    let mut i = 14u32;
182    while i >= 8 {
183        if result & (1 << i) != 0 {
184            result ^= POLYNOMIAL << i.wrapping_sub(8);
185        }
186        i = i.wrapping_sub(1);
187    }
188
189    result as u8
190}
miri/intrinsics/x86/gfni.rs

miri/intrinsics/x86/
gfni.rs
