rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks => {
27                        let target = target.unwrap();
28                        block.statements.push(Statement::new(
29                            terminator.source_info,
30                            StatementKind::Assign(Box::new((
31                                *destination,
32                                Rvalue::NullaryOp(NullOp::UbChecks, tcx.types.bool),
33                            ))),
34                        ));
35                        terminator.kind = TerminatorKind::Goto { target };
36                    }
37                    sym::forget => {
38                        let target = target.unwrap();
39                        block.statements.push(Statement::new(
40                            terminator.source_info,
41                            StatementKind::Assign(Box::new((
42                                *destination,
43                                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
44                                    span: terminator.source_info.span,
45                                    user_ty: None,
46                                    const_: Const::zero_sized(tcx.types.unit),
47                                }))),
48                            ))),
49                        ));
50                        terminator.kind = TerminatorKind::Goto { target };
51                    }
52                    sym::copy_nonoverlapping => {
53                        let target = target.unwrap();
54                        let Ok([src, dst, count]) = take_array(args) else {
55                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
56                        };
57                        block.statements.push(Statement::new(
58                            terminator.source_info,
59                            StatementKind::Intrinsic(Box::new(
60                                NonDivergingIntrinsic::CopyNonOverlapping(
61                                    rustc_middle::mir::CopyNonOverlapping {
62                                        src: src.node,
63                                        dst: dst.node,
64                                        count: count.node,
65                                    },
66                                ),
67                            )),
68                        ));
69                        terminator.kind = TerminatorKind::Goto { target };
70                    }
71                    sym::assume => {
72                        let target = target.unwrap();
73                        let Ok([arg]) = take_array(args) else {
74                            bug!("Wrong arguments for assume intrinsic");
75                        };
76                        block.statements.push(Statement::new(
77                            terminator.source_info,
78                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
79                                arg.node,
80                            ))),
81                        ));
82                        terminator.kind = TerminatorKind::Goto { target };
83                    }
84                    sym::wrapping_add
85                    | sym::wrapping_sub
86                    | sym::wrapping_mul
87                    | sym::three_way_compare
88                    | sym::unchecked_add
89                    | sym::unchecked_sub
90                    | sym::unchecked_mul
91                    | sym::unchecked_div
92                    | sym::unchecked_rem
93                    | sym::unchecked_shl
94                    | sym::unchecked_shr => {
95                        let target = target.unwrap();
96                        let Ok([lhs, rhs]) = take_array(args) else {
97                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
98                        };
99                        let bin_op = match intrinsic.name {
100                            sym::wrapping_add => BinOp::Add,
101                            sym::wrapping_sub => BinOp::Sub,
102                            sym::wrapping_mul => BinOp::Mul,
103                            sym::three_way_compare => BinOp::Cmp,
104                            sym::unchecked_add => BinOp::AddUnchecked,
105                            sym::unchecked_sub => BinOp::SubUnchecked,
106                            sym::unchecked_mul => BinOp::MulUnchecked,
107                            sym::unchecked_div => BinOp::Div,
108                            sym::unchecked_rem => BinOp::Rem,
109                            sym::unchecked_shl => BinOp::ShlUnchecked,
110                            sym::unchecked_shr => BinOp::ShrUnchecked,
111                            _ => bug!("unexpected intrinsic"),
112                        };
113                        block.statements.push(Statement::new(
114                            terminator.source_info,
115                            StatementKind::Assign(Box::new((
116                                *destination,
117                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
118                            ))),
119                        ));
120                        terminator.kind = TerminatorKind::Goto { target };
121                    }
122                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
123                        let target = target.unwrap();
124                        let Ok([lhs, rhs]) = take_array(args) else {
125                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
126                        };
127                        let bin_op = match intrinsic.name {
128                            sym::add_with_overflow => BinOp::AddWithOverflow,
129                            sym::sub_with_overflow => BinOp::SubWithOverflow,
130                            sym::mul_with_overflow => BinOp::MulWithOverflow,
131                            _ => bug!("unexpected intrinsic"),
132                        };
133                        block.statements.push(Statement::new(
134                            terminator.source_info,
135                            StatementKind::Assign(Box::new((
136                                *destination,
137                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
138                            ))),
139                        ));
140                        terminator.kind = TerminatorKind::Goto { target };
141                    }
142                    sym::read_via_copy => {
143                        let Ok([arg]) = take_array(args) else {
144                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
145                        };
146                        let derefed_place = if let Some(place) = arg.node.place()
147                            && let Some(local) = place.as_local()
148                        {
149                            tcx.mk_place_deref(local.into())
150                        } else {
151                            span_bug!(
152                                terminator.source_info.span,
153                                "Only passing a local is supported"
154                            );
155                        };
156                        // Add new statement at the end of the block that does the read, and patch
157                        // up the terminator.
158                        block.statements.push(Statement::new(
159                            terminator.source_info,
160                            StatementKind::Assign(Box::new((
161                                *destination,
162                                Rvalue::Use(Operand::Copy(derefed_place)),
163                            ))),
164                        ));
165                        terminator.kind = match *target {
166                            None => {
167                                // No target means this read something uninhabited,
168                                // so it must be unreachable.
169                                TerminatorKind::Unreachable
170                            }
171                            Some(target) => TerminatorKind::Goto { target },
172                        }
173                    }
174                    sym::write_via_move => {
175                        let target = target.unwrap();
176                        let Ok([ptr, val]) = take_array(args) else {
177                            span_bug!(
178                                terminator.source_info.span,
179                                "Wrong number of arguments for write_via_move intrinsic",
180                            );
181                        };
182                        let derefed_place = if let Some(place) = ptr.node.place()
183                            && let Some(local) = place.as_local()
184                        {
185                            tcx.mk_place_deref(local.into())
186                        } else {
187                            span_bug!(
188                                terminator.source_info.span,
189                                "Only passing a local is supported"
190                            );
191                        };
192                        block.statements.push(Statement::new(
193                            terminator.source_info,
194                            StatementKind::Assign(Box::new((derefed_place, Rvalue::Use(val.node)))),
195                        ));
196                        terminator.kind = TerminatorKind::Goto { target };
197                    }
198                    sym::discriminant_value => {
199                        let target = target.unwrap();
200                        let Ok([arg]) = take_array(args) else {
201                            span_bug!(
202                                terminator.source_info.span,
203                                "Wrong arguments for discriminant_value intrinsic"
204                            );
205                        };
206                        let arg = arg.node.place().unwrap();
207                        let arg = tcx.mk_place_deref(arg);
208                        block.statements.push(Statement::new(
209                            terminator.source_info,
210                            StatementKind::Assign(Box::new((
211                                *destination,
212                                Rvalue::Discriminant(arg),
213                            ))),
214                        ));
215                        terminator.kind = TerminatorKind::Goto { target };
216                    }
217                    sym::offset => {
218                        let target = target.unwrap();
219                        let Ok([ptr, delta]) = take_array(args) else {
220                            span_bug!(
221                                terminator.source_info.span,
222                                "Wrong number of arguments for offset intrinsic",
223                            );
224                        };
225                        block.statements.push(Statement::new(
226                            terminator.source_info,
227                            StatementKind::Assign(Box::new((
228                                *destination,
229                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
230                            ))),
231                        ));
232                        terminator.kind = TerminatorKind::Goto { target };
233                    }
234                    sym::slice_get_unchecked => {
235                        let target = target.unwrap();
236                        let Ok([ptrish, index]) = take_array(args) else {
237                            span_bug!(
238                                terminator.source_info.span,
239                                "Wrong number of arguments for {intrinsic:?}",
240                            );
241                        };
242
243                        let place = ptrish.node.place().unwrap();
244                        assert!(!place.is_indirect());
245                        let updated_place = place.project_deeper(
246                            &[
247                                ProjectionElem::Deref,
248                                ProjectionElem::Index(
249                                    index.node.place().unwrap().as_local().unwrap(),
250                                ),
251                            ],
252                            tcx,
253                        );
254
255                        let ret_ty = generic_args.type_at(0);
256                        let rvalue = match *ret_ty.kind() {
257                            ty::RawPtr(_, Mutability::Not) => {
258                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
259                            }
260                            ty::RawPtr(_, Mutability::Mut) => {
261                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
262                            }
263                            ty::Ref(region, _, Mutability::Not) => {
264                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
265                            }
266                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
267                                region,
268                                BorrowKind::Mut { kind: MutBorrowKind::Default },
269                                updated_place,
270                            ),
271                            _ => bug!("Unknown return type {ret_ty:?}"),
272                        };
273
274                        block.statements.push(Statement::new(
275                            terminator.source_info,
276                            StatementKind::Assign(Box::new((*destination, rvalue))),
277                        ));
278                        terminator.kind = TerminatorKind::Goto { target };
279                    }
280                    sym::transmute | sym::transmute_unchecked => {
281                        let dst_ty = destination.ty(local_decls, tcx).ty;
282                        let Ok([arg]) = take_array(args) else {
283                            span_bug!(
284                                terminator.source_info.span,
285                                "Wrong number of arguments for transmute intrinsic",
286                            );
287                        };
288
289                        // Always emit the cast, even if we transmute to an uninhabited type,
290                        // because that lets CTFE and codegen generate better error messages
291                        // when such a transmute actually ends up reachable.
292                        block.statements.push(Statement::new(
293                            terminator.source_info,
294                            StatementKind::Assign(Box::new((
295                                *destination,
296                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
297                            ))),
298                        ));
299                        if let Some(target) = *target {
300                            terminator.kind = TerminatorKind::Goto { target };
301                        } else {
302                            terminator.kind = TerminatorKind::Unreachable;
303                        }
304                    }
305                    sym::aggregate_raw_ptr => {
306                        let Ok([data, meta]) = take_array(args) else {
307                            span_bug!(
308                                terminator.source_info.span,
309                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
310                            );
311                        };
312                        let target = target.unwrap();
313                        let pointer_ty = generic_args.type_at(0);
314                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
315                            AggregateKind::RawPtr(*pointee_ty, *mutability)
316                        } else {
317                            span_bug!(
318                                terminator.source_info.span,
319                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
320                            );
321                        };
322                        let fields = [data.node, meta.node];
323                        block.statements.push(Statement::new(
324                            terminator.source_info,
325                            StatementKind::Assign(Box::new((
326                                *destination,
327                                Rvalue::Aggregate(Box::new(kind), fields.into()),
328                            ))),
329                        ));
330                        terminator.kind = TerminatorKind::Goto { target };
331                    }
332                    sym::ptr_metadata => {
333                        let Ok([ptr]) = take_array(args) else {
334                            span_bug!(
335                                terminator.source_info.span,
336                                "Wrong number of arguments for ptr_metadata intrinsic",
337                            );
338                        };
339                        let target = target.unwrap();
340                        block.statements.push(Statement::new(
341                            terminator.source_info,
342                            StatementKind::Assign(Box::new((
343                                *destination,
344                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
345                            ))),
346                        ));
347                        terminator.kind = TerminatorKind::Goto { target };
348                    }
349                    _ => {}
350                }
351            }
352        }
353    }
354
355    fn is_required(&self) -> bool {
356        true
357    }
358}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
