rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                let generic_args = generic_args.no_bound_vars().unwrap();
23                match intrinsic.name {
24                    sym::unreachable => {
25                        terminator.kind = TerminatorKind::Unreachable;
26                    }
27                    sym::ub_checks | sym::overflow_checks | sym::contract_checks => {
28                        let op = match intrinsic.name {
29                            sym::ub_checks => RuntimeChecks::UbChecks,
30                            sym::contract_checks => RuntimeChecks::ContractChecks,
31                            sym::overflow_checks => RuntimeChecks::OverflowChecks,
32                            _ => unreachable!(),
33                        };
34                        let target = target.unwrap();
35                        block.statements.push(Statement::new(
36                            terminator.source_info,
37                            StatementKind::Assign(Box::new((
38                                *destination,
39                                Rvalue::Use(Operand::RuntimeChecks(op), WithRetag::Yes),
40                            ))),
41                        ));
42                        terminator.kind = TerminatorKind::Goto { target };
43                    }
44                    sym::forget => {
45                        let target = target.unwrap();
46                        block.statements.push(Statement::new(
47                            terminator.source_info,
48                            StatementKind::Assign(Box::new((
49                                *destination,
50                                Rvalue::Use(
51                                    Operand::Constant(Box::new(ConstOperand {
52                                        span: terminator.source_info.span,
53                                        user_ty: None,
54                                        const_: Const::zero_sized(tcx.types.unit),
55                                    })),
56                                    WithRetag::Yes,
57                                ),
58                            ))),
59                        ));
60                        terminator.kind = TerminatorKind::Goto { target };
61                    }
62                    sym::copy_nonoverlapping => {
63                        let target = target.unwrap();
64                        let Ok([src, dst, count]) = take_array(args) else {
65                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
66                        };
67                        block.statements.push(Statement::new(
68                            terminator.source_info,
69                            StatementKind::Intrinsic(Box::new(
70                                NonDivergingIntrinsic::CopyNonOverlapping(
71                                    rustc_middle::mir::CopyNonOverlapping {
72                                        src: src.node,
73                                        dst: dst.node,
74                                        count: count.node,
75                                    },
76                                ),
77                            )),
78                        ));
79                        terminator.kind = TerminatorKind::Goto { target };
80                    }
81                    sym::assume => {
82                        let target = target.unwrap();
83                        let Ok([arg]) = take_array(args) else {
84                            bug!("Wrong arguments for assume intrinsic");
85                        };
86                        block.statements.push(Statement::new(
87                            terminator.source_info,
88                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
89                                arg.node,
90                            ))),
91                        ));
92                        terminator.kind = TerminatorKind::Goto { target };
93                    }
94                    sym::wrapping_add
95                    | sym::wrapping_sub
96                    | sym::wrapping_mul
97                    | sym::three_way_compare
98                    | sym::unchecked_add
99                    | sym::unchecked_sub
100                    | sym::unchecked_mul
101                    | sym::unchecked_div
102                    | sym::unchecked_rem
103                    | sym::unchecked_shl
104                    | sym::unchecked_shr => {
105                        let target = target.unwrap();
106                        let Ok([lhs, rhs]) = take_array(args) else {
107                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
108                        };
109                        let bin_op = match intrinsic.name {
110                            sym::wrapping_add => BinOp::Add,
111                            sym::wrapping_sub => BinOp::Sub,
112                            sym::wrapping_mul => BinOp::Mul,
113                            sym::three_way_compare => BinOp::Cmp,
114                            sym::unchecked_add => BinOp::AddUnchecked,
115                            sym::unchecked_sub => BinOp::SubUnchecked,
116                            sym::unchecked_mul => BinOp::MulUnchecked,
117                            sym::unchecked_div => BinOp::Div,
118                            sym::unchecked_rem => BinOp::Rem,
119                            sym::unchecked_shl => BinOp::ShlUnchecked,
120                            sym::unchecked_shr => BinOp::ShrUnchecked,
121                            _ => bug!("unexpected intrinsic"),
122                        };
123                        block.statements.push(Statement::new(
124                            terminator.source_info,
125                            StatementKind::Assign(Box::new((
126                                *destination,
127                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
128                            ))),
129                        ));
130                        terminator.kind = TerminatorKind::Goto { target };
131                    }
132                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
133                        let target = target.unwrap();
134                        let Ok([lhs, rhs]) = take_array(args) else {
135                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
136                        };
137                        let bin_op = match intrinsic.name {
138                            sym::add_with_overflow => BinOp::AddWithOverflow,
139                            sym::sub_with_overflow => BinOp::SubWithOverflow,
140                            sym::mul_with_overflow => BinOp::MulWithOverflow,
141                            _ => bug!("unexpected intrinsic"),
142                        };
143                        block.statements.push(Statement::new(
144                            terminator.source_info,
145                            StatementKind::Assign(Box::new((
146                                *destination,
147                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
148                            ))),
149                        ));
150                        terminator.kind = TerminatorKind::Goto { target };
151                    }
152                    sym::read_via_copy => {
153                        let Ok([arg]) = take_array(args) else {
154                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
155                        };
156                        let derefed_place = if let Some(place) = arg.node.place()
157                            && let Some(local) = place.as_local()
158                        {
159                            tcx.mk_place_deref(local.into())
160                        } else {
161                            span_bug!(
162                                terminator.source_info.span,
163                                "Only passing a local is supported"
164                            );
165                        };
166                        // Add new statement at the end of the block that does the read, and patch
167                        // up the terminator.
168                        block.statements.push(Statement::new(
169                            terminator.source_info,
170                            StatementKind::Assign(Box::new((
171                                *destination,
172                                Rvalue::Use(Operand::Copy(derefed_place), WithRetag::Yes),
173                            ))),
174                        ));
175                        terminator.kind = match *target {
176                            None => {
177                                // No target means this read something uninhabited,
178                                // so it must be unreachable.
179                                TerminatorKind::Unreachable
180                            }
181                            Some(target) => TerminatorKind::Goto { target },
182                        }
183                    }
184                    // `write_via_move` is already lowered during MIR building.
185                    sym::discriminant_value => {
186                        let target = target.unwrap();
187                        let Ok([arg]) = take_array(args) else {
188                            span_bug!(
189                                terminator.source_info.span,
190                                "Wrong arguments for discriminant_value intrinsic"
191                            );
192                        };
193                        let arg = arg.node.place().unwrap();
194                        let arg = tcx.mk_place_deref(arg);
195                        block.statements.push(Statement::new(
196                            terminator.source_info,
197                            StatementKind::Assign(Box::new((
198                                *destination,
199                                Rvalue::Discriminant(arg),
200                            ))),
201                        ));
202                        terminator.kind = TerminatorKind::Goto { target };
203                    }
204                    sym::offset => {
205                        let target = target.unwrap();
206                        let Ok([ptr, delta]) = take_array(args) else {
207                            span_bug!(
208                                terminator.source_info.span,
209                                "Wrong number of arguments for offset intrinsic",
210                            );
211                        };
212                        block.statements.push(Statement::new(
213                            terminator.source_info,
214                            StatementKind::Assign(Box::new((
215                                *destination,
216                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
217                            ))),
218                        ));
219                        terminator.kind = TerminatorKind::Goto { target };
220                    }
221                    sym::slice_get_unchecked => {
222                        let target = target.unwrap();
223                        let Ok([ptrish, index]) = take_array(args) else {
224                            span_bug!(
225                                terminator.source_info.span,
226                                "Wrong number of arguments for {intrinsic:?}",
227                            );
228                        };
229
230                        let place = ptrish.node.place().unwrap();
231                        assert!(!place.is_indirect());
232                        let updated_place = place.project_deeper(
233                            &[
234                                ProjectionElem::Deref,
235                                ProjectionElem::Index(
236                                    index.node.place().unwrap().as_local().unwrap(),
237                                ),
238                            ],
239                            tcx,
240                        );
241
242                        let ret_ty = generic_args.type_at(0);
243                        let rvalue = match *ret_ty.kind() {
244                            ty::RawPtr(_, Mutability::Not) => {
245                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
246                            }
247                            ty::RawPtr(_, Mutability::Mut) => {
248                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
249                            }
250                            ty::Ref(region, _, Mutability::Not) => {
251                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
252                            }
253                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
254                                region,
255                                BorrowKind::Mut { kind: MutBorrowKind::Default },
256                                updated_place,
257                            ),
258                            _ => bug!("Unknown return type {ret_ty:?}"),
259                        };
260
261                        block.statements.push(Statement::new(
262                            terminator.source_info,
263                            StatementKind::Assign(Box::new((*destination, rvalue))),
264                        ));
265                        terminator.kind = TerminatorKind::Goto { target };
266                    }
267                    sym::transmute | sym::transmute_unchecked => {
268                        let dst_ty = destination.ty(local_decls, tcx).ty;
269                        let Ok([arg]) = take_array(args) else {
270                            span_bug!(
271                                terminator.source_info.span,
272                                "Wrong number of arguments for transmute intrinsic",
273                            );
274                        };
275
276                        // Always emit the cast, even if we transmute to an uninhabited type,
277                        // because that lets CTFE and codegen generate better error messages
278                        // when such a transmute actually ends up reachable.
279                        block.statements.push(Statement::new(
280                            terminator.source_info,
281                            StatementKind::Assign(Box::new((
282                                *destination,
283                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
284                            ))),
285                        ));
286                        if let Some(target) = *target {
287                            terminator.kind = TerminatorKind::Goto { target };
288                        } else {
289                            terminator.kind = TerminatorKind::Unreachable;
290                        }
291                    }
292                    sym::aggregate_raw_ptr => {
293                        let Ok([data, meta]) = take_array(args) else {
294                            span_bug!(
295                                terminator.source_info.span,
296                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
297                            );
298                        };
299                        let target = target.unwrap();
300                        let pointer_ty = generic_args.type_at(0);
301                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
302                            AggregateKind::RawPtr(*pointee_ty, *mutability)
303                        } else {
304                            span_bug!(
305                                terminator.source_info.span,
306                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
307                            );
308                        };
309                        let fields = [data.node, meta.node];
310                        block.statements.push(Statement::new(
311                            terminator.source_info,
312                            StatementKind::Assign(Box::new((
313                                *destination,
314                                Rvalue::Aggregate(Box::new(kind), fields.into()),
315                            ))),
316                        ));
317                        terminator.kind = TerminatorKind::Goto { target };
318                    }
319                    sym::ptr_metadata => {
320                        let Ok([ptr]) = take_array(args) else {
321                            span_bug!(
322                                terminator.source_info.span,
323                                "Wrong number of arguments for ptr_metadata intrinsic",
324                            );
325                        };
326                        let target = target.unwrap();
327                        block.statements.push(Statement::new(
328                            terminator.source_info,
329                            StatementKind::Assign(Box::new((
330                                *destination,
331                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
332                            ))),
333                        ));
334                        terminator.kind = TerminatorKind::Goto { target };
335                    }
336                    _ => {}
337                }
338            }
339        }
340    }
341
342    fn is_required(&self) -> bool {
343        true
344    }
345}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
