rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks | sym::overflow_checks | sym::contract_checks => {
27                        let op = match intrinsic.name {
28                            sym::ub_checks => RuntimeChecks::UbChecks,
29                            sym::contract_checks => RuntimeChecks::ContractChecks,
30                            sym::overflow_checks => RuntimeChecks::OverflowChecks,
31                            _ => unreachable!(),
32                        };
33                        let target = target.unwrap();
34                        block.statements.push(Statement::new(
35                            terminator.source_info,
36                            StatementKind::Assign(Box::new((
37                                *destination,
38                                Rvalue::NullaryOp(NullOp::RuntimeChecks(op), tcx.types.bool),
39                            ))),
40                        ));
41                        terminator.kind = TerminatorKind::Goto { target };
42                    }
43                    sym::forget => {
44                        let target = target.unwrap();
45                        block.statements.push(Statement::new(
46                            terminator.source_info,
47                            StatementKind::Assign(Box::new((
48                                *destination,
49                                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
50                                    span: terminator.source_info.span,
51                                    user_ty: None,
52                                    const_: Const::zero_sized(tcx.types.unit),
53                                }))),
54                            ))),
55                        ));
56                        terminator.kind = TerminatorKind::Goto { target };
57                    }
58                    sym::copy_nonoverlapping => {
59                        let target = target.unwrap();
60                        let Ok([src, dst, count]) = take_array(args) else {
61                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
62                        };
63                        block.statements.push(Statement::new(
64                            terminator.source_info,
65                            StatementKind::Intrinsic(Box::new(
66                                NonDivergingIntrinsic::CopyNonOverlapping(
67                                    rustc_middle::mir::CopyNonOverlapping {
68                                        src: src.node,
69                                        dst: dst.node,
70                                        count: count.node,
71                                    },
72                                ),
73                            )),
74                        ));
75                        terminator.kind = TerminatorKind::Goto { target };
76                    }
77                    sym::assume => {
78                        let target = target.unwrap();
79                        let Ok([arg]) = take_array(args) else {
80                            bug!("Wrong arguments for assume intrinsic");
81                        };
82                        block.statements.push(Statement::new(
83                            terminator.source_info,
84                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
85                                arg.node,
86                            ))),
87                        ));
88                        terminator.kind = TerminatorKind::Goto { target };
89                    }
90                    sym::wrapping_add
91                    | sym::wrapping_sub
92                    | sym::wrapping_mul
93                    | sym::three_way_compare
94                    | sym::unchecked_add
95                    | sym::unchecked_sub
96                    | sym::unchecked_mul
97                    | sym::unchecked_div
98                    | sym::unchecked_rem
99                    | sym::unchecked_shl
100                    | sym::unchecked_shr => {
101                        let target = target.unwrap();
102                        let Ok([lhs, rhs]) = take_array(args) else {
103                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
104                        };
105                        let bin_op = match intrinsic.name {
106                            sym::wrapping_add => BinOp::Add,
107                            sym::wrapping_sub => BinOp::Sub,
108                            sym::wrapping_mul => BinOp::Mul,
109                            sym::three_way_compare => BinOp::Cmp,
110                            sym::unchecked_add => BinOp::AddUnchecked,
111                            sym::unchecked_sub => BinOp::SubUnchecked,
112                            sym::unchecked_mul => BinOp::MulUnchecked,
113                            sym::unchecked_div => BinOp::Div,
114                            sym::unchecked_rem => BinOp::Rem,
115                            sym::unchecked_shl => BinOp::ShlUnchecked,
116                            sym::unchecked_shr => BinOp::ShrUnchecked,
117                            _ => bug!("unexpected intrinsic"),
118                        };
119                        block.statements.push(Statement::new(
120                            terminator.source_info,
121                            StatementKind::Assign(Box::new((
122                                *destination,
123                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
124                            ))),
125                        ));
126                        terminator.kind = TerminatorKind::Goto { target };
127                    }
128                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
129                        let target = target.unwrap();
130                        let Ok([lhs, rhs]) = take_array(args) else {
131                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
132                        };
133                        let bin_op = match intrinsic.name {
134                            sym::add_with_overflow => BinOp::AddWithOverflow,
135                            sym::sub_with_overflow => BinOp::SubWithOverflow,
136                            sym::mul_with_overflow => BinOp::MulWithOverflow,
137                            _ => bug!("unexpected intrinsic"),
138                        };
139                        block.statements.push(Statement::new(
140                            terminator.source_info,
141                            StatementKind::Assign(Box::new((
142                                *destination,
143                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
144                            ))),
145                        ));
146                        terminator.kind = TerminatorKind::Goto { target };
147                    }
148                    sym::read_via_copy => {
149                        let Ok([arg]) = take_array(args) else {
150                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
151                        };
152                        let derefed_place = if let Some(place) = arg.node.place()
153                            && let Some(local) = place.as_local()
154                        {
155                            tcx.mk_place_deref(local.into())
156                        } else {
157                            span_bug!(
158                                terminator.source_info.span,
159                                "Only passing a local is supported"
160                            );
161                        };
162                        // Add new statement at the end of the block that does the read, and patch
163                        // up the terminator.
164                        block.statements.push(Statement::new(
165                            terminator.source_info,
166                            StatementKind::Assign(Box::new((
167                                *destination,
168                                Rvalue::Use(Operand::Copy(derefed_place)),
169                            ))),
170                        ));
171                        terminator.kind = match *target {
172                            None => {
173                                // No target means this read something uninhabited,
174                                // so it must be unreachable.
175                                TerminatorKind::Unreachable
176                            }
177                            Some(target) => TerminatorKind::Goto { target },
178                        }
179                    }
180                    sym::write_via_move => {
181                        let target = target.unwrap();
182                        let Ok([ptr, val]) = take_array(args) else {
183                            span_bug!(
184                                terminator.source_info.span,
185                                "Wrong number of arguments for write_via_move intrinsic",
186                            );
187                        };
188                        let derefed_place = if let Some(place) = ptr.node.place()
189                            && let Some(local) = place.as_local()
190                        {
191                            tcx.mk_place_deref(local.into())
192                        } else {
193                            span_bug!(
194                                terminator.source_info.span,
195                                "Only passing a local is supported"
196                            );
197                        };
198                        block.statements.push(Statement::new(
199                            terminator.source_info,
200                            StatementKind::Assign(Box::new((derefed_place, Rvalue::Use(val.node)))),
201                        ));
202                        terminator.kind = TerminatorKind::Goto { target };
203                    }
204                    sym::discriminant_value => {
205                        let target = target.unwrap();
206                        let Ok([arg]) = take_array(args) else {
207                            span_bug!(
208                                terminator.source_info.span,
209                                "Wrong arguments for discriminant_value intrinsic"
210                            );
211                        };
212                        let arg = arg.node.place().unwrap();
213                        let arg = tcx.mk_place_deref(arg);
214                        block.statements.push(Statement::new(
215                            terminator.source_info,
216                            StatementKind::Assign(Box::new((
217                                *destination,
218                                Rvalue::Discriminant(arg),
219                            ))),
220                        ));
221                        terminator.kind = TerminatorKind::Goto { target };
222                    }
223                    sym::offset => {
224                        let target = target.unwrap();
225                        let Ok([ptr, delta]) = take_array(args) else {
226                            span_bug!(
227                                terminator.source_info.span,
228                                "Wrong number of arguments for offset intrinsic",
229                            );
230                        };
231                        block.statements.push(Statement::new(
232                            terminator.source_info,
233                            StatementKind::Assign(Box::new((
234                                *destination,
235                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
236                            ))),
237                        ));
238                        terminator.kind = TerminatorKind::Goto { target };
239                    }
240                    sym::slice_get_unchecked => {
241                        let target = target.unwrap();
242                        let Ok([ptrish, index]) = take_array(args) else {
243                            span_bug!(
244                                terminator.source_info.span,
245                                "Wrong number of arguments for {intrinsic:?}",
246                            );
247                        };
248
249                        let place = ptrish.node.place().unwrap();
250                        assert!(!place.is_indirect());
251                        let updated_place = place.project_deeper(
252                            &[
253                                ProjectionElem::Deref,
254                                ProjectionElem::Index(
255                                    index.node.place().unwrap().as_local().unwrap(),
256                                ),
257                            ],
258                            tcx,
259                        );
260
261                        let ret_ty = generic_args.type_at(0);
262                        let rvalue = match *ret_ty.kind() {
263                            ty::RawPtr(_, Mutability::Not) => {
264                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
265                            }
266                            ty::RawPtr(_, Mutability::Mut) => {
267                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
268                            }
269                            ty::Ref(region, _, Mutability::Not) => {
270                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
271                            }
272                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
273                                region,
274                                BorrowKind::Mut { kind: MutBorrowKind::Default },
275                                updated_place,
276                            ),
277                            _ => bug!("Unknown return type {ret_ty:?}"),
278                        };
279
280                        block.statements.push(Statement::new(
281                            terminator.source_info,
282                            StatementKind::Assign(Box::new((*destination, rvalue))),
283                        ));
284                        terminator.kind = TerminatorKind::Goto { target };
285                    }
286                    sym::transmute | sym::transmute_unchecked => {
287                        let dst_ty = destination.ty(local_decls, tcx).ty;
288                        let Ok([arg]) = take_array(args) else {
289                            span_bug!(
290                                terminator.source_info.span,
291                                "Wrong number of arguments for transmute intrinsic",
292                            );
293                        };
294
295                        // Always emit the cast, even if we transmute to an uninhabited type,
296                        // because that lets CTFE and codegen generate better error messages
297                        // when such a transmute actually ends up reachable.
298                        block.statements.push(Statement::new(
299                            terminator.source_info,
300                            StatementKind::Assign(Box::new((
301                                *destination,
302                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
303                            ))),
304                        ));
305                        if let Some(target) = *target {
306                            terminator.kind = TerminatorKind::Goto { target };
307                        } else {
308                            terminator.kind = TerminatorKind::Unreachable;
309                        }
310                    }
311                    sym::aggregate_raw_ptr => {
312                        let Ok([data, meta]) = take_array(args) else {
313                            span_bug!(
314                                terminator.source_info.span,
315                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
316                            );
317                        };
318                        let target = target.unwrap();
319                        let pointer_ty = generic_args.type_at(0);
320                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
321                            AggregateKind::RawPtr(*pointee_ty, *mutability)
322                        } else {
323                            span_bug!(
324                                terminator.source_info.span,
325                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
326                            );
327                        };
328                        let fields = [data.node, meta.node];
329                        block.statements.push(Statement::new(
330                            terminator.source_info,
331                            StatementKind::Assign(Box::new((
332                                *destination,
333                                Rvalue::Aggregate(Box::new(kind), fields.into()),
334                            ))),
335                        ));
336                        terminator.kind = TerminatorKind::Goto { target };
337                    }
338                    sym::ptr_metadata => {
339                        let Ok([ptr]) = take_array(args) else {
340                            span_bug!(
341                                terminator.source_info.span,
342                                "Wrong number of arguments for ptr_metadata intrinsic",
343                            );
344                        };
345                        let target = target.unwrap();
346                        block.statements.push(Statement::new(
347                            terminator.source_info,
348                            StatementKind::Assign(Box::new((
349                                *destination,
350                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
351                            ))),
352                        ));
353                        terminator.kind = TerminatorKind::Goto { target };
354                    }
355                    _ => {}
356                }
357            }
358        }
359    }
360
361    fn is_required(&self) -> bool {
362        true
363    }
364}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
