rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks | sym::overflow_checks | sym::contract_checks => {
27                        let op = match intrinsic.name {
28                            sym::ub_checks => RuntimeChecks::UbChecks,
29                            sym::contract_checks => RuntimeChecks::ContractChecks,
30                            sym::overflow_checks => RuntimeChecks::OverflowChecks,
31                            _ => unreachable!(),
32                        };
33                        let target = target.unwrap();
34                        block.statements.push(Statement::new(
35                            terminator.source_info,
36                            StatementKind::Assign(Box::new((
37                                *destination,
38                                Rvalue::Use(Operand::RuntimeChecks(op)),
39                            ))),
40                        ));
41                        terminator.kind = TerminatorKind::Goto { target };
42                    }
43                    sym::forget => {
44                        let target = target.unwrap();
45                        block.statements.push(Statement::new(
46                            terminator.source_info,
47                            StatementKind::Assign(Box::new((
48                                *destination,
49                                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
50                                    span: terminator.source_info.span,
51                                    user_ty: None,
52                                    const_: Const::zero_sized(tcx.types.unit),
53                                }))),
54                            ))),
55                        ));
56                        terminator.kind = TerminatorKind::Goto { target };
57                    }
58                    sym::copy_nonoverlapping => {
59                        let target = target.unwrap();
60                        let Ok([src, dst, count]) = take_array(args) else {
61                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
62                        };
63                        block.statements.push(Statement::new(
64                            terminator.source_info,
65                            StatementKind::Intrinsic(Box::new(
66                                NonDivergingIntrinsic::CopyNonOverlapping(
67                                    rustc_middle::mir::CopyNonOverlapping {
68                                        src: src.node,
69                                        dst: dst.node,
70                                        count: count.node,
71                                    },
72                                ),
73                            )),
74                        ));
75                        terminator.kind = TerminatorKind::Goto { target };
76                    }
77                    sym::assume => {
78                        let target = target.unwrap();
79                        let Ok([arg]) = take_array(args) else {
80                            bug!("Wrong arguments for assume intrinsic");
81                        };
82                        block.statements.push(Statement::new(
83                            terminator.source_info,
84                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
85                                arg.node,
86                            ))),
87                        ));
88                        terminator.kind = TerminatorKind::Goto { target };
89                    }
90                    sym::wrapping_add
91                    | sym::wrapping_sub
92                    | sym::wrapping_mul
93                    | sym::three_way_compare
94                    | sym::unchecked_add
95                    | sym::unchecked_sub
96                    | sym::unchecked_mul
97                    | sym::unchecked_div
98                    | sym::unchecked_rem
99                    | sym::unchecked_shl
100                    | sym::unchecked_shr => {
101                        let target = target.unwrap();
102                        let Ok([lhs, rhs]) = take_array(args) else {
103                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
104                        };
105                        let bin_op = match intrinsic.name {
106                            sym::wrapping_add => BinOp::Add,
107                            sym::wrapping_sub => BinOp::Sub,
108                            sym::wrapping_mul => BinOp::Mul,
109                            sym::three_way_compare => BinOp::Cmp,
110                            sym::unchecked_add => BinOp::AddUnchecked,
111                            sym::unchecked_sub => BinOp::SubUnchecked,
112                            sym::unchecked_mul => BinOp::MulUnchecked,
113                            sym::unchecked_div => BinOp::Div,
114                            sym::unchecked_rem => BinOp::Rem,
115                            sym::unchecked_shl => BinOp::ShlUnchecked,
116                            sym::unchecked_shr => BinOp::ShrUnchecked,
117                            _ => bug!("unexpected intrinsic"),
118                        };
119                        block.statements.push(Statement::new(
120                            terminator.source_info,
121                            StatementKind::Assign(Box::new((
122                                *destination,
123                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
124                            ))),
125                        ));
126                        terminator.kind = TerminatorKind::Goto { target };
127                    }
128                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
129                        let target = target.unwrap();
130                        let Ok([lhs, rhs]) = take_array(args) else {
131                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
132                        };
133                        let bin_op = match intrinsic.name {
134                            sym::add_with_overflow => BinOp::AddWithOverflow,
135                            sym::sub_with_overflow => BinOp::SubWithOverflow,
136                            sym::mul_with_overflow => BinOp::MulWithOverflow,
137                            _ => bug!("unexpected intrinsic"),
138                        };
139                        block.statements.push(Statement::new(
140                            terminator.source_info,
141                            StatementKind::Assign(Box::new((
142                                *destination,
143                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
144                            ))),
145                        ));
146                        terminator.kind = TerminatorKind::Goto { target };
147                    }
148                    sym::read_via_copy => {
149                        let Ok([arg]) = take_array(args) else {
150                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
151                        };
152                        let derefed_place = if let Some(place) = arg.node.place()
153                            && let Some(local) = place.as_local()
154                        {
155                            tcx.mk_place_deref(local.into())
156                        } else {
157                            span_bug!(
158                                terminator.source_info.span,
159                                "Only passing a local is supported"
160                            );
161                        };
162                        // Add new statement at the end of the block that does the read, and patch
163                        // up the terminator.
164                        block.statements.push(Statement::new(
165                            terminator.source_info,
166                            StatementKind::Assign(Box::new((
167                                *destination,
168                                Rvalue::Use(Operand::Copy(derefed_place)),
169                            ))),
170                        ));
171                        terminator.kind = match *target {
172                            None => {
173                                // No target means this read something uninhabited,
174                                // so it must be unreachable.
175                                TerminatorKind::Unreachable
176                            }
177                            Some(target) => TerminatorKind::Goto { target },
178                        }
179                    }
180                    // `write_via_move` is already lowered during MIR building.
181                    sym::discriminant_value => {
182                        let target = target.unwrap();
183                        let Ok([arg]) = take_array(args) else {
184                            span_bug!(
185                                terminator.source_info.span,
186                                "Wrong arguments for discriminant_value intrinsic"
187                            );
188                        };
189                        let arg = arg.node.place().unwrap();
190                        let arg = tcx.mk_place_deref(arg);
191                        block.statements.push(Statement::new(
192                            terminator.source_info,
193                            StatementKind::Assign(Box::new((
194                                *destination,
195                                Rvalue::Discriminant(arg),
196                            ))),
197                        ));
198                        terminator.kind = TerminatorKind::Goto { target };
199                    }
200                    sym::offset => {
201                        let target = target.unwrap();
202                        let Ok([ptr, delta]) = take_array(args) else {
203                            span_bug!(
204                                terminator.source_info.span,
205                                "Wrong number of arguments for offset intrinsic",
206                            );
207                        };
208                        block.statements.push(Statement::new(
209                            terminator.source_info,
210                            StatementKind::Assign(Box::new((
211                                *destination,
212                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
213                            ))),
214                        ));
215                        terminator.kind = TerminatorKind::Goto { target };
216                    }
217                    sym::slice_get_unchecked => {
218                        let target = target.unwrap();
219                        let Ok([ptrish, index]) = take_array(args) else {
220                            span_bug!(
221                                terminator.source_info.span,
222                                "Wrong number of arguments for {intrinsic:?}",
223                            );
224                        };
225
226                        let place = ptrish.node.place().unwrap();
227                        assert!(!place.is_indirect());
228                        let updated_place = place.project_deeper(
229                            &[
230                                ProjectionElem::Deref,
231                                ProjectionElem::Index(
232                                    index.node.place().unwrap().as_local().unwrap(),
233                                ),
234                            ],
235                            tcx,
236                        );
237
238                        let ret_ty = generic_args.type_at(0);
239                        let rvalue = match *ret_ty.kind() {
240                            ty::RawPtr(_, Mutability::Not) => {
241                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
242                            }
243                            ty::RawPtr(_, Mutability::Mut) => {
244                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
245                            }
246                            ty::Ref(region, _, Mutability::Not) => {
247                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
248                            }
249                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
250                                region,
251                                BorrowKind::Mut { kind: MutBorrowKind::Default },
252                                updated_place,
253                            ),
254                            _ => bug!("Unknown return type {ret_ty:?}"),
255                        };
256
257                        block.statements.push(Statement::new(
258                            terminator.source_info,
259                            StatementKind::Assign(Box::new((*destination, rvalue))),
260                        ));
261                        terminator.kind = TerminatorKind::Goto { target };
262                    }
263                    sym::transmute | sym::transmute_unchecked => {
264                        let dst_ty = destination.ty(local_decls, tcx).ty;
265                        let Ok([arg]) = take_array(args) else {
266                            span_bug!(
267                                terminator.source_info.span,
268                                "Wrong number of arguments for transmute intrinsic",
269                            );
270                        };
271
272                        // Always emit the cast, even if we transmute to an uninhabited type,
273                        // because that lets CTFE and codegen generate better error messages
274                        // when such a transmute actually ends up reachable.
275                        block.statements.push(Statement::new(
276                            terminator.source_info,
277                            StatementKind::Assign(Box::new((
278                                *destination,
279                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
280                            ))),
281                        ));
282                        if let Some(target) = *target {
283                            terminator.kind = TerminatorKind::Goto { target };
284                        } else {
285                            terminator.kind = TerminatorKind::Unreachable;
286                        }
287                    }
288                    sym::aggregate_raw_ptr => {
289                        let Ok([data, meta]) = take_array(args) else {
290                            span_bug!(
291                                terminator.source_info.span,
292                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
293                            );
294                        };
295                        let target = target.unwrap();
296                        let pointer_ty = generic_args.type_at(0);
297                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
298                            AggregateKind::RawPtr(*pointee_ty, *mutability)
299                        } else {
300                            span_bug!(
301                                terminator.source_info.span,
302                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
303                            );
304                        };
305                        let fields = [data.node, meta.node];
306                        block.statements.push(Statement::new(
307                            terminator.source_info,
308                            StatementKind::Assign(Box::new((
309                                *destination,
310                                Rvalue::Aggregate(Box::new(kind), fields.into()),
311                            ))),
312                        ));
313                        terminator.kind = TerminatorKind::Goto { target };
314                    }
315                    sym::ptr_metadata => {
316                        let Ok([ptr]) = take_array(args) else {
317                            span_bug!(
318                                terminator.source_info.span,
319                                "Wrong number of arguments for ptr_metadata intrinsic",
320                            );
321                        };
322                        let target = target.unwrap();
323                        block.statements.push(Statement::new(
324                            terminator.source_info,
325                            StatementKind::Assign(Box::new((
326                                *destination,
327                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
328                            ))),
329                        ));
330                        terminator.kind = TerminatorKind::Goto { target };
331                    }
332                    _ => {}
333                }
334            }
335        }
336    }
337
338    fn is_required(&self) -> bool {
339        true
340    }
341}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
