rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks => {
27                        let target = target.unwrap();
28                        block.statements.push(Statement::new(
29                            terminator.source_info,
30                            StatementKind::Assign(Box::new((
31                                *destination,
32                                Rvalue::NullaryOp(NullOp::UbChecks, tcx.types.bool),
33                            ))),
34                        ));
35                        terminator.kind = TerminatorKind::Goto { target };
36                    }
37                    sym::forget => {
38                        let target = target.unwrap();
39                        block.statements.push(Statement::new(
40                            terminator.source_info,
41                            StatementKind::Assign(Box::new((
42                                *destination,
43                                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
44                                    span: terminator.source_info.span,
45                                    user_ty: None,
46                                    const_: Const::zero_sized(tcx.types.unit),
47                                }))),
48                            ))),
49                        ));
50                        terminator.kind = TerminatorKind::Goto { target };
51                    }
52                    sym::copy_nonoverlapping => {
53                        let target = target.unwrap();
54                        let Ok([src, dst, count]) = take_array(args) else {
55                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
56                        };
57                        block.statements.push(Statement::new(
58                            terminator.source_info,
59                            StatementKind::Intrinsic(Box::new(
60                                NonDivergingIntrinsic::CopyNonOverlapping(
61                                    rustc_middle::mir::CopyNonOverlapping {
62                                        src: src.node,
63                                        dst: dst.node,
64                                        count: count.node,
65                                    },
66                                ),
67                            )),
68                        ));
69                        terminator.kind = TerminatorKind::Goto { target };
70                    }
71                    sym::assume => {
72                        let target = target.unwrap();
73                        let Ok([arg]) = take_array(args) else {
74                            bug!("Wrong arguments for assume intrinsic");
75                        };
76                        block.statements.push(Statement::new(
77                            terminator.source_info,
78                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
79                                arg.node,
80                            ))),
81                        ));
82                        terminator.kind = TerminatorKind::Goto { target };
83                    }
84                    sym::wrapping_add
85                    | sym::wrapping_sub
86                    | sym::wrapping_mul
87                    | sym::three_way_compare
88                    | sym::unchecked_add
89                    | sym::unchecked_sub
90                    | sym::unchecked_mul
91                    | sym::unchecked_div
92                    | sym::unchecked_rem
93                    | sym::unchecked_shl
94                    | sym::unchecked_shr => {
95                        let target = target.unwrap();
96                        let Ok([lhs, rhs]) = take_array(args) else {
97                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
98                        };
99                        let bin_op = match intrinsic.name {
100                            sym::wrapping_add => BinOp::Add,
101                            sym::wrapping_sub => BinOp::Sub,
102                            sym::wrapping_mul => BinOp::Mul,
103                            sym::three_way_compare => BinOp::Cmp,
104                            sym::unchecked_add => BinOp::AddUnchecked,
105                            sym::unchecked_sub => BinOp::SubUnchecked,
106                            sym::unchecked_mul => BinOp::MulUnchecked,
107                            sym::unchecked_div => BinOp::Div,
108                            sym::unchecked_rem => BinOp::Rem,
109                            sym::unchecked_shl => BinOp::ShlUnchecked,
110                            sym::unchecked_shr => BinOp::ShrUnchecked,
111                            _ => bug!("unexpected intrinsic"),
112                        };
113                        block.statements.push(Statement::new(
114                            terminator.source_info,
115                            StatementKind::Assign(Box::new((
116                                *destination,
117                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
118                            ))),
119                        ));
120                        terminator.kind = TerminatorKind::Goto { target };
121                    }
122                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
123                        let target = target.unwrap();
124                        let Ok([lhs, rhs]) = take_array(args) else {
125                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
126                        };
127                        let bin_op = match intrinsic.name {
128                            sym::add_with_overflow => BinOp::AddWithOverflow,
129                            sym::sub_with_overflow => BinOp::SubWithOverflow,
130                            sym::mul_with_overflow => BinOp::MulWithOverflow,
131                            _ => bug!("unexpected intrinsic"),
132                        };
133                        block.statements.push(Statement::new(
134                            terminator.source_info,
135                            StatementKind::Assign(Box::new((
136                                *destination,
137                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
138                            ))),
139                        ));
140                        terminator.kind = TerminatorKind::Goto { target };
141                    }
142                    sym::size_of | sym::align_of => {
143                        let target = target.unwrap();
144                        let tp_ty = generic_args.type_at(0);
145                        let null_op = match intrinsic.name {
146                            sym::size_of => NullOp::SizeOf,
147                            sym::align_of => NullOp::AlignOf,
148                            _ => bug!("unexpected intrinsic"),
149                        };
150                        block.statements.push(Statement::new(
151                            terminator.source_info,
152                            StatementKind::Assign(Box::new((
153                                *destination,
154                                Rvalue::NullaryOp(null_op, tp_ty),
155                            ))),
156                        ));
157                        terminator.kind = TerminatorKind::Goto { target };
158                    }
159                    sym::read_via_copy => {
160                        let Ok([arg]) = take_array(args) else {
161                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
162                        };
163                        let derefed_place = if let Some(place) = arg.node.place()
164                            && let Some(local) = place.as_local()
165                        {
166                            tcx.mk_place_deref(local.into())
167                        } else {
168                            span_bug!(
169                                terminator.source_info.span,
170                                "Only passing a local is supported"
171                            );
172                        };
173                        // Add new statement at the end of the block that does the read, and patch
174                        // up the terminator.
175                        block.statements.push(Statement::new(
176                            terminator.source_info,
177                            StatementKind::Assign(Box::new((
178                                *destination,
179                                Rvalue::Use(Operand::Copy(derefed_place)),
180                            ))),
181                        ));
182                        terminator.kind = match *target {
183                            None => {
184                                // No target means this read something uninhabited,
185                                // so it must be unreachable.
186                                TerminatorKind::Unreachable
187                            }
188                            Some(target) => TerminatorKind::Goto { target },
189                        }
190                    }
191                    sym::write_via_move => {
192                        let target = target.unwrap();
193                        let Ok([ptr, val]) = take_array(args) else {
194                            span_bug!(
195                                terminator.source_info.span,
196                                "Wrong number of arguments for write_via_move intrinsic",
197                            );
198                        };
199                        let derefed_place = if let Some(place) = ptr.node.place()
200                            && let Some(local) = place.as_local()
201                        {
202                            tcx.mk_place_deref(local.into())
203                        } else {
204                            span_bug!(
205                                terminator.source_info.span,
206                                "Only passing a local is supported"
207                            );
208                        };
209                        block.statements.push(Statement::new(
210                            terminator.source_info,
211                            StatementKind::Assign(Box::new((derefed_place, Rvalue::Use(val.node)))),
212                        ));
213                        terminator.kind = TerminatorKind::Goto { target };
214                    }
215                    sym::discriminant_value => {
216                        let target = target.unwrap();
217                        let Ok([arg]) = take_array(args) else {
218                            span_bug!(
219                                terminator.source_info.span,
220                                "Wrong arguments for discriminant_value intrinsic"
221                            );
222                        };
223                        let arg = arg.node.place().unwrap();
224                        let arg = tcx.mk_place_deref(arg);
225                        block.statements.push(Statement::new(
226                            terminator.source_info,
227                            StatementKind::Assign(Box::new((
228                                *destination,
229                                Rvalue::Discriminant(arg),
230                            ))),
231                        ));
232                        terminator.kind = TerminatorKind::Goto { target };
233                    }
234                    sym::offset => {
235                        let target = target.unwrap();
236                        let Ok([ptr, delta]) = take_array(args) else {
237                            span_bug!(
238                                terminator.source_info.span,
239                                "Wrong number of arguments for offset intrinsic",
240                            );
241                        };
242                        block.statements.push(Statement::new(
243                            terminator.source_info,
244                            StatementKind::Assign(Box::new((
245                                *destination,
246                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
247                            ))),
248                        ));
249                        terminator.kind = TerminatorKind::Goto { target };
250                    }
251                    sym::slice_get_unchecked => {
252                        let target = target.unwrap();
253                        let Ok([ptrish, index]) = take_array(args) else {
254                            span_bug!(
255                                terminator.source_info.span,
256                                "Wrong number of arguments for {intrinsic:?}",
257                            );
258                        };
259
260                        let place = ptrish.node.place().unwrap();
261                        assert!(!place.is_indirect());
262                        let updated_place = place.project_deeper(
263                            &[
264                                ProjectionElem::Deref,
265                                ProjectionElem::Index(
266                                    index.node.place().unwrap().as_local().unwrap(),
267                                ),
268                            ],
269                            tcx,
270                        );
271
272                        let ret_ty = generic_args.type_at(0);
273                        let rvalue = match *ret_ty.kind() {
274                            ty::RawPtr(_, Mutability::Not) => {
275                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
276                            }
277                            ty::RawPtr(_, Mutability::Mut) => {
278                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
279                            }
280                            ty::Ref(region, _, Mutability::Not) => {
281                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
282                            }
283                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
284                                region,
285                                BorrowKind::Mut { kind: MutBorrowKind::Default },
286                                updated_place,
287                            ),
288                            _ => bug!("Unknown return type {ret_ty:?}"),
289                        };
290
291                        block.statements.push(Statement::new(
292                            terminator.source_info,
293                            StatementKind::Assign(Box::new((*destination, rvalue))),
294                        ));
295                        terminator.kind = TerminatorKind::Goto { target };
296                    }
297                    sym::transmute | sym::transmute_unchecked => {
298                        let dst_ty = destination.ty(local_decls, tcx).ty;
299                        let Ok([arg]) = take_array(args) else {
300                            span_bug!(
301                                terminator.source_info.span,
302                                "Wrong number of arguments for transmute intrinsic",
303                            );
304                        };
305
306                        // Always emit the cast, even if we transmute to an uninhabited type,
307                        // because that lets CTFE and codegen generate better error messages
308                        // when such a transmute actually ends up reachable.
309                        block.statements.push(Statement::new(
310                            terminator.source_info,
311                            StatementKind::Assign(Box::new((
312                                *destination,
313                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
314                            ))),
315                        ));
316                        if let Some(target) = *target {
317                            terminator.kind = TerminatorKind::Goto { target };
318                        } else {
319                            terminator.kind = TerminatorKind::Unreachable;
320                        }
321                    }
322                    sym::aggregate_raw_ptr => {
323                        let Ok([data, meta]) = take_array(args) else {
324                            span_bug!(
325                                terminator.source_info.span,
326                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
327                            );
328                        };
329                        let target = target.unwrap();
330                        let pointer_ty = generic_args.type_at(0);
331                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
332                            AggregateKind::RawPtr(*pointee_ty, *mutability)
333                        } else {
334                            span_bug!(
335                                terminator.source_info.span,
336                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
337                            );
338                        };
339                        let fields = [data.node, meta.node];
340                        block.statements.push(Statement::new(
341                            terminator.source_info,
342                            StatementKind::Assign(Box::new((
343                                *destination,
344                                Rvalue::Aggregate(Box::new(kind), fields.into()),
345                            ))),
346                        ));
347                        terminator.kind = TerminatorKind::Goto { target };
348                    }
349                    sym::ptr_metadata => {
350                        let Ok([ptr]) = take_array(args) else {
351                            span_bug!(
352                                terminator.source_info.span,
353                                "Wrong number of arguments for ptr_metadata intrinsic",
354                            );
355                        };
356                        let target = target.unwrap();
357                        block.statements.push(Statement::new(
358                            terminator.source_info,
359                            StatementKind::Assign(Box::new((
360                                *destination,
361                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
362                            ))),
363                        ));
364                        terminator.kind = TerminatorKind::Goto { target };
365                    }
366                    _ => {}
367                }
368            }
369        }
370    }
371
372    fn is_required(&self) -> bool {
373        true
374    }
375}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
