rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_span::{bug, span_bug, sym};
6
7use crate::{PassPolicy, take_array};
8
9pub(super) struct LowerIntrinsics;
10
11impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
12    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
13        let local_decls = &body.local_decls;
14        for block in body.basic_blocks.as_mut() {
15            let terminator = block.terminator.as_mut().unwrap();
16            if let TerminatorKind::Call { func, args, destination, target, .. } =
17                &mut terminator.kind
18                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
19                && let Some(intrinsic) = tcx.intrinsic(def_id)
20            {
21                let generic_args = generic_args.no_bound_vars().unwrap();
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks | sym::overflow_checks | sym::contract_checks => {
27                        let op = match intrinsic.name {
28                            sym::ub_checks => RuntimeChecks::UbChecks,
29                            sym::contract_checks => RuntimeChecks::ContractChecks,
30                            sym::overflow_checks => RuntimeChecks::OverflowChecks,
31                            _ => unreachable!(),
32                        };
33                        let target = target.unwrap();
34                        block.statements.push(Statement::new(
35                            terminator.source_info,
36                            StatementKind::Assign(Box::new((
37                                *destination,
38                                Rvalue::Use(Operand::RuntimeChecks(op), WithRetag::Yes),
39                            ))),
40                        ));
41                        terminator.kind = TerminatorKind::goto(target);
42                    }
43                    sym::forget => {
44                        let target = target.unwrap();
45                        block.statements.push(Statement::new(
46                            terminator.source_info,
47                            StatementKind::Assign(Box::new((
48                                *destination,
49                                Rvalue::Use(
50                                    Operand::zero_sized_constant(
51                                        tcx.types.unit,
52                                        terminator.source_info.span,
53                                    ),
54                                    WithRetag::Yes,
55                                ),
56                            ))),
57                        ));
58                        terminator.kind = TerminatorKind::goto(target);
59                    }
60                    sym::copy_nonoverlapping => {
61                        let target = target.unwrap();
62                        let Ok([src, dst, count]) = take_array(args) else {
63                            bug!("Wrong arguments for copy_nonoverlapping intrinsic");
64                        };
65                        block.statements.push(Statement::new(
66                            terminator.source_info,
67                            StatementKind::Intrinsic(Box::new(
68                                NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
69                                    src: src.node,
70                                    dst: dst.node,
71                                    count: count.node,
72                                }),
73                            )),
74                        ));
75                        block.statements.push(Statement::new(
76                            terminator.source_info,
77                            StatementKind::Assign(Box::new((
78                                *destination,
79                                Rvalue::Use(
80                                    Operand::zero_sized_constant(
81                                        tcx.types.unit,
82                                        terminator.source_info.span,
83                                    ),
84                                    WithRetag::Yes,
85                                ),
86                            ))),
87                        ));
88                        terminator.kind = TerminatorKind::goto(target);
89                    }
90                    sym::assume => {
91                        let target = target.unwrap();
92                        let Ok([arg]) = take_array(args) else {
93                            bug!("Wrong arguments for assume intrinsic");
94                        };
95                        block.statements.push(Statement::new(
96                            terminator.source_info,
97                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
98                                arg.node,
99                            ))),
100                        ));
101                        block.statements.push(Statement::new(
102                            terminator.source_info,
103                            StatementKind::Assign(Box::new((
104                                *destination,
105                                Rvalue::Use(
106                                    Operand::zero_sized_constant(
107                                        tcx.types.unit,
108                                        terminator.source_info.span,
109                                    ),
110                                    WithRetag::Yes,
111                                ),
112                            ))),
113                        ));
114                        terminator.kind = TerminatorKind::goto(target);
115                    }
116                    sym::wrapping_add
117                    | sym::wrapping_sub
118                    | sym::wrapping_mul
119                    | sym::three_way_compare
120                    | sym::unchecked_add
121                    | sym::unchecked_sub
122                    | sym::unchecked_mul
123                    | sym::unchecked_div
124                    | sym::unchecked_rem
125                    | sym::unchecked_shl
126                    | sym::unchecked_shr => {
127                        let target = target.unwrap();
128                        let Ok([lhs, rhs]) = take_array(args) else {
129                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
130                        };
131                        let bin_op = match intrinsic.name {
132                            sym::wrapping_add => BinOp::Add,
133                            sym::wrapping_sub => BinOp::Sub,
134                            sym::wrapping_mul => BinOp::Mul,
135                            sym::three_way_compare => BinOp::Cmp,
136                            sym::unchecked_add => BinOp::AddUnchecked,
137                            sym::unchecked_sub => BinOp::SubUnchecked,
138                            sym::unchecked_mul => BinOp::MulUnchecked,
139                            sym::unchecked_div => BinOp::Div,
140                            sym::unchecked_rem => BinOp::Rem,
141                            sym::unchecked_shl => BinOp::ShlUnchecked,
142                            sym::unchecked_shr => BinOp::ShrUnchecked,
143                            _ => bug!("unexpected intrinsic"),
144                        };
145                        block.statements.push(Statement::new(
146                            terminator.source_info,
147                            StatementKind::Assign(Box::new((
148                                *destination,
149                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
150                            ))),
151                        ));
152                        terminator.kind = TerminatorKind::goto(target);
153                    }
154                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
155                        let target = target.unwrap();
156                        let Ok([lhs, rhs]) = take_array(args) else {
157                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
158                        };
159                        let bin_op = match intrinsic.name {
160                            sym::add_with_overflow => BinOp::AddWithOverflow,
161                            sym::sub_with_overflow => BinOp::SubWithOverflow,
162                            sym::mul_with_overflow => BinOp::MulWithOverflow,
163                            _ => bug!("unexpected intrinsic"),
164                        };
165                        block.statements.push(Statement::new(
166                            terminator.source_info,
167                            StatementKind::Assign(Box::new((
168                                *destination,
169                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
170                            ))),
171                        ));
172                        terminator.kind = TerminatorKind::goto(target);
173                    }
174                    sym::read_via_copy => {
175                        let Ok([arg]) = take_array(args) else {
176                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
177                        };
178                        let derefed_place = if let Some(place) = arg.node.place()
179                            && let Some(local) = place.as_local()
180                        {
181                            tcx.mk_place_deref(local.into())
182                        } else {
183                            span_bug!(
184                                terminator.source_info.span,
185                                "Only passing a local is supported"
186                            );
187                        };
188                        // Add new statement at the end of the block that does the read, and patch
189                        // up the terminator.
190                        block.statements.push(Statement::new(
191                            terminator.source_info,
192                            StatementKind::Assign(Box::new((
193                                *destination,
194                                Rvalue::Use(Operand::Copy(derefed_place), WithRetag::Yes),
195                            ))),
196                        ));
197                        terminator.kind = match *target {
198                            None => {
199                                // No target means this read something uninhabited,
200                                // so it must be unreachable.
201                                TerminatorKind::Unreachable
202                            }
203                            Some(target) => TerminatorKind::goto(target),
204                        }
205                    }
206                    // `write_via_move` is already lowered during MIR building.
207                    sym::discriminant_value => {
208                        let target = target.unwrap();
209                        let Ok([arg]) = take_array(args) else {
210                            span_bug!(
211                                terminator.source_info.span,
212                                "Wrong arguments for discriminant_value intrinsic"
213                            );
214                        };
215                        let arg = arg.node.place().unwrap();
216                        let arg = tcx.mk_place_deref(arg);
217                        block.statements.push(Statement::new(
218                            terminator.source_info,
219                            StatementKind::Assign(Box::new((
220                                *destination,
221                                Rvalue::Discriminant(arg),
222                            ))),
223                        ));
224                        terminator.kind = TerminatorKind::goto(target);
225                    }
226                    sym::offset => {
227                        let target = target.unwrap();
228                        let Ok([ptr, delta]) = take_array(args) else {
229                            span_bug!(
230                                terminator.source_info.span,
231                                "Wrong number of arguments for offset intrinsic",
232                            );
233                        };
234                        block.statements.push(Statement::new(
235                            terminator.source_info,
236                            StatementKind::Assign(Box::new((
237                                *destination,
238                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
239                            ))),
240                        ));
241                        terminator.kind = TerminatorKind::goto(target);
242                    }
243                    sym::slice_get_unchecked => {
244                        let target = target.unwrap();
245                        let Ok([ptrish, index]) = take_array(args) else {
246                            span_bug!(
247                                terminator.source_info.span,
248                                "Wrong number of arguments for {intrinsic:?}",
249                            );
250                        };
251
252                        let place = ptrish.node.place().unwrap();
253                        assert!(!place.is_indirect());
254                        let updated_place = place.project_deeper(
255                            &[
256                                ProjectionElem::Deref,
257                                ProjectionElem::Index(
258                                    index.node.place().unwrap().as_local().unwrap(),
259                                ),
260                            ],
261                            tcx,
262                        );
263
264                        let ret_ty = generic_args.type_at(0);
265                        let rvalue = match *ret_ty.kind() {
266                            ty::RawPtr(_, Mutability::Not) => {
267                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
268                            }
269                            ty::RawPtr(_, Mutability::Mut) => {
270                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
271                            }
272                            ty::Ref(region, _, Mutability::Not) => {
273                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
274                            }
275                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
276                                region,
277                                BorrowKind::Mut { kind: MutBorrowKind::Default },
278                                updated_place,
279                            ),
280                            _ => bug!("Unknown return type {ret_ty:?}"),
281                        };
282
283                        block.statements.push(Statement::new(
284                            terminator.source_info,
285                            StatementKind::Assign(Box::new((*destination, rvalue))),
286                        ));
287                        terminator.kind = TerminatorKind::goto(target);
288                    }
289                    sym::transmute | sym::transmute_unchecked => {
290                        let dst_ty = destination.ty(local_decls, tcx).ty;
291                        let Ok([arg]) = take_array(args) else {
292                            span_bug!(
293                                terminator.source_info.span,
294                                "Wrong number of arguments for transmute intrinsic",
295                            );
296                        };
297
298                        // Always emit the cast, even if we transmute to an uninhabited type,
299                        // because that lets CTFE and codegen generate better error messages
300                        // when such a transmute actually ends up reachable.
301                        block.statements.push(Statement::new(
302                            terminator.source_info,
303                            StatementKind::Assign(Box::new((
304                                *destination,
305                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
306                            ))),
307                        ));
308                        if let Some(target) = *target {
309                            terminator.kind = TerminatorKind::goto(target);
310                        } else {
311                            terminator.kind = TerminatorKind::Unreachable;
312                        }
313                    }
314                    sym::aggregate_raw_ptr => {
315                        let Ok([data, meta]) = take_array(args) else {
316                            span_bug!(
317                                terminator.source_info.span,
318                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
319                            );
320                        };
321                        let target = target.unwrap();
322                        let pointer_ty = generic_args.type_at(0);
323                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
324                            AggregateKind::RawPtr(*pointee_ty, *mutability)
325                        } else {
326                            span_bug!(
327                                terminator.source_info.span,
328                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
329                            );
330                        };
331                        let fields = [data.node, meta.node];
332                        block.statements.push(Statement::new(
333                            terminator.source_info,
334                            StatementKind::Assign(Box::new((
335                                *destination,
336                                Rvalue::Aggregate(Box::new(kind), fields.into()),
337                            ))),
338                        ));
339                        terminator.kind = TerminatorKind::goto(target);
340                    }
341                    sym::ptr_metadata => {
342                        let Ok([ptr]) = take_array(args) else {
343                            span_bug!(
344                                terminator.source_info.span,
345                                "Wrong number of arguments for ptr_metadata intrinsic",
346                            );
347                        };
348                        let target = target.unwrap();
349                        block.statements.push(Statement::new(
350                            terminator.source_info,
351                            StatementKind::Assign(Box::new((
352                                *destination,
353                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
354                            ))),
355                        ));
356                        terminator.kind = TerminatorKind::goto(target);
357                    }
358                    _ => {}
359                }
360            }
361        }
362    }
363
364    fn policy(&self, _ctx: &crate::PassCtx<'_>) -> PassPolicy {
365        // Implements intrinsic semantics by lowering intrinsic calls to ordinary MIR operations.
366        PassPolicy::Required
367    }
368}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
