rustc_mir_transform/
lower_intrinsics.rs

Help
1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks => {
27                        let target = target.unwrap();
28                        block.statements.push(Statement {
29                            source_info: terminator.source_info,
30                            kind: StatementKind::Assign(Box::new((
31                                *destination,
32                                Rvalue::NullaryOp(NullOp::UbChecks, tcx.types.bool),
33                            ))),
34                        });
35                        terminator.kind = TerminatorKind::Goto { target };
36                    }
37                    sym::contract_checks => {
38                        let target = target.unwrap();
39                        block.statements.push(Statement {
40                            source_info: terminator.source_info,
41                            kind: StatementKind::Assign(Box::new((
42                                *destination,
43                                Rvalue::NullaryOp(NullOp::ContractChecks, tcx.types.bool),
44                            ))),
45                        });
46                        terminator.kind = TerminatorKind::Goto { target };
47                    }
48                    sym::forget => {
49                        let target = target.unwrap();
50                        block.statements.push(Statement {
51                            source_info: terminator.source_info,
52                            kind: StatementKind::Assign(Box::new((
53                                *destination,
54                                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
55                                    span: terminator.source_info.span,
56                                    user_ty: None,
57                                    const_: Const::zero_sized(tcx.types.unit),
58                                }))),
59                            ))),
60                        });
61                        terminator.kind = TerminatorKind::Goto { target };
62                    }
63                    sym::copy_nonoverlapping => {
64                        let target = target.unwrap();
65                        let Ok([src, dst, count]) = take_array(args) else {
66                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
67                        };
68                        block.statements.push(Statement {
69                            source_info: terminator.source_info,
70                            kind: StatementKind::Intrinsic(Box::new(
71                                NonDivergingIntrinsic::CopyNonOverlapping(
72                                    rustc_middle::mir::CopyNonOverlapping {
73                                        src: src.node,
74                                        dst: dst.node,
75                                        count: count.node,
76                                    },
77                                ),
78                            )),
79                        });
80                        terminator.kind = TerminatorKind::Goto { target };
81                    }
82                    sym::assume => {
83                        let target = target.unwrap();
84                        let Ok([arg]) = take_array(args) else {
85                            bug!("Wrong arguments for assume intrinsic");
86                        };
87                        block.statements.push(Statement {
88                            source_info: terminator.source_info,
89                            kind: StatementKind::Intrinsic(Box::new(
90                                NonDivergingIntrinsic::Assume(arg.node),
91                            )),
92                        });
93                        terminator.kind = TerminatorKind::Goto { target };
94                    }
95                    sym::wrapping_add
96                    | sym::wrapping_sub
97                    | sym::wrapping_mul
98                    | sym::three_way_compare
99                    | sym::unchecked_add
100                    | sym::unchecked_sub
101                    | sym::unchecked_mul
102                    | sym::unchecked_div
103                    | sym::unchecked_rem
104                    | sym::unchecked_shl
105                    | sym::unchecked_shr => {
106                        let target = target.unwrap();
107                        let Ok([lhs, rhs]) = take_array(args) else {
108                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
109                        };
110                        let bin_op = match intrinsic.name {
111                            sym::wrapping_add => BinOp::Add,
112                            sym::wrapping_sub => BinOp::Sub,
113                            sym::wrapping_mul => BinOp::Mul,
114                            sym::three_way_compare => BinOp::Cmp,
115                            sym::unchecked_add => BinOp::AddUnchecked,
116                            sym::unchecked_sub => BinOp::SubUnchecked,
117                            sym::unchecked_mul => BinOp::MulUnchecked,
118                            sym::unchecked_div => BinOp::Div,
119                            sym::unchecked_rem => BinOp::Rem,
120                            sym::unchecked_shl => BinOp::ShlUnchecked,
121                            sym::unchecked_shr => BinOp::ShrUnchecked,
122                            _ => bug!("unexpected intrinsic"),
123                        };
124                        block.statements.push(Statement {
125                            source_info: terminator.source_info,
126                            kind: StatementKind::Assign(Box::new((
127                                *destination,
128                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
129                            ))),
130                        });
131                        terminator.kind = TerminatorKind::Goto { target };
132                    }
133                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
134                        let target = target.unwrap();
135                        let Ok([lhs, rhs]) = take_array(args) else {
136                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
137                        };
138                        let bin_op = match intrinsic.name {
139                            sym::add_with_overflow => BinOp::AddWithOverflow,
140                            sym::sub_with_overflow => BinOp::SubWithOverflow,
141                            sym::mul_with_overflow => BinOp::MulWithOverflow,
142                            _ => bug!("unexpected intrinsic"),
143                        };
144                        block.statements.push(Statement {
145                            source_info: terminator.source_info,
146                            kind: StatementKind::Assign(Box::new((
147                                *destination,
148                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
149                            ))),
150                        });
151                        terminator.kind = TerminatorKind::Goto { target };
152                    }
153                    sym::size_of | sym::min_align_of => {
154                        let target = target.unwrap();
155                        let tp_ty = generic_args.type_at(0);
156                        let null_op = match intrinsic.name {
157                            sym::size_of => NullOp::SizeOf,
158                            sym::min_align_of => NullOp::AlignOf,
159                            _ => bug!("unexpected intrinsic"),
160                        };
161                        block.statements.push(Statement {
162                            source_info: terminator.source_info,
163                            kind: StatementKind::Assign(Box::new((
164                                *destination,
165                                Rvalue::NullaryOp(null_op, tp_ty),
166                            ))),
167                        });
168                        terminator.kind = TerminatorKind::Goto { target };
169                    }
170                    sym::read_via_copy => {
171                        let Ok([arg]) = take_array(args) else {
172                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
173                        };
174                        let derefed_place = if let Some(place) = arg.node.place()
175                            && let Some(local) = place.as_local()
176                        {
177                            tcx.mk_place_deref(local.into())
178                        } else {
179                            span_bug!(
180                                terminator.source_info.span,
181                                "Only passing a local is supported"
182                            );
183                        };
184                        // Add new statement at the end of the block that does the read, and patch
185                        // up the terminator.
186                        block.statements.push(Statement {
187                            source_info: terminator.source_info,
188                            kind: StatementKind::Assign(Box::new((
189                                *destination,
190                                Rvalue::Use(Operand::Copy(derefed_place)),
191                            ))),
192                        });
193                        terminator.kind = match *target {
194                            None => {
195                                // No target means this read something uninhabited,
196                                // so it must be unreachable.
197                                TerminatorKind::Unreachable
198                            }
199                            Some(target) => TerminatorKind::Goto { target },
200                        }
201                    }
202                    sym::write_via_move => {
203                        let target = target.unwrap();
204                        let Ok([ptr, val]) = take_array(args) else {
205                            span_bug!(
206                                terminator.source_info.span,
207                                "Wrong number of arguments for write_via_move intrinsic",
208                            );
209                        };
210                        let derefed_place = if let Some(place) = ptr.node.place()
211                            && let Some(local) = place.as_local()
212                        {
213                            tcx.mk_place_deref(local.into())
214                        } else {
215                            span_bug!(
216                                terminator.source_info.span,
217                                "Only passing a local is supported"
218                            );
219                        };
220                        block.statements.push(Statement {
221                            source_info: terminator.source_info,
222                            kind: StatementKind::Assign(Box::new((
223                                derefed_place,
224                                Rvalue::Use(val.node),
225                            ))),
226                        });
227                        terminator.kind = TerminatorKind::Goto { target };
228                    }
229                    sym::discriminant_value => {
230                        let target = target.unwrap();
231                        let Ok([arg]) = take_array(args) else {
232                            span_bug!(
233                                terminator.source_info.span,
234                                "Wrong arguments for discriminant_value intrinsic"
235                            );
236                        };
237                        let arg = arg.node.place().unwrap();
238                        let arg = tcx.mk_place_deref(arg);
239                        block.statements.push(Statement {
240                            source_info: terminator.source_info,
241                            kind: StatementKind::Assign(Box::new((
242                                *destination,
243                                Rvalue::Discriminant(arg),
244                            ))),
245                        });
246                        terminator.kind = TerminatorKind::Goto { target };
247                    }
248                    sym::offset => {
249                        let target = target.unwrap();
250                        let Ok([ptr, delta]) = take_array(args) else {
251                            span_bug!(
252                                terminator.source_info.span,
253                                "Wrong number of arguments for offset intrinsic",
254                            );
255                        };
256                        block.statements.push(Statement {
257                            source_info: terminator.source_info,
258                            kind: StatementKind::Assign(Box::new((
259                                *destination,
260                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
261                            ))),
262                        });
263                        terminator.kind = TerminatorKind::Goto { target };
264                    }
265                    sym::transmute | sym::transmute_unchecked => {
266                        let dst_ty = destination.ty(local_decls, tcx).ty;
267                        let Ok([arg]) = take_array(args) else {
268                            span_bug!(
269                                terminator.source_info.span,
270                                "Wrong number of arguments for transmute intrinsic",
271                            );
272                        };
273
274                        // Always emit the cast, even if we transmute to an uninhabited type,
275                        // because that lets CTFE and codegen generate better error messages
276                        // when such a transmute actually ends up reachable.
277                        block.statements.push(Statement {
278                            source_info: terminator.source_info,
279                            kind: StatementKind::Assign(Box::new((
280                                *destination,
281                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
282                            ))),
283                        });
284                        if let Some(target) = *target {
285                            terminator.kind = TerminatorKind::Goto { target };
286                        } else {
287                            terminator.kind = TerminatorKind::Unreachable;
288                        }
289                    }
290                    sym::aggregate_raw_ptr => {
291                        let Ok([data, meta]) = take_array(args) else {
292                            span_bug!(
293                                terminator.source_info.span,
294                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
295                            );
296                        };
297                        let target = target.unwrap();
298                        let pointer_ty = generic_args.type_at(0);
299                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
300                            AggregateKind::RawPtr(*pointee_ty, *mutability)
301                        } else {
302                            span_bug!(
303                                terminator.source_info.span,
304                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
305                            );
306                        };
307                        let fields = [data.node, meta.node];
308                        block.statements.push(Statement {
309                            source_info: terminator.source_info,
310                            kind: StatementKind::Assign(Box::new((
311                                *destination,
312                                Rvalue::Aggregate(Box::new(kind), fields.into()),
313                            ))),
314                        });
315                        terminator.kind = TerminatorKind::Goto { target };
316                    }
317                    sym::ptr_metadata => {
318                        let Ok([ptr]) = take_array(args) else {
319                            span_bug!(
320                                terminator.source_info.span,
321                                "Wrong number of arguments for ptr_metadata intrinsic",
322                            );
323                        };
324                        let target = target.unwrap();
325                        block.statements.push(Statement {
326                            source_info: terminator.source_info,
327                            kind: StatementKind::Assign(Box::new((
328                                *destination,
329                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
330                            ))),
331                        });
332                        terminator.kind = TerminatorKind::Goto { target };
333                    }
334                    _ => {}
335                }
336            }
337        }
338    }
339
340    fn is_required(&self) -> bool {
341        true
342    }
343}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
