1use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13 fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14 let local_decls = &body.local_decls;
15 for block in body.basic_blocks.as_mut() {
16 let terminator = block.terminator.as_mut().unwrap();
17 if let TerminatorKind::Call { func, args, destination, target, .. } =
18 &mut terminator.kind
19 && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20 && let Some(intrinsic) = tcx.intrinsic(def_id)
21 {
22 match intrinsic.name {
23 sym::unreachable => {
24 terminator.kind = TerminatorKind::Unreachable;
25 }
26 sym::ub_checks => {
27 let target = target.unwrap();
28 block.statements.push(Statement::new(
29 terminator.source_info,
30 StatementKind::Assign(Box::new((
31 *destination,
32 Rvalue::NullaryOp(NullOp::UbChecks, tcx.types.bool),
33 ))),
34 ));
35 terminator.kind = TerminatorKind::Goto { target };
36 }
37 sym::forget => {
38 let target = target.unwrap();
39 block.statements.push(Statement::new(
40 terminator.source_info,
41 StatementKind::Assign(Box::new((
42 *destination,
43 Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
44 span: terminator.source_info.span,
45 user_ty: None,
46 const_: Const::zero_sized(tcx.types.unit),
47 }))),
48 ))),
49 ));
50 terminator.kind = TerminatorKind::Goto { target };
51 }
52 sym::copy_nonoverlapping => {
53 let target = target.unwrap();
54 let Ok([src, dst, count]) = take_array(args) else {
55 bug!("Wrong arguments for copy_non_overlapping intrinsic");
56 };
57 block.statements.push(Statement::new(
58 terminator.source_info,
59 StatementKind::Intrinsic(Box::new(
60 NonDivergingIntrinsic::CopyNonOverlapping(
61 rustc_middle::mir::CopyNonOverlapping {
62 src: src.node,
63 dst: dst.node,
64 count: count.node,
65 },
66 ),
67 )),
68 ));
69 terminator.kind = TerminatorKind::Goto { target };
70 }
71 sym::assume => {
72 let target = target.unwrap();
73 let Ok([arg]) = take_array(args) else {
74 bug!("Wrong arguments for assume intrinsic");
75 };
76 block.statements.push(Statement::new(
77 terminator.source_info,
78 StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
79 arg.node,
80 ))),
81 ));
82 terminator.kind = TerminatorKind::Goto { target };
83 }
84 sym::wrapping_add
85 | sym::wrapping_sub
86 | sym::wrapping_mul
87 | sym::three_way_compare
88 | sym::unchecked_add
89 | sym::unchecked_sub
90 | sym::unchecked_mul
91 | sym::unchecked_div
92 | sym::unchecked_rem
93 | sym::unchecked_shl
94 | sym::unchecked_shr => {
95 let target = target.unwrap();
96 let Ok([lhs, rhs]) = take_array(args) else {
97 bug!("Wrong arguments for {} intrinsic", intrinsic.name);
98 };
99 let bin_op = match intrinsic.name {
100 sym::wrapping_add => BinOp::Add,
101 sym::wrapping_sub => BinOp::Sub,
102 sym::wrapping_mul => BinOp::Mul,
103 sym::three_way_compare => BinOp::Cmp,
104 sym::unchecked_add => BinOp::AddUnchecked,
105 sym::unchecked_sub => BinOp::SubUnchecked,
106 sym::unchecked_mul => BinOp::MulUnchecked,
107 sym::unchecked_div => BinOp::Div,
108 sym::unchecked_rem => BinOp::Rem,
109 sym::unchecked_shl => BinOp::ShlUnchecked,
110 sym::unchecked_shr => BinOp::ShrUnchecked,
111 _ => bug!("unexpected intrinsic"),
112 };
113 block.statements.push(Statement::new(
114 terminator.source_info,
115 StatementKind::Assign(Box::new((
116 *destination,
117 Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
118 ))),
119 ));
120 terminator.kind = TerminatorKind::Goto { target };
121 }
122 sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
123 let target = target.unwrap();
124 let Ok([lhs, rhs]) = take_array(args) else {
125 bug!("Wrong arguments for {} intrinsic", intrinsic.name);
126 };
127 let bin_op = match intrinsic.name {
128 sym::add_with_overflow => BinOp::AddWithOverflow,
129 sym::sub_with_overflow => BinOp::SubWithOverflow,
130 sym::mul_with_overflow => BinOp::MulWithOverflow,
131 _ => bug!("unexpected intrinsic"),
132 };
133 block.statements.push(Statement::new(
134 terminator.source_info,
135 StatementKind::Assign(Box::new((
136 *destination,
137 Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
138 ))),
139 ));
140 terminator.kind = TerminatorKind::Goto { target };
141 }
142 sym::size_of | sym::align_of => {
143 let target = target.unwrap();
144 let tp_ty = generic_args.type_at(0);
145 let null_op = match intrinsic.name {
146 sym::size_of => NullOp::SizeOf,
147 sym::align_of => NullOp::AlignOf,
148 _ => bug!("unexpected intrinsic"),
149 };
150 block.statements.push(Statement::new(
151 terminator.source_info,
152 StatementKind::Assign(Box::new((
153 *destination,
154 Rvalue::NullaryOp(null_op, tp_ty),
155 ))),
156 ));
157 terminator.kind = TerminatorKind::Goto { target };
158 }
159 sym::read_via_copy => {
160 let Ok([arg]) = take_array(args) else {
161 span_bug!(terminator.source_info.span, "Wrong number of arguments");
162 };
163 let derefed_place = if let Some(place) = arg.node.place()
164 && let Some(local) = place.as_local()
165 {
166 tcx.mk_place_deref(local.into())
167 } else {
168 span_bug!(
169 terminator.source_info.span,
170 "Only passing a local is supported"
171 );
172 };
173 block.statements.push(Statement::new(
176 terminator.source_info,
177 StatementKind::Assign(Box::new((
178 *destination,
179 Rvalue::Use(Operand::Copy(derefed_place)),
180 ))),
181 ));
182 terminator.kind = match *target {
183 None => {
184 TerminatorKind::Unreachable
187 }
188 Some(target) => TerminatorKind::Goto { target },
189 }
190 }
191 sym::write_via_move => {
192 let target = target.unwrap();
193 let Ok([ptr, val]) = take_array(args) else {
194 span_bug!(
195 terminator.source_info.span,
196 "Wrong number of arguments for write_via_move intrinsic",
197 );
198 };
199 let derefed_place = if let Some(place) = ptr.node.place()
200 && let Some(local) = place.as_local()
201 {
202 tcx.mk_place_deref(local.into())
203 } else {
204 span_bug!(
205 terminator.source_info.span,
206 "Only passing a local is supported"
207 );
208 };
209 block.statements.push(Statement::new(
210 terminator.source_info,
211 StatementKind::Assign(Box::new((derefed_place, Rvalue::Use(val.node)))),
212 ));
213 terminator.kind = TerminatorKind::Goto { target };
214 }
215 sym::discriminant_value => {
216 let target = target.unwrap();
217 let Ok([arg]) = take_array(args) else {
218 span_bug!(
219 terminator.source_info.span,
220 "Wrong arguments for discriminant_value intrinsic"
221 );
222 };
223 let arg = arg.node.place().unwrap();
224 let arg = tcx.mk_place_deref(arg);
225 block.statements.push(Statement::new(
226 terminator.source_info,
227 StatementKind::Assign(Box::new((
228 *destination,
229 Rvalue::Discriminant(arg),
230 ))),
231 ));
232 terminator.kind = TerminatorKind::Goto { target };
233 }
234 sym::offset => {
235 let target = target.unwrap();
236 let Ok([ptr, delta]) = take_array(args) else {
237 span_bug!(
238 terminator.source_info.span,
239 "Wrong number of arguments for offset intrinsic",
240 );
241 };
242 block.statements.push(Statement::new(
243 terminator.source_info,
244 StatementKind::Assign(Box::new((
245 *destination,
246 Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
247 ))),
248 ));
249 terminator.kind = TerminatorKind::Goto { target };
250 }
251 sym::slice_get_unchecked => {
252 let target = target.unwrap();
253 let Ok([ptrish, index]) = take_array(args) else {
254 span_bug!(
255 terminator.source_info.span,
256 "Wrong number of arguments for {intrinsic:?}",
257 );
258 };
259
260 let place = ptrish.node.place().unwrap();
261 assert!(!place.is_indirect());
262 let updated_place = place.project_deeper(
263 &[
264 ProjectionElem::Deref,
265 ProjectionElem::Index(
266 index.node.place().unwrap().as_local().unwrap(),
267 ),
268 ],
269 tcx,
270 );
271
272 let ret_ty = generic_args.type_at(0);
273 let rvalue = match *ret_ty.kind() {
274 ty::RawPtr(_, Mutability::Not) => {
275 Rvalue::RawPtr(RawPtrKind::Const, updated_place)
276 }
277 ty::RawPtr(_, Mutability::Mut) => {
278 Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
279 }
280 ty::Ref(region, _, Mutability::Not) => {
281 Rvalue::Ref(region, BorrowKind::Shared, updated_place)
282 }
283 ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
284 region,
285 BorrowKind::Mut { kind: MutBorrowKind::Default },
286 updated_place,
287 ),
288 _ => bug!("Unknown return type {ret_ty:?}"),
289 };
290
291 block.statements.push(Statement::new(
292 terminator.source_info,
293 StatementKind::Assign(Box::new((*destination, rvalue))),
294 ));
295 terminator.kind = TerminatorKind::Goto { target };
296 }
297 sym::transmute | sym::transmute_unchecked => {
298 let dst_ty = destination.ty(local_decls, tcx).ty;
299 let Ok([arg]) = take_array(args) else {
300 span_bug!(
301 terminator.source_info.span,
302 "Wrong number of arguments for transmute intrinsic",
303 );
304 };
305
306 block.statements.push(Statement::new(
310 terminator.source_info,
311 StatementKind::Assign(Box::new((
312 *destination,
313 Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
314 ))),
315 ));
316 if let Some(target) = *target {
317 terminator.kind = TerminatorKind::Goto { target };
318 } else {
319 terminator.kind = TerminatorKind::Unreachable;
320 }
321 }
322 sym::aggregate_raw_ptr => {
323 let Ok([data, meta]) = take_array(args) else {
324 span_bug!(
325 terminator.source_info.span,
326 "Wrong number of arguments for aggregate_raw_ptr intrinsic",
327 );
328 };
329 let target = target.unwrap();
330 let pointer_ty = generic_args.type_at(0);
331 let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
332 AggregateKind::RawPtr(*pointee_ty, *mutability)
333 } else {
334 span_bug!(
335 terminator.source_info.span,
336 "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
337 );
338 };
339 let fields = [data.node, meta.node];
340 block.statements.push(Statement::new(
341 terminator.source_info,
342 StatementKind::Assign(Box::new((
343 *destination,
344 Rvalue::Aggregate(Box::new(kind), fields.into()),
345 ))),
346 ));
347 terminator.kind = TerminatorKind::Goto { target };
348 }
349 sym::ptr_metadata => {
350 let Ok([ptr]) = take_array(args) else {
351 span_bug!(
352 terminator.source_info.span,
353 "Wrong number of arguments for ptr_metadata intrinsic",
354 );
355 };
356 let target = target.unwrap();
357 block.statements.push(Statement::new(
358 terminator.source_info,
359 StatementKind::Assign(Box::new((
360 *destination,
361 Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
362 ))),
363 ));
364 terminator.kind = TerminatorKind::Goto { target };
365 }
366 _ => {}
367 }
368 }
369 }
370 }
371
372 fn is_required(&self) -> bool {
373 true
374 }
375}