rustc_codegen_ssa/traits/
builder.rs

1use std::assert_matches;
2use std::ops::Deref;
3
4use rustc_abi::{Align, Scalar, Size, WrappingRange};
5use rustc_hir::attrs::AttributeKind;
6use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
7use rustc_middle::mir;
8use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
9use rustc_middle::ty::{AtomicOrdering, Instance, Ty};
10use rustc_session::config::OptLevel;
11use rustc_span::Span;
12use rustc_target::callconv::FnAbi;
13
14use super::abi::AbiBuilderMethods;
15use super::asm::AsmBuilderMethods;
16use super::consts::ConstCodegenMethods;
17use super::coverageinfo::CoverageInfoBuilderMethods;
18use super::debuginfo::DebugInfoBuilderMethods;
19use super::intrinsic::IntrinsicCallBuilderMethods;
20use super::misc::MiscCodegenMethods;
21use super::type_::{ArgAbiBuilderMethods, BaseTypeCodegenMethods, LayoutTypeCodegenMethods};
22use super::{CodegenMethods, StaticBuilderMethods};
23use crate::MemFlags;
24use crate::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
25use crate::mir::operand::{OperandRef, OperandValue};
26use crate::mir::place::{PlaceRef, PlaceValue};
27
28#[derive(#[automatically_derived]
impl ::core::marker::Copy for OverflowOp { }Copy, #[automatically_derived]
impl ::core::clone::Clone for OverflowOp {
    #[inline]
    fn clone(&self) -> OverflowOp { *self }
}Clone, #[automatically_derived]
impl ::core::fmt::Debug for OverflowOp {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                OverflowOp::Add => "Add",
                OverflowOp::Sub => "Sub",
                OverflowOp::Mul => "Mul",
            })
    }
}Debug, #[automatically_derived]
impl ::core::cmp::PartialEq for OverflowOp {
    #[inline]
    fn eq(&self, other: &OverflowOp) -> bool {
        let __self_discr = ::core::intrinsics::discriminant_value(self);
        let __arg1_discr = ::core::intrinsics::discriminant_value(other);
        __self_discr == __arg1_discr
    }
}PartialEq, #[automatically_derived]
impl ::core::cmp::Eq for OverflowOp {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {}
}Eq)]
29pub enum OverflowOp {
30    Add,
31    Sub,
32    Mul,
33}
34
35pub trait BuilderMethods<'a, 'tcx>:
36    Sized
37    + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
38    + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
39    + Deref<Target = Self::CodegenCx>
40    + CoverageInfoBuilderMethods<'tcx>
41    + DebugInfoBuilderMethods<'tcx>
42    + ArgAbiBuilderMethods<'tcx>
43    + AbiBuilderMethods
44    + IntrinsicCallBuilderMethods<'tcx>
45    + AsmBuilderMethods<'tcx>
46    + StaticBuilderMethods
47{
48    // `BackendTypes` is a supertrait of both `CodegenMethods` and
49    // `BuilderMethods`. This bound ensures all impls agree on the associated
50    // types within.
51    type CodegenCx: CodegenMethods<
52            'tcx,
53            Value = Self::Value,
54            Function = Self::Function,
55            BasicBlock = Self::BasicBlock,
56            Type = Self::Type,
57            FunctionSignature = Self::FunctionSignature,
58            Funclet = Self::Funclet,
59            DIScope = Self::DIScope,
60            DILocation = Self::DILocation,
61            DIVariable = Self::DIVariable,
62        >;
63
64    fn build(cx: &'a Self::CodegenCx, llbb: Self::BasicBlock) -> Self;
65
66    fn cx(&self) -> &Self::CodegenCx;
67    fn llbb(&self) -> Self::BasicBlock;
68
69    fn set_span(&mut self, span: Span);
70
71    // FIXME(eddyb) replace uses of this with `append_sibling_block`.
72    fn append_block(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &str) -> Self::BasicBlock;
73
74    fn append_sibling_block(&mut self, name: &str) -> Self::BasicBlock;
75
76    fn switch_to_block(&mut self, llbb: Self::BasicBlock);
77
78    fn ret_void(&mut self);
79    fn ret(&mut self, v: Self::Value);
80    fn br(&mut self, dest: Self::BasicBlock);
81    fn br_with_attrs(&mut self, dest: Self::BasicBlock, _attributes: &[AttributeKind]) {
82        self.br(dest)
83    }
84    fn cond_br(
85        &mut self,
86        cond: Self::Value,
87        then_llbb: Self::BasicBlock,
88        else_llbb: Self::BasicBlock,
89    );
90
91    // Conditional with expectation.
92    //
93    // This function is opt-in for back ends.
94    //
95    // The default implementation calls `self.expect()` before emitting the branch
96    // by calling `self.cond_br()`
97    fn cond_br_with_expect(
98        &mut self,
99        mut cond: Self::Value,
100        then_llbb: Self::BasicBlock,
101        else_llbb: Self::BasicBlock,
102        expect: Option<bool>,
103    ) {
104        if let Some(expect) = expect {
105            cond = self.expect(cond, expect);
106        }
107        self.cond_br(cond, then_llbb, else_llbb)
108    }
109
110    fn switch(
111        &mut self,
112        v: Self::Value,
113        else_llbb: Self::BasicBlock,
114        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock)>,
115    );
116
117    // This is like `switch()`, but every case has a bool flag indicating whether it's cold.
118    //
119    // Default implementation throws away the cold flags and calls `switch()`.
120    fn switch_with_weights(
121        &mut self,
122        v: Self::Value,
123        else_llbb: Self::BasicBlock,
124        _else_is_cold: bool,
125        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
126    ) {
127        self.switch(v, else_llbb, cases.map(|(val, bb, _)| (val, bb)))
128    }
129
130    fn invoke(
131        &mut self,
132        llty: Self::FunctionSignature,
133        fn_attrs: Option<&CodegenFnAttrs>,
134        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
135        llfn: Self::Value,
136        args: &[Self::Value],
137        then: Self::BasicBlock,
138        catch: Self::BasicBlock,
139        funclet: Option<&Self::Funclet>,
140        instance: Option<Instance<'tcx>>,
141    ) -> Self::Value;
142    fn unreachable(&mut self);
143
144    /// Like [`Self::unreachable`], but for use in the middle of a basic block.
145    fn unreachable_nonterminator(&mut self) {
146        // This is the preferred LLVM incantation for this per
147        // https://llvm.org/docs/Frontend/PerformanceTips.html#other-things-to-consider
148        // Other backends may override if they have a better way.
149        let const_true = self.cx().const_bool(true);
150        let poison_ptr = self.const_poison(self.cx().type_ptr());
151        self.store(const_true, poison_ptr, Align::ONE);
152    }
153
154    fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
155    fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
156    fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
157    fn fadd_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
158    fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
159    fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
160    fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
161    fn fsub_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
162    fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
163    fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
164    fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
165    fn fmul_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
166    fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
167    fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
168    fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
169    fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
170    fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
171    fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
172    fn fdiv_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
173    fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
174    fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
175    fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
176    fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
177    fn frem_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
178    /// Generate a left-shift. Both operands must have the same size. The right operand must be
179    /// interpreted as unsigned and can be assumed to be less than the size of the left operand.
180    fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
181    /// Generate a logical right-shift. Both operands must have the same size. The right operand
182    /// must be interpreted as unsigned and can be assumed to be less than the size of the left
183    /// operand.
184    fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
185    /// Generate an arithmetic right-shift. Both operands must have the same size. The right operand
186    /// must be interpreted as unsigned and can be assumed to be less than the size of the left
187    /// operand.
188    fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
189    fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
190        self.add(lhs, rhs)
191    }
192    fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
193        self.add(lhs, rhs)
194    }
195    fn unchecked_suadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
196        self.unchecked_sadd(lhs, rhs)
197    }
198    fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
199        self.sub(lhs, rhs)
200    }
201    fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
202        self.sub(lhs, rhs)
203    }
204    fn unchecked_susub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
205        self.unchecked_ssub(lhs, rhs)
206    }
207    fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
208        self.mul(lhs, rhs)
209    }
210    fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
211        self.mul(lhs, rhs)
212    }
213    fn unchecked_sumul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
214        // Which to default to is a fairly arbitrary choice,
215        // but this is what slice layout was using before.
216        self.unchecked_smul(lhs, rhs)
217    }
218    fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
219    fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
220    /// Defaults to [`Self::or`], but guarantees `(lhs & rhs) == 0` so some backends
221    /// can emit something more helpful for optimizations.
222    fn or_disjoint(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
223        self.or(lhs, rhs)
224    }
225    fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
226    fn neg(&mut self, v: Self::Value) -> Self::Value;
227    fn fneg(&mut self, v: Self::Value) -> Self::Value;
228    fn not(&mut self, v: Self::Value) -> Self::Value;
229
230    fn checked_binop(
231        &mut self,
232        oop: OverflowOp,
233        ty: Ty<'tcx>,
234        lhs: Self::Value,
235        rhs: Self::Value,
236    ) -> (Self::Value, Self::Value);
237
238    fn from_immediate(&mut self, val: Self::Value) -> Self::Value;
239    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value;
240
241    fn alloca(&mut self, size: Size, align: Align) -> Self::Value;
242    fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value;
243
244    fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
245    fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value;
246    fn atomic_load(
247        &mut self,
248        ty: Self::Type,
249        ptr: Self::Value,
250        order: AtomicOrdering,
251        size: Size,
252    ) -> Self::Value;
253    fn load_from_place(&mut self, ty: Self::Type, place: PlaceValue<Self::Value>) -> Self::Value {
254        match (&place.llextra, &None) {
    (left_val, right_val) => {
        if !(*left_val == *right_val) {
            let kind = ::core::panicking::AssertKind::Eq;
            ::core::panicking::assert_failed(kind, &*left_val, &*right_val,
                ::core::option::Option::None);
        }
    }
};assert_eq!(place.llextra, None);
255        self.load(ty, place.llval, place.align)
256    }
257    fn load_operand(&mut self, place: PlaceRef<'tcx, Self::Value>)
258    -> OperandRef<'tcx, Self::Value>;
259
260    /// Called for Rvalue::Repeat when the elem is neither a ZST nor optimizable using memset.
261    fn write_operand_repeatedly(
262        &mut self,
263        elem: OperandRef<'tcx, Self::Value>,
264        count: u64,
265        dest: PlaceRef<'tcx, Self::Value>,
266    );
267
268    /// Emits an `assume` that the integer value `imm` of type `ty` is contained in `range`.
269    ///
270    /// This *always* emits the assumption, so you probably want to check the
271    /// optimization level and `Scalar::is_always_valid` before calling it.
272    fn assume_integer_range(&mut self, imm: Self::Value, ty: Self::Type, range: WrappingRange) {
273        let WrappingRange { start, end } = range;
274
275        // Perhaps one day we'll be able to use assume operand bundles for this,
276        // but for now this encoding with a single icmp+assume is best per
277        // <https://github.com/llvm/llvm-project/issues/123278#issuecomment-2597440158>
278        let shifted = if start == 0 {
279            imm
280        } else {
281            let low = self.const_uint_big(ty, start);
282            self.sub(imm, low)
283        };
284        let width = self.const_uint_big(ty, u128::wrapping_sub(end, start));
285        let cmp = self.icmp(IntPredicate::IntULE, shifted, width);
286        self.assume(cmp);
287    }
288
289    /// Emits an `assume` that the `val` of pointer type is non-null.
290    ///
291    /// You may want to check the optimization level before bothering calling this.
292    fn assume_nonnull(&mut self, val: Self::Value) {
293        // Arguably in LLVM it'd be better to emit an assume operand bundle instead
294        // <https://llvm.org/docs/LangRef.html#assume-operand-bundles>
295        // but this works fine for all backends.
296
297        let null = self.const_null(self.type_ptr());
298        let is_null = self.icmp(IntPredicate::IntNE, val, null);
299        self.assume(is_null);
300    }
301
302    fn range_metadata(&mut self, load: Self::Value, range: WrappingRange);
303    fn nonnull_metadata(&mut self, load: Self::Value);
304
305    fn store(&mut self, val: Self::Value, ptr: Self::Value, align: Align) -> Self::Value;
306    fn store_to_place(&mut self, val: Self::Value, place: PlaceValue<Self::Value>) -> Self::Value {
307        match (&place.llextra, &None) {
    (left_val, right_val) => {
        if !(*left_val == *right_val) {
            let kind = ::core::panicking::AssertKind::Eq;
            ::core::panicking::assert_failed(kind, &*left_val, &*right_val,
                ::core::option::Option::None);
        }
    }
};assert_eq!(place.llextra, None);
308        self.store(val, place.llval, place.align)
309    }
310    fn store_with_flags(
311        &mut self,
312        val: Self::Value,
313        ptr: Self::Value,
314        align: Align,
315        flags: MemFlags,
316    ) -> Self::Value;
317    fn store_to_place_with_flags(
318        &mut self,
319        val: Self::Value,
320        place: PlaceValue<Self::Value>,
321        flags: MemFlags,
322    ) -> Self::Value {
323        match (&place.llextra, &None) {
    (left_val, right_val) => {
        if !(*left_val == *right_val) {
            let kind = ::core::panicking::AssertKind::Eq;
            ::core::panicking::assert_failed(kind, &*left_val, &*right_val,
                ::core::option::Option::None);
        }
    }
};assert_eq!(place.llextra, None);
324        self.store_with_flags(val, place.llval, place.align, flags)
325    }
326    fn atomic_store(
327        &mut self,
328        val: Self::Value,
329        ptr: Self::Value,
330        order: AtomicOrdering,
331        size: Size,
332    );
333
334    fn gep(&mut self, ty: Self::Type, ptr: Self::Value, indices: &[Self::Value]) -> Self::Value;
335    fn inbounds_gep(
336        &mut self,
337        ty: Self::Type,
338        ptr: Self::Value,
339        indices: &[Self::Value],
340    ) -> Self::Value;
341    fn inbounds_nuw_gep(
342        &mut self,
343        ty: Self::Type,
344        ptr: Self::Value,
345        indices: &[Self::Value],
346    ) -> Self::Value {
347        self.inbounds_gep(ty, ptr, indices)
348    }
349    fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
350        self.gep(self.cx().type_i8(), ptr, &[offset])
351    }
352    fn inbounds_ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
353        self.inbounds_gep(self.cx().type_i8(), ptr, &[offset])
354    }
355
356    fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
357    /// Produces the same value as [`Self::trunc`] (and defaults to that),
358    /// but is UB unless the *zero*-extending the result can reproduce `val`.
359    fn unchecked_utrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value {
360        self.trunc(val, dest_ty)
361    }
362    /// Produces the same value as [`Self::trunc`] (and defaults to that),
363    /// but is UB unless the *sign*-extending the result can reproduce `val`.
364    fn unchecked_strunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value {
365        self.trunc(val, dest_ty)
366    }
367
368    fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
369    fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
370    fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
371    fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
372    fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
373    fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
374    fn sitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
375    fn fptrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
376    fn fpext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
377    fn ptrtoint(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
378    fn inttoptr(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
379    fn bitcast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
380    fn intcast(&mut self, val: Self::Value, dest_ty: Self::Type, is_signed: bool) -> Self::Value;
381    fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
382
383    fn cast_float_to_int(
384        &mut self,
385        signed: bool,
386        x: Self::Value,
387        dest_ty: Self::Type,
388    ) -> Self::Value {
389        let in_ty = self.cx().val_ty(x);
390        let (float_ty, int_ty) = if self.cx().type_kind(dest_ty) == TypeKind::Vector
391            && self.cx().type_kind(in_ty) == TypeKind::Vector
392        {
393            (self.cx().element_type(in_ty), self.cx().element_type(dest_ty))
394        } else {
395            (in_ty, dest_ty)
396        };
397        {
    match self.cx().type_kind(float_ty) {
        TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128
            => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128",
                ::core::option::Option::None);
        }
    }
};assert_matches!(
398            self.cx().type_kind(float_ty),
399            TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128
400        );
401        match (&self.cx().type_kind(int_ty), &TypeKind::Integer) {
    (left_val, right_val) => {
        if !(*left_val == *right_val) {
            let kind = ::core::panicking::AssertKind::Eq;
            ::core::panicking::assert_failed(kind, &*left_val, &*right_val,
                ::core::option::Option::None);
        }
    }
};assert_eq!(self.cx().type_kind(int_ty), TypeKind::Integer);
402
403        if let Some(false) = self.cx().sess().opts.unstable_opts.saturating_float_casts {
404            return if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) };
405        }
406
407        if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }
408    }
409
410    fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
411    fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
412
413    /// Returns `-1` if `lhs < rhs`, `0` if `lhs == rhs`, and `1` if `lhs > rhs`.
414    fn three_way_compare(
415        &mut self,
416        ty: Ty<'tcx>,
417        lhs: Self::Value,
418        rhs: Self::Value,
419    ) -> Self::Value {
420        // FIXME: This implementation was designed around LLVM's ability to optimize, but `cg_llvm`
421        // overrides this to just use `@llvm.scmp`/`ucmp` since LLVM 20. This default impl should be
422        // reevaluated with respect to the remaining backends like cg_gcc, whether they might use
423        // specialized implementations as well, or continue to use a generic implementation here.
424        use std::cmp::Ordering;
425        let pred = |op| crate::base::bin_op_to_icmp_predicate(op, ty.is_signed());
426        if self.cx().sess().opts.optimize == OptLevel::No {
427            // This actually generates tighter assembly, and is a classic trick:
428            // <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>.
429            // However, as of 2023-11 it optimized worse in LLVM in things like derived
430            // `PartialOrd`, so we were only using it in debug. Since LLVM now uses its own
431            // intrinsics, it may be be worth trying it in optimized builds for other backends.
432            let is_gt = self.icmp(pred(mir::BinOp::Gt), lhs, rhs);
433            let gtext = self.zext(is_gt, self.type_i8());
434            let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs);
435            let ltext = self.zext(is_lt, self.type_i8());
436            self.unchecked_ssub(gtext, ltext)
437        } else {
438            // These operations were better optimized by LLVM, before `@llvm.scmp`/`ucmp` in 20.
439            // See <https://github.com/rust-lang/rust/pull/63767>.
440            let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs);
441            let is_ne = self.icmp(pred(mir::BinOp::Ne), lhs, rhs);
442            let ge = self.select(
443                is_ne,
444                self.cx().const_i8(Ordering::Greater as i8),
445                self.cx().const_i8(Ordering::Equal as i8),
446            );
447            self.select(is_lt, self.cx().const_i8(Ordering::Less as i8), ge)
448        }
449    }
450
451    fn memcpy(
452        &mut self,
453        dst: Self::Value,
454        dst_align: Align,
455        src: Self::Value,
456        src_align: Align,
457        size: Self::Value,
458        flags: MemFlags,
459        tt: Option<rustc_ast::expand::typetree::FncTree>,
460    );
461    fn memmove(
462        &mut self,
463        dst: Self::Value,
464        dst_align: Align,
465        src: Self::Value,
466        src_align: Align,
467        size: Self::Value,
468        flags: MemFlags,
469    );
470    fn memset(
471        &mut self,
472        ptr: Self::Value,
473        fill_byte: Self::Value,
474        size: Self::Value,
475        align: Align,
476        flags: MemFlags,
477    );
478
479    /// *Typed* copy for non-overlapping places.
480    ///
481    /// Has a default implementation in terms of `memcpy`, but specific backends
482    /// can override to do something smarter if possible.
483    ///
484    /// (For example, typed load-stores with alias metadata.)
485    fn typed_place_copy(
486        &mut self,
487        dst: PlaceValue<Self::Value>,
488        src: PlaceValue<Self::Value>,
489        layout: TyAndLayout<'tcx>,
490    ) {
491        self.typed_place_copy_with_flags(dst, src, layout, MemFlags::empty());
492    }
493
494    fn typed_place_copy_with_flags(
495        &mut self,
496        dst: PlaceValue<Self::Value>,
497        src: PlaceValue<Self::Value>,
498        layout: TyAndLayout<'tcx>,
499        flags: MemFlags,
500    ) {
501        if !layout.is_sized() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot typed-copy an unsigned type"));
    }
};assert!(layout.is_sized(), "cannot typed-copy an unsigned type");
502        if !src.llextra.is_none() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot directly copy from unsized values"));
    }
};assert!(src.llextra.is_none(), "cannot directly copy from unsized values");
503        if !dst.llextra.is_none() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot directly copy into unsized values"));
    }
};assert!(dst.llextra.is_none(), "cannot directly copy into unsized values");
504        if flags.contains(MemFlags::NONTEMPORAL) {
505            // HACK(nox): This is inefficient but there is no nontemporal memcpy.
506            let ty = self.backend_type(layout);
507            let val = self.load_from_place(ty, src);
508            self.store_to_place_with_flags(val, dst, flags);
509        } else if self.sess().opts.optimize == OptLevel::No && self.is_backend_immediate(layout) {
510            // If we're not optimizing, the aliasing information from `memcpy`
511            // isn't useful, so just load-store the value for smaller code.
512            let temp = self.load_operand(src.with_type(layout));
513            temp.val.store_with_flags(self, dst.with_type(layout), flags);
514        } else if !layout.is_zst() {
515            let bytes = self.const_usize(layout.size.bytes());
516            self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags, None);
517        }
518    }
519
520    /// *Typed* swap for non-overlapping places.
521    ///
522    /// Avoids `alloca`s for Immediates and ScalarPairs.
523    ///
524    /// FIXME: Maybe do something smarter for Ref types too?
525    /// For now, the `typed_swap_nonoverlapping` intrinsic just doesn't call this for those
526    /// cases (in non-debug), preferring the fallback body instead.
527    fn typed_place_swap(
528        &mut self,
529        left: PlaceValue<Self::Value>,
530        right: PlaceValue<Self::Value>,
531        layout: TyAndLayout<'tcx>,
532    ) {
533        let mut temp = self.load_operand(left.with_type(layout));
534        if let OperandValue::Ref(..) = temp.val {
535            // The SSA value isn't stand-alone, so we need to copy it elsewhere
536            let alloca = PlaceRef::alloca(self, layout);
537            self.typed_place_copy(alloca.val, left, layout);
538            temp = self.load_operand(alloca);
539        }
540        self.typed_place_copy(left, right, layout);
541        temp.val.store(self, right.with_type(layout));
542    }
543
544    fn select(
545        &mut self,
546        cond: Self::Value,
547        then_val: Self::Value,
548        else_val: Self::Value,
549    ) -> Self::Value;
550
551    fn va_arg(&mut self, list: Self::Value, ty: Self::Type) -> Self::Value;
552    fn extract_element(&mut self, vec: Self::Value, idx: Self::Value) -> Self::Value;
553    fn vector_splat(&mut self, num_elts: usize, elt: Self::Value) -> Self::Value;
554    fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value;
555    fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value;
556
557    fn set_personality_fn(&mut self, personality: Self::Function);
558
559    // These are used by everyone except msvc and wasm EH
560    fn cleanup_landing_pad(&mut self, pers_fn: Self::Function) -> (Self::Value, Self::Value);
561    fn filter_landing_pad(&mut self, pers_fn: Self::Function);
562    fn resume(&mut self, exn0: Self::Value, exn1: Self::Value);
563
564    // These are used by msvc and wasm EH
565    fn cleanup_pad(&mut self, parent: Option<Self::Value>, args: &[Self::Value]) -> Self::Funclet;
566    fn cleanup_ret(&mut self, funclet: &Self::Funclet, unwind: Option<Self::BasicBlock>);
567    fn catch_pad(&mut self, parent: Self::Value, args: &[Self::Value]) -> Self::Funclet;
568    fn catch_switch(
569        &mut self,
570        parent: Option<Self::Value>,
571        unwind: Option<Self::BasicBlock>,
572        handlers: &[Self::BasicBlock],
573    ) -> Self::Value;
574    fn get_funclet_cleanuppad(&self, funclet: &Self::Funclet) -> Self::Value;
575
576    fn atomic_cmpxchg(
577        &mut self,
578        dst: Self::Value,
579        cmp: Self::Value,
580        src: Self::Value,
581        order: AtomicOrdering,
582        failure_order: AtomicOrdering,
583        weak: bool,
584    ) -> (Self::Value, Self::Value);
585    /// `ret_ptr` indicates whether the return type (which is also the type `dst` points to)
586    /// is a pointer or the same type as `src`.
587    fn atomic_rmw(
588        &mut self,
589        op: AtomicRmwBinOp,
590        dst: Self::Value,
591        src: Self::Value,
592        order: AtomicOrdering,
593        ret_ptr: bool,
594    ) -> Self::Value;
595    fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope);
596    fn set_invariant_load(&mut self, load: Self::Value);
597
598    /// Called for `StorageLive`
599    fn lifetime_start(&mut self, ptr: Self::Value, size: Size);
600
601    /// Called for `StorageDead`
602    fn lifetime_end(&mut self, ptr: Self::Value, size: Size);
603
604    /// "Finally codegen the call"
605    ///
606    /// ## Arguments
607    ///
608    /// `caller_attrs` are the attributes of the surrounding caller; they have nothing to do with
609    /// the callee.
610    ///
611    /// The `caller_attrs`, `fn_abi`, and `callee_instance` arguments are Options because they are
612    /// advisory. They relate to optional codegen enhancements like LLVM CFI, and do not affect ABI
613    /// per se. Any ABI-related transformations should be handled by different, earlier stages of
614    /// codegen. For instance, in the caller of `BuilderMethods::call`.
615    ///
616    /// This means that a codegen backend which disregards `fn_attrs`, `fn_abi`, and `instance`
617    /// should still do correct codegen, and code should not be miscompiled if they are omitted.
618    /// It is not a miscompilation in this sense if it fails to run under CFI, other sanitizers, or
619    /// in the context of other compiler-enhanced security features.
620    ///
621    /// The typical case that they are None is during the codegen of intrinsics and lang-items,
622    /// as those are "fake functions" with only a trivial ABI if any, et cetera.
623    ///
624    /// ## Return
625    ///
626    /// Must return the value the function will return so it can be written to the destination,
627    /// assuming the function does not explicitly pass the destination as a pointer in `args`.
628    fn call(
629        &mut self,
630        llty: Self::FunctionSignature,
631        caller_attrs: Option<&CodegenFnAttrs>,
632        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
633        fn_val: Self::Value,
634        args: &[Self::Value],
635        funclet: Option<&Self::Funclet>,
636        callee_instance: Option<Instance<'tcx>>,
637    ) -> Self::Value;
638
639    fn tail_call(
640        &mut self,
641        llty: Self::FunctionSignature,
642        caller_attrs: Option<&CodegenFnAttrs>,
643        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
644        llfn: Self::Value,
645        args: &[Self::Value],
646        funclet: Option<&Self::Funclet>,
647        callee_instance: Option<Instance<'tcx>>,
648    );
649
650    fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
651
652    fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value);
653}
rustc_codegen_ssa/traits/builder.rs

rustc_codegen_ssa/traits/
builder.rs
