rustc_codegen_ssa/traits/
builder.rs

1use std::ops::Deref;
2
3use rustc_abi::{Align, Scalar, Size, WrappingRange};
4use rustc_ast::expand::typetree::{FncTree, TypeTree};
5use rustc_attr_ir::UnrollAttr;
6use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
7use rustc_middle::mir;
8use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
9use rustc_middle::ty::typetree::typetree_from_ty;
10use rustc_middle::ty::{AtomicOrdering, Instance, Ty};
11use rustc_session::config::OptLevel;
12use rustc_span::Span;
13use rustc_target::callconv::FnAbi;
14
15use super::abi::AbiBuilderMethods;
16use super::asm::AsmBuilderMethods;
17use super::consts::ConstCodegenMethods;
18use super::coverageinfo::CoverageInfoBuilderMethods;
19use super::debuginfo::DebugInfoBuilderMethods;
20use super::intrinsic::IntrinsicCallBuilderMethods;
21use super::misc::MiscCodegenMethods;
22use super::type_::{ArgAbiBuilderMethods, BaseTypeCodegenMethods, LayoutTypeCodegenMethods};
23use super::{CodegenMethods, StaticBuilderMethods};
24use crate::MemFlags;
25use crate::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope};
26use crate::mir::operand::{OperandRef, OperandValue};
27use crate::mir::place::{PlaceRef, PlaceValue};
28
29#[derive(#[automatically_derived]impl ::core::marker::Copy for OverflowOp { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for OverflowOp {
}
#[automatically_derived]impl ::core::clone::Clone for OverflowOp {
    #[inline]
    fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::fmt::Debug for OverflowOp {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                OverflowOp::Add => "Add",
                OverflowOp::Sub => "Sub",
                OverflowOp::Mul => "Mul",
            })
    }
}Debug, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    OverflowOp {
}
#[automatically_derived]impl ::core::cmp::PartialEq for OverflowOp {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
            ::core::intrinsics::discriminant_value(other)
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for OverflowOp { }Eq)]
30pub enum OverflowOp {
31    Add,
32    Sub,
33    Mul,
34}
35
36/// The location of the return value for the call.
37#[derive(#[automatically_derived]impl<V: ::core::marker::Copy> ::core::marker::Copy for
    ReturnSlot<V> {
}Copy, #[automatically_derived]impl<V: ::core::clone::Clone> ::core::clone::Clone for
    ReturnSlot<V> {
    #[inline]
    fn clone(&self) -> Self {
        match self {
            Self::Direct => ReturnSlot::Direct,
            Self::Indirect(__self_0) =>
                ReturnSlot::Indirect(::core::clone::Clone::clone(__self_0)),
        }
    }
}Clone, #[automatically_derived]impl<V: ::core::fmt::Debug> ::core::fmt::Debug for
    ReturnSlot<V> {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        match self {
            Self::Direct => ::core::fmt::Formatter::write_str(f, "Direct"),
            Self::Indirect(__self_0) =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "Indirect", &__self_0),
        }
    }
}Debug)]
38pub enum ReturnSlot<V> {
39    Direct,
40    /// The return value will be passed via sret (e.g. `PassMode::Indirect`).
41    Indirect(V),
42}
43
44impl<V> ReturnSlot<V> {
45    pub fn is_indirect(&self) -> bool {
46        #[allow(non_exhaustive_omitted_patterns)] match self {
    ReturnSlot::Indirect(_) => true,
    _ => false,
}matches!(self, ReturnSlot::Indirect(_))
47    }
48}
49
50pub trait BuilderMethods<'a, 'tcx>:
51    Sized
52    + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
53    + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
54    + Deref<Target = Self::CodegenCx>
55    + CoverageInfoBuilderMethods<'tcx>
56    + DebugInfoBuilderMethods<'tcx>
57    + ArgAbiBuilderMethods<'tcx>
58    + AbiBuilderMethods
59    + IntrinsicCallBuilderMethods<'tcx>
60    + AsmBuilderMethods<'tcx>
61    + StaticBuilderMethods
62{
63    // `BackendTypes` is a supertrait of both `CodegenMethods` and
64    // `BuilderMethods`. This bound ensures all impls agree on the associated
65    // types within.
66    type CodegenCx: CodegenMethods<
67            'tcx,
68            Value = Self::Value,
69            Function = Self::Function,
70            BasicBlock = Self::BasicBlock,
71            Type = Self::Type,
72            FunctionSignature = Self::FunctionSignature,
73            Funclet = Self::Funclet,
74            DIScope = Self::DIScope,
75            DILocation = Self::DILocation,
76            DIVariable = Self::DIVariable,
77        >;
78
79    fn build(cx: &'a Self::CodegenCx, llbb: Self::BasicBlock) -> Self;
80
81    fn cx(&self) -> &Self::CodegenCx;
82    fn llbb(&self) -> Self::BasicBlock;
83
84    fn set_span(&mut self, span: Span);
85
86    // FIXME(eddyb) replace uses of this with `append_sibling_block`.
87    fn append_block(cx: &'a Self::CodegenCx, llfn: Self::Function, name: &str) -> Self::BasicBlock;
88
89    fn append_sibling_block(&mut self, name: &str) -> Self::BasicBlock;
90
91    fn switch_to_block(&mut self, llbb: Self::BasicBlock);
92
93    fn ret_void(&mut self);
94    fn ret(&mut self, v: Self::Value);
95    fn br(&mut self, dest: Self::BasicBlock);
96    fn br_with_attrs(&mut self, dest: Self::BasicBlock, _loop_hint_attrs: &[UnrollAttr]) {
97        self.br(dest)
98    }
99    fn cond_br(
100        &mut self,
101        cond: Self::Value,
102        then_llbb: Self::BasicBlock,
103        else_llbb: Self::BasicBlock,
104    );
105
106    // Conditional with expectation.
107    //
108    // This function is opt-in for back ends.
109    //
110    // The default implementation calls `self.expect()` before emitting the branch
111    // by calling `self.cond_br()`
112    fn cond_br_with_expect(
113        &mut self,
114        mut cond: Self::Value,
115        then_llbb: Self::BasicBlock,
116        else_llbb: Self::BasicBlock,
117        expect: Option<bool>,
118    ) {
119        if let Some(expect) = expect {
120            cond = self.expect(cond, expect);
121        }
122        self.cond_br(cond, then_llbb, else_llbb)
123    }
124
125    fn switch(
126        &mut self,
127        v: Self::Value,
128        else_llbb: Self::BasicBlock,
129        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock)>,
130    );
131
132    // This is like `switch()`, but every case has a bool flag indicating whether it's cold.
133    //
134    // Default implementation throws away the cold flags and calls `switch()`.
135    fn switch_with_weights(
136        &mut self,
137        v: Self::Value,
138        else_llbb: Self::BasicBlock,
139        _else_is_cold: bool,
140        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
141    ) {
142        self.switch(v, else_llbb, cases.map(|(val, bb, _)| (val, bb)))
143    }
144
145    fn invoke(
146        &mut self,
147        llty: Self::FunctionSignature,
148        fn_attrs: Option<&CodegenFnAttrs>,
149        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
150        llfn: Self::Value,
151        return_slot: ReturnSlot<Self::Value>,
152        args: &[Self::Value],
153        then: Self::BasicBlock,
154        catch: Self::BasicBlock,
155        funclet: Option<&Self::Funclet>,
156        instance: Option<Instance<'tcx>>,
157    ) -> Self::Value;
158    fn unreachable(&mut self);
159
160    /// Like [`Self::unreachable`], but for use in the middle of a basic block.
161    fn unreachable_nonterminator(&mut self) {
162        // This is the preferred LLVM incantation for this per
163        // https://llvm.org/docs/Frontend/PerformanceTips.html#other-things-to-consider
164        // Other backends may override if they have a better way.
165        let const_true = self.cx().const_bool(true);
166        let poison_ptr = self.const_poison(self.cx().type_ptr());
167        self.store(const_true, poison_ptr, Align::ONE);
168    }
169
170    fn add(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
171    fn fadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
172    fn fadd_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
173    fn fadd_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
174    fn sub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
175    fn fsub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
176    fn fsub_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
177    fn fsub_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
178    fn mul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
179    fn fmul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
180    fn fmul_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
181    fn fmul_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
182    fn udiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
183    fn exactudiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
184    fn sdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
185    fn exactsdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
186    fn fdiv(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
187    fn fdiv_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
188    fn fdiv_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
189    fn urem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
190    fn srem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
191    fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
192    fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
193    fn frem_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
194    /// Generate a left-shift. Both operands must have the same size. The right operand must be
195    /// interpreted as unsigned and can be assumed to be less than the size of the left operand.
196    fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
197    /// Generate a logical right-shift. Both operands must have the same size. The right operand
198    /// must be interpreted as unsigned and can be assumed to be less than the size of the left
199    /// operand.
200    fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
201    /// Generate an arithmetic right-shift. Both operands must have the same size. The right operand
202    /// must be interpreted as unsigned and can be assumed to be less than the size of the left
203    /// operand.
204    fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
205    fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
206        self.add(lhs, rhs)
207    }
208    fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
209        self.add(lhs, rhs)
210    }
211    fn unchecked_suadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
212        self.unchecked_sadd(lhs, rhs)
213    }
214    fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
215        self.sub(lhs, rhs)
216    }
217    fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
218        self.sub(lhs, rhs)
219    }
220    fn unchecked_susub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
221        self.unchecked_ssub(lhs, rhs)
222    }
223    fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
224        self.mul(lhs, rhs)
225    }
226    fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
227        self.mul(lhs, rhs)
228    }
229    fn unchecked_sumul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
230        // Which to default to is a fairly arbitrary choice,
231        // but this is what slice layout was using before.
232        self.unchecked_smul(lhs, rhs)
233    }
234    fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
235    fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
236    /// Defaults to [`Self::or`], but guarantees `(lhs & rhs) == 0` so some backends
237    /// can emit something more helpful for optimizations.
238    fn or_disjoint(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value {
239        self.or(lhs, rhs)
240    }
241    fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
242    fn neg(&mut self, v: Self::Value) -> Self::Value;
243    fn fneg(&mut self, v: Self::Value) -> Self::Value;
244    fn not(&mut self, v: Self::Value) -> Self::Value;
245
246    fn checked_binop(
247        &mut self,
248        oop: OverflowOp,
249        ty: Ty<'tcx>,
250        lhs: Self::Value,
251        rhs: Self::Value,
252    ) -> (Self::Value, Self::Value);
253
254    fn from_immediate(&mut self, val: Self::Value) -> Self::Value;
255    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value;
256
257    fn alloca(&mut self, size: Size, align: Align) -> Self::Value;
258    fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value;
259
260    fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
261    fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
262    fn atomic_load(
263        &mut self,
264        ty: Self::Type,
265        ptr: Self::Value,
266        order: AtomicOrdering,
267        volatile: bool,
268        size: Size,
269    ) -> Self::Value;
270    fn load_from_place(&mut self, ty: Self::Type, place: PlaceValue<Self::Value>) -> Self::Value {
271        {
    match (&place.llextra, &None) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(place.llextra, None);
272        self.load(ty, place.llval, place.align)
273    }
274    fn load_operand(&mut self, place: PlaceRef<'tcx, Self::Value>)
275    -> OperandRef<'tcx, Self::Value>;
276
277    /// Called for Rvalue::Repeat when the elem is neither a ZST nor optimizable using memset.
278    fn write_operand_repeatedly(
279        &mut self,
280        elem: OperandRef<'tcx, Self::Value>,
281        count: u64,
282        dest: PlaceRef<'tcx, Self::Value>,
283    );
284
285    /// Emits an `assume` that the integer value `imm` of type `ty` is contained in `range`.
286    ///
287    /// This *always* emits the assumption, so you probably want to check the
288    /// optimization level and `Scalar::is_always_valid` before calling it.
289    fn assume_integer_range(&mut self, imm: Self::Value, ty: Self::Type, range: WrappingRange) {
290        let WrappingRange { start, end } = range;
291
292        // Perhaps one day we'll be able to use assume operand bundles for this,
293        // but for now this encoding with a single icmp+assume is best per
294        // <https://github.com/llvm/llvm-project/issues/123278#issuecomment-2597440158>
295        let shifted = if start == 0 {
296            imm
297        } else {
298            let low = self.const_uint_big(ty, start);
299            self.sub(imm, low)
300        };
301        let width = self.const_uint_big(ty, u128::wrapping_sub(end, start));
302        let cmp = self.icmp(IntPredicate::IntULE, shifted, width);
303        self.assume(cmp);
304    }
305
306    /// Emits an `assume` that the `val` of pointer type is non-null.
307    ///
308    /// You may want to check the optimization level before bothering calling this.
309    fn assume_nonnull(&mut self, val: Self::Value) {
310        // Arguably in LLVM it'd be better to emit an assume operand bundle instead
311        // <https://llvm.org/docs/LangRef.html#assume-operand-bundles>
312        // but this works fine for all backends.
313
314        let null = self.const_null(self.type_ptr());
315        let is_null = self.icmp(IntPredicate::IntNE, val, null);
316        self.assume(is_null);
317    }
318
319    fn range_metadata(&mut self, load: Self::Value, range: WrappingRange);
320    fn nonnull_metadata(&mut self, load: Self::Value);
321
322    fn store(&mut self, val: Self::Value, ptr: Self::Value, align: Align) {
323        self.store_with_flags(val, ptr, align, MemFlags::empty())
324    }
325    fn store_to_place(&mut self, val: Self::Value, place: PlaceValue<Self::Value>) {
326        {
    match (&place.llextra, &None) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(place.llextra, None);
327        self.store(val, place.llval, place.align);
328    }
329    fn store_with_flags(
330        &mut self,
331        val: Self::Value,
332        ptr: Self::Value,
333        align: Align,
334        flags: MemFlags,
335    );
336    fn store_to_place_with_flags(
337        &mut self,
338        val: Self::Value,
339        place: PlaceValue<Self::Value>,
340        flags: MemFlags,
341    ) {
342        {
    match (&place.llextra, &None) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(place.llextra, None);
343        self.store_with_flags(val, place.llval, place.align, flags)
344    }
345    fn atomic_store(
346        &mut self,
347        val: Self::Value,
348        ptr: Self::Value,
349        order: AtomicOrdering,
350        volatile: bool,
351        size: Size,
352    );
353
354    fn gep(&mut self, stride: Size, ptr: Self::Value, index: Self::Value) -> Self::Value;
355    fn inbounds_gep(&mut self, stride: Size, ptr: Self::Value, index: Self::Value) -> Self::Value {
356        self.gep(stride, ptr, index)
357    }
358    fn inbounds_nuw_gep(
359        &mut self,
360        stride: Size,
361        ptr: Self::Value,
362        index: Self::Value,
363    ) -> Self::Value {
364        self.inbounds_gep(stride, ptr, index)
365    }
366    fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
367        self.gep(Size::from_bytes(1), ptr, offset)
368    }
369    fn inbounds_ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value {
370        self.inbounds_gep(Size::from_bytes(1), ptr, offset)
371    }
372
373    fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
374    /// Produces the same value as [`Self::trunc`] (and defaults to that),
375    /// but is UB unless the *zero*-extending the result can reproduce `val`.
376    fn unchecked_utrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value {
377        self.trunc(val, dest_ty)
378    }
379    /// Produces the same value as [`Self::trunc`] (and defaults to that),
380    /// but is UB unless the *sign*-extending the result can reproduce `val`.
381    fn unchecked_strunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value {
382        self.trunc(val, dest_ty)
383    }
384
385    fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
386    fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
387    fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
388    fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
389    fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
390    fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
391    fn sitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
392    fn fptrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
393    fn fpext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
394    fn ptrtoint(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
395    fn inttoptr(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
396    fn bitcast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
397    fn intcast(&mut self, val: Self::Value, dest_ty: Self::Type, is_signed: bool) -> Self::Value;
398    fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
399
400    fn cast_float_to_int(
401        &mut self,
402        signed: bool,
403        x: Self::Value,
404        dest_ty: Self::Type,
405    ) -> Self::Value {
406        let in_ty = self.cx().val_ty(x);
407        let (float_ty, int_ty) = if let Some(dest_ty) = self.cx().vector_element_type(dest_ty)
408            && let Some(in_ty) = self.cx().vector_element_type(in_ty)
409        {
410            (in_ty, dest_ty)
411        } else {
412            (in_ty, dest_ty)
413        };
414        if !self.cx().float_type(float_ty).is_some_and(|f| f.is_ieee()) {
    ::core::panicking::panic("assertion failed: self.cx().float_type(float_ty).is_some_and(|f| f.is_ieee())")
};assert!(self.cx().float_type(float_ty).is_some_and(|f| f.is_ieee()));
415        if !self.cx().int_width(int_ty).is_some() {
    ::core::panicking::panic("assertion failed: self.cx().int_width(int_ty).is_some()")
};assert!(self.cx().int_width(int_ty).is_some());
416
417        if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }
418    }
419
420    fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
421    fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
422
423    /// Returns `-1` if `lhs < rhs`, `0` if `lhs == rhs`, and `1` if `lhs > rhs`.
424    fn three_way_compare(
425        &mut self,
426        ty: Ty<'tcx>,
427        lhs: Self::Value,
428        rhs: Self::Value,
429    ) -> Self::Value {
430        // FIXME: This implementation was designed around LLVM's ability to optimize, but `cg_llvm`
431        // overrides this to just use `@llvm.scmp`/`ucmp` since LLVM 20. This default impl should be
432        // reevaluated with respect to the remaining backends like cg_gcc, whether they might use
433        // specialized implementations as well, or continue to use a generic implementation here.
434        use std::cmp::Ordering;
435        let pred = |op| crate::base::bin_op_to_icmp_predicate(op, ty.is_signed());
436        if self.cx().sess().opts.optimize == OptLevel::No {
437            // This actually generates tighter assembly, and is a classic trick:
438            // <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>.
439            // However, as of 2023-11 it optimized worse in LLVM in things like derived
440            // `PartialOrd`, so we were only using it in debug. Since LLVM now uses its own
441            // intrinsics, it may be be worth trying it in optimized builds for other backends.
442            let is_gt = self.icmp(pred(mir::BinOp::Gt), lhs, rhs);
443            let gtext = self.zext(is_gt, self.type_i8());
444            let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs);
445            let ltext = self.zext(is_lt, self.type_i8());
446            self.unchecked_ssub(gtext, ltext)
447        } else {
448            // These operations were better optimized by LLVM, before `@llvm.scmp`/`ucmp` in 20.
449            // See <https://github.com/rust-lang/rust/pull/63767>.
450            let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs);
451            let is_ne = self.icmp(pred(mir::BinOp::Ne), lhs, rhs);
452            let ge = self.select(
453                is_ne,
454                self.cx().const_i8(Ordering::Greater as i8),
455                self.cx().const_i8(Ordering::Equal as i8),
456            );
457            self.select(is_lt, self.cx().const_i8(Ordering::Less as i8), ge)
458        }
459    }
460
461    fn memcpy(
462        &mut self,
463        dst: Self::Value,
464        dst_align: Align,
465        src: Self::Value,
466        src_align: Align,
467        size: Self::Value,
468        flags: MemFlags,
469        tt: Option<FncTree>,
470    );
471    fn memmove(
472        &mut self,
473        dst: Self::Value,
474        dst_align: Align,
475        src: Self::Value,
476        src_align: Align,
477        size: Self::Value,
478        flags: MemFlags,
479    );
480    fn memset(
481        &mut self,
482        ptr: Self::Value,
483        fill_byte: Self::Value,
484        size: Self::Value,
485        align: Align,
486        flags: MemFlags,
487    );
488
489    // Produce a value from calling the `vscale` intrinsic (containing the `vscale` multiplier that
490    // a scalable vector's element size and count can be multiplied by to get the real size of the
491    // vector)
492    fn vscale(&mut self, ty: Self::Type) -> Self::Value;
493
494    /// *Typed* copy for non-overlapping places.
495    ///
496    /// Has a default implementation in terms of `memcpy`, but specific backends
497    /// can override to do something smarter if possible.
498    ///
499    /// (For example, typed load-stores with alias metadata.)
500    fn typed_place_copy(
501        &mut self,
502        dst: PlaceValue<Self::Value>,
503        src: PlaceValue<Self::Value>,
504        layout: TyAndLayout<'tcx>,
505    ) {
506        self.typed_place_copy_with_flags(dst, src, layout, MemFlags::empty());
507    }
508
509    fn typed_place_copy_with_flags(
510        &mut self,
511        dst: PlaceValue<Self::Value>,
512        src: PlaceValue<Self::Value>,
513        layout: TyAndLayout<'tcx>,
514        flags: MemFlags,
515    ) {
516        if !layout.is_sized() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot typed-copy an unsigned type"));
    }
};assert!(layout.is_sized(), "cannot typed-copy an unsigned type");
517        if !src.llextra.is_none() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot directly copy from unsized values"));
    }
};assert!(src.llextra.is_none(), "cannot directly copy from unsized values");
518        if !dst.llextra.is_none() {
    {
        ::core::panicking::panic_fmt(format_args!("cannot directly copy into unsized values"));
    }
};assert!(dst.llextra.is_none(), "cannot directly copy into unsized values");
519        if flags.contains(MemFlags::NONTEMPORAL) {
520            // HACK(nox): This is inefficient but there is no nontemporal memcpy.
521            let ty = self.backend_type(layout);
522            let val = self.load_from_place(ty, src);
523            self.store_to_place_with_flags(val, dst, flags);
524        } else if self.sess().opts.optimize == OptLevel::No
525            && layout.backend_repr.is_scalar_or_simd()
526        {
527            // If we're not optimizing, the aliasing information from `memcpy`
528            // isn't useful, so just load-store the value for smaller code.
529            let temp = self.load_operand(src.with_type(layout));
530            temp.val.store_with_flags(self, dst.with_type(layout), flags);
531        } else if !layout.is_zst() {
532            let tt = typetree_from_ty(self.tcx(), layout.ty);
533            // We seem to pass all values to memcpy with one more indirection.
534            let tt = tt.add_indirection();
535            let fnc_tree = FncTree { args: ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [tt.clone(), tt]))vec![tt.clone(), tt], ret: TypeTree::new() };
536            let bytes = self.const_usize(layout.size.bytes());
537            let bytes =
538                // FIXME(rustc_scalable_vector/stdarch_aarch64_sve): Scalable vectors aren't
539                // actually sized, but we pretend they are. Here we have to hack around that.
540                if layout.peel_transparent_wrappers_from_non_1zst(self).ty.is_scalable_vector() {
541                    let vscale = self.vscale(self.type_i64());
542                    self.mul(vscale, bytes)
543                } else {
544                    bytes
545                };
546            self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags, Some(fnc_tree));
547        }
548    }
549
550    /// *Typed* swap for non-overlapping places.
551    ///
552    /// Avoids `alloca`s for Immediates and ScalarPairs.
553    ///
554    /// FIXME: Maybe do something smarter for Ref types too?
555    /// For now, the `typed_swap_nonoverlapping` intrinsic just doesn't call this for those
556    /// cases (in non-debug), preferring the fallback body instead.
557    fn typed_place_swap(
558        &mut self,
559        left: PlaceValue<Self::Value>,
560        right: PlaceValue<Self::Value>,
561        layout: TyAndLayout<'tcx>,
562    ) {
563        let mut temp = self.load_operand(left.with_type(layout));
564        if let OperandValue::Ref(..) = temp.val {
565            // The SSA value isn't stand-alone, so we need to copy it elsewhere
566            let alloca = PlaceRef::alloca(self, layout);
567            self.typed_place_copy(alloca.val, left, layout);
568            temp = self.load_operand(alloca);
569        }
570        self.typed_place_copy(left, right, layout);
571        temp.val.store(self, right.with_type(layout));
572    }
573
574    fn select(
575        &mut self,
576        cond: Self::Value,
577        then_val: Self::Value,
578        else_val: Self::Value,
579    ) -> Self::Value;
580
581    fn va_arg(&mut self, list: Self::Value, ty: Self::Type) -> Self::Value;
582    fn extract_element(&mut self, vec: Self::Value, idx: Self::Value) -> Self::Value;
583    fn vector_splat(&mut self, num_elts: u64, elt: Self::Value) -> Self::Value;
584    fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value;
585    fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value;
586
587    fn set_personality_fn(&mut self, personality: Self::Function);
588
589    // These are used by everyone except msvc and wasm EH
590    fn cleanup_landing_pad(&mut self, pers_fn: Self::Function) -> (Self::Value, Self::Value);
591    fn filter_landing_pad(&mut self, pers_fn: Self::Function);
592    fn resume(&mut self, exn0: Self::Value, exn1: Self::Value);
593
594    // These are used by msvc and wasm EH
595    fn cleanup_pad(&mut self, parent: Option<Self::Value>, args: &[Self::Value]) -> Self::Funclet;
596    fn cleanup_ret(&mut self, funclet: &Self::Funclet, unwind: Option<Self::BasicBlock>);
597    fn catch_pad(&mut self, parent: Self::Value, args: &[Self::Value]) -> Self::Funclet;
598    fn catch_switch(
599        &mut self,
600        parent: Option<Self::Value>,
601        unwind: Option<Self::BasicBlock>,
602        handlers: &[Self::BasicBlock],
603    ) -> Self::Value;
604    fn get_funclet_cleanuppad(&self, funclet: &Self::Funclet) -> Self::Value;
605
606    fn atomic_cmpxchg(
607        &mut self,
608        dst: Self::Value,
609        cmp: Self::Value,
610        src: Self::Value,
611        order: AtomicOrdering,
612        failure_order: AtomicOrdering,
613        weak: bool,
614    ) -> (Self::Value, Self::Value);
615    /// `ret_ptr` indicates whether the return type (which is also the type `dst` points to)
616    /// is a pointer or the same type as `src`.
617    fn atomic_rmw(
618        &mut self,
619        op: AtomicRmwBinOp,
620        dst: Self::Value,
621        src: Self::Value,
622        order: AtomicOrdering,
623        ret_ptr: bool,
624    ) -> Self::Value;
625    fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope);
626    fn set_invariant_load(&mut self, load: Self::Value);
627
628    /// Called for `StorageLive`
629    fn lifetime_start(&mut self, ptr: Self::Value, size: Size);
630
631    /// Called for `StorageDead`
632    fn lifetime_end(&mut self, ptr: Self::Value, size: Size);
633
634    /// "Finally codegen the call"
635    ///
636    /// ## Arguments
637    ///
638    /// `caller_attrs` are the attributes of the surrounding caller; they have nothing to do with
639    /// the callee.
640    ///
641    /// The `caller_attrs`, `fn_abi`, and `callee_instance` arguments are Options because they are
642    /// advisory. They relate to optional codegen enhancements like LLVM CFI, and do not affect ABI
643    /// per se. Any ABI-related transformations should be handled by different, earlier stages of
644    /// codegen. For instance, in the caller of `BuilderMethods::call`.
645    ///
646    /// This means that a codegen backend which disregards `fn_attrs`, `fn_abi`, and `instance`
647    /// should still do correct codegen, and code should not be miscompiled if they are omitted.
648    /// It is not a miscompilation in this sense if it fails to run under CFI, other sanitizers, or
649    /// in the context of other compiler-enhanced security features.
650    ///
651    /// The typical case that they are None is during the codegen of intrinsics and lang-items,
652    /// as those are "fake functions" with only a trivial ABI if any, et cetera.
653    ///
654    /// `return_slot` must be `ReturnSlot::Indirect` if an argument uses `PassMode::Indirect`.
655    ///
656    /// ## Return
657    ///
658    /// Must return the value the function will return so it can be written to the destination.
659    /// For calls with an indirect return, the returned value is meaningless and must not be
660    /// used: the return value lives in the return slot.
661    fn call(
662        &mut self,
663        llty: Self::FunctionSignature,
664        caller_attrs: Option<&CodegenFnAttrs>,
665        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
666        fn_val: Self::Value,
667        return_slot: ReturnSlot<Self::Value>,
668        args: &[Self::Value],
669        funclet: Option<&Self::Funclet>,
670        callee_instance: Option<Instance<'tcx>>,
671    ) -> Self::Value;
672
673    fn tail_call(
674        &mut self,
675        llty: Self::FunctionSignature,
676        caller_attrs: Option<&CodegenFnAttrs>,
677        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
678        llfn: Self::Value,
679        return_slot: ReturnSlot<Self::Value>,
680        args: &[Self::Value],
681        funclet: Option<&Self::Funclet>,
682        callee_instance: Option<Instance<'tcx>>,
683    );
684
685    fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
686
687    fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value);
688}
rustc_codegen_ssa/traits/builder.rs

rustc_codegen_ssa/traits/
builder.rs
