1use rustc_middle::bug;
2use rustc_middle::ty::{self, GenericArgKind, Ty, TyCtxt};
3use rustc_session::config::Lto;
4use rustc_symbol_mangling::typeid_for_trait_ref;
5use rustc_target::callconv::FnAbi;
6use tracing::{debug, instrument};
7
8use crate::traits::*;
9
10#[derive(Copy, Clone, Debug)]
11pub(crate) struct VirtualIndex(u64);
12
13impl<'a, 'tcx> VirtualIndex {
14 pub(crate) fn from_index(index: usize) -> Self {
15 VirtualIndex(index as u64)
16 }
17
18 fn get_fn_inner<Bx: BuilderMethods<'a, 'tcx>>(
19 self,
20 bx: &mut Bx,
21 llvtable: Bx::Value,
22 ty: Ty<'tcx>,
23 fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
24 nonnull: bool,
25 ) -> Bx::Value {
26 debug!("get_fn({llvtable:?}, {ty:?}, {self:?})");
28
29 let llty = bx.fn_ptr_backend_type(fn_abi);
30 let ptr_size = bx.data_layout().pointer_size;
31 let vtable_byte_offset = self.0 * ptr_size.bytes();
32
33 load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, nonnull)
34 }
35
36 pub(crate) fn get_optional_fn<Bx: BuilderMethods<'a, 'tcx>>(
37 self,
38 bx: &mut Bx,
39 llvtable: Bx::Value,
40 ty: Ty<'tcx>,
41 fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
42 ) -> Bx::Value {
43 self.get_fn_inner(bx, llvtable, ty, fn_abi, false)
44 }
45
46 pub(crate) fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
47 self,
48 bx: &mut Bx,
49 llvtable: Bx::Value,
50 ty: Ty<'tcx>,
51 fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
52 ) -> Bx::Value {
53 self.get_fn_inner(bx, llvtable, ty, fn_abi, true)
54 }
55
56 pub(crate) fn get_usize<Bx: BuilderMethods<'a, 'tcx>>(
57 self,
58 bx: &mut Bx,
59 llvtable: Bx::Value,
60 ty: Ty<'tcx>,
61 ) -> Bx::Value {
62 debug!("get_int({:?}, {:?})", llvtable, self);
64
65 let llty = bx.type_isize();
66 let ptr_size = bx.data_layout().pointer_size;
67 let vtable_byte_offset = self.0 * ptr_size.bytes();
68
69 load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, false)
70 }
71}
72
73fn dyn_trait_in_self<'tcx>(
76 tcx: TyCtxt<'tcx>,
77 ty: Ty<'tcx>,
78) -> Option<ty::ExistentialTraitRef<'tcx>> {
79 for arg in ty.peel_refs().walk() {
80 if let GenericArgKind::Type(ty) = arg.unpack()
81 && let ty::Dynamic(data, _, _) = ty.kind()
82 {
83 return data
86 .principal()
87 .map(|principal| tcx.instantiate_bound_regions_with_erased(principal));
88 }
89 }
90
91 bug!("expected a `dyn Trait` ty, found {ty:?}")
92}
93
94#[instrument(level = "debug", skip(cx))]
103pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
104 cx: &Cx,
105 ty: Ty<'tcx>,
106 trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
107) -> Cx::Value {
108 let tcx = cx.tcx();
109
110 if let Some(&val) = cx.vtables().borrow().get(&(ty, trait_ref)) {
112 return val;
113 }
114
115 let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref));
116 let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory();
117 let vtable_const = cx.const_data_from_alloc(vtable_allocation);
118 let align = cx.data_layout().pointer_align.abi;
119 let vtable = cx.static_addr_of(vtable_const, align, Some("vtable"));
120
121 cx.apply_vcall_visibility_metadata(ty, trait_ref, vtable);
122 cx.create_vtable_debuginfo(ty, trait_ref, vtable);
123 cx.vtables().borrow_mut().insert((ty, trait_ref), vtable);
124 vtable
125}
126
127pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
129 bx: &mut Bx,
130 llvtable: Bx::Value,
131 llty: Bx::Type,
132 vtable_byte_offset: u64,
133 ty: Ty<'tcx>,
134 nonnull: bool,
135) -> Bx::Value {
136 let ptr_align = bx.data_layout().pointer_align.abi;
137
138 if bx.cx().sess().opts.unstable_opts.virtual_function_elimination
139 && bx.cx().sess().lto() == Lto::Fat
140 {
141 if let Some(trait_ref) = dyn_trait_in_self(bx.tcx(), ty) {
142 let typeid = bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), trait_ref)).unwrap();
143 let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid);
144 return func;
145 } else if nonnull {
146 bug!("load nonnull value from a vtable without a principal trait")
147 }
148 }
149
150 let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset));
151 let ptr = bx.load(llty, gep, ptr_align);
152 bx.set_invariant_load(ptr);
154 if nonnull {
155 bx.nonnull_metadata(ptr);
156 }
157 ptr
158}