Skip to main content

rustc_codegen_llvm/builder/
gpu_offload.rs

1use std::ffi::CString;
2
3use bitflags::Flags;
4use llvm::Linkage::*;
5use rustc_abi::Align;
6use rustc_codegen_ssa::MemFlags;
7use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
8use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot};
9use rustc_middle::ty::offload_meta::{MappingFlags, OffloadMetadata, OffloadSize};
10use rustc_span::bug;
11
12use crate::attributes;
13use crate::builder::Builder;
14use crate::common::CodegenCx;
15use crate::context::SimpleCx;
16use crate::llvm::AttributePlace::Function;
17use crate::llvm::{self, Linkage, Type, TypeKind, Value};
18
19// LLVM kernel-independent globals required for offloading
20pub(crate) struct OffloadGlobals<'ll> {
21    pub launcher_fn: &'ll llvm::Value,
22    pub launcher_ty: &'ll llvm::Type,
23
24    pub kernel_args_ty: &'ll llvm::Type,
25
26    pub offload_entry_ty: &'ll llvm::Type,
27
28    pub begin_mapper: &'ll llvm::Value,
29    pub end_mapper: &'ll llvm::Value,
30    pub mapper_fn_ty: &'ll llvm::Type,
31
32    pub ident_t_global: &'ll llvm::Value,
33}
34
35impl<'ll> OffloadGlobals<'ll> {
36    pub(crate) fn declare(cx: &CodegenCx<'ll, '_>) -> Self {
37        let (launcher_fn, launcher_ty) = generate_launcher(cx);
38        let kernel_args_ty = KernelArgsTy::new_decl(cx);
39        let offload_entry_ty = TgtOffloadEntry::new_decl(cx);
40        let (begin_mapper, _, end_mapper, mapper_fn_ty) = gen_tgt_data_mappers(cx);
41        let ident_t_global = generate_at_one(cx);
42
43        // We want LLVM's openmp-opt pass to pick up and optimize this module, since it covers both
44        // openmp and offload optimizations.
45        llvm::add_module_flag_u32(cx.llmod(), llvm::ModuleFlagMergeBehavior::Max, "openmp", 51);
46
47        OffloadGlobals {
48            launcher_fn,
49            launcher_ty,
50            kernel_args_ty,
51            offload_entry_ty,
52            begin_mapper,
53            end_mapper,
54            mapper_fn_ty,
55            ident_t_global,
56        }
57    }
58}
59
60pub(crate) struct OffloadKernelDims<'ll> {
61    num_workgroups: &'ll Value,
62    threads_per_block: &'ll Value,
63    workgroup_dims: &'ll Value,
64    thread_dims: &'ll Value,
65}
66
67impl<'ll> OffloadKernelDims<'ll> {
68    pub(crate) fn from_operands<'tcx>(
69        builder: &mut Builder<'_, 'll, 'tcx>,
70        workgroup_op: &OperandRef<'tcx, &'ll llvm::Value>,
71        thread_op: &OperandRef<'tcx, &'ll llvm::Value>,
72    ) -> Self {
73        let cx = builder.cx;
74        let arr_ty = cx.type_array(cx.type_i32(), 3);
75        let four = Align::from_bytes(4).unwrap();
76
77        let OperandValue::Ref(place) = workgroup_op.val else {
78            ::rustc_span::macros::bug_impl(None,
    format_args!("expected array operand by reference"), Location::caller());bug!("expected array operand by reference");
79        };
80        let workgroup_val = builder.load(arr_ty, place.llval, four);
81
82        let OperandValue::Ref(place) = thread_op.val else {
83            ::rustc_span::macros::bug_impl(None,
    format_args!("expected array operand by reference"), Location::caller());bug!("expected array operand by reference");
84        };
85        let thread_val = builder.load(arr_ty, place.llval, four);
86
87        fn mul_dim3<'ll, 'tcx>(
88            builder: &mut Builder<'_, 'll, 'tcx>,
89            arr: &'ll Value,
90        ) -> &'ll Value {
91            let x = builder.extract_value(arr, 0);
92            let y = builder.extract_value(arr, 1);
93            let z = builder.extract_value(arr, 2);
94
95            let xy = builder.mul(x, y);
96            builder.mul(xy, z)
97        }
98
99        let num_workgroups = mul_dim3(builder, workgroup_val);
100        let threads_per_block = mul_dim3(builder, thread_val);
101
102        OffloadKernelDims {
103            workgroup_dims: workgroup_val,
104            thread_dims: thread_val,
105            num_workgroups,
106            threads_per_block,
107        }
108    }
109}
110
111// ; Function Attrs: nounwind
112// declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) #2
113fn generate_launcher<'ll>(cx: &CodegenCx<'ll, '_>) -> (&'ll llvm::Value, &'ll llvm::Type) {
114    let tptr = cx.type_ptr();
115    let ti64 = cx.type_i64();
116    let ti32 = cx.type_i32();
117    let args = ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [tptr, ti64, ti32, ti32, tptr, tptr]))vec![tptr, ti64, ti32, ti32, tptr, tptr];
118    let tgt_fn_ty = cx.type_func(&args, ti32);
119    let name = "__tgt_target_kernel";
120    let tgt_decl = declare_offload_fn(&cx, name, tgt_fn_ty);
121    let nounwind = llvm::AttributeKind::NoUnwind.create_attr(cx.llcx);
122    attributes::apply_to_llfn(tgt_decl, Function, &[nounwind]);
123    (tgt_decl, tgt_fn_ty)
124}
125
126/// Declares the `omp_get_num_devices` runtime function and returns the
127/// declaration together with its type.
128pub(crate) fn declare_omp_get_num_devices<'ll>(
129    cx: &CodegenCx<'ll, '_>,
130) -> (&'ll llvm::Value, &'ll llvm::Type) {
131    let ti32 = cx.type_i32();
132    let tgt_fn_ty = cx.type_func(&[], ti32);
133    let name = "omp_get_num_devices";
134    let tgt_decl = declare_offload_fn(&cx, name, tgt_fn_ty);
135    let nounwind = llvm::AttributeKind::NoUnwind.create_attr(cx.llcx);
136    attributes::apply_to_llfn(tgt_decl, Function, &[nounwind]);
137    (tgt_decl, tgt_fn_ty)
138}
139
140// What is our @1 here? A magic global, used in our data_{begin/update/end}_mapper:
141// @0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1
142// @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8
143// FIXME(offload): @0 should include the file name (e.g. lib.rs) in which the function to be
144// offloaded was defined.
145pub(crate) fn generate_at_one<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Value {
146    let unknown_txt = ";unknown;unknown;0;0;;";
147    let c_entry_name = CString::new(unknown_txt).unwrap();
148    let c_val = c_entry_name.as_bytes_with_nul();
149    let initializer = crate::common::bytes_in_context(cx.llcx, c_val);
150    let at_zero = add_unnamed_global(&cx, &"", initializer, PrivateLinkage);
151    llvm::set_alignment(at_zero, Align::ONE);
152
153    // @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8
154    let struct_ident_ty = cx.type_named_struct("struct.ident_t");
155    let struct_elems = ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [cx.get_const_i32(0), cx.get_const_i32(2), cx.get_const_i32(0),
                cx.get_const_i32(22), at_zero]))vec![
156        cx.get_const_i32(0),
157        cx.get_const_i32(2),
158        cx.get_const_i32(0),
159        cx.get_const_i32(22),
160        at_zero,
161    ];
162    let struct_elems_ty: Vec<_> = struct_elems.iter().map(|&x| cx.val_ty(x)).collect();
163    let initializer = crate::common::named_struct(struct_ident_ty, &struct_elems);
164    cx.set_struct_body(struct_ident_ty, &struct_elems_ty, false);
165    let at_one = add_unnamed_global(&cx, &"", initializer, PrivateLinkage);
166    llvm::set_alignment(at_one, Align::EIGHT);
167    at_one
168}
169
170pub(crate) struct TgtOffloadEntry {
171    //   uint64_t Reserved;
172    //   uint16_t Version;
173    //   uint16_t Kind;
174    //   uint32_t Flags; Flags associated with the entry (see Target Region Entry Flags)
175    //   void *Address; Address of global symbol within device image (function or global)
176    //   char *SymbolName;
177    //   uint64_t Size; Size of the entry info (0 if it is a function)
178    //   uint64_t Data;
179    //   void *AuxAddr;
180}
181
182impl TgtOffloadEntry {
183    pub(crate) fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Type {
184        let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry");
185        let tptr = cx.type_ptr();
186        let ti64 = cx.type_i64();
187        let ti32 = cx.type_i32();
188        let ti16 = cx.type_i16();
189        // For each kernel to run on the gpu, we will later generate one entry of this type.
190        // copied from LLVM
191        let entry_elements = ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [ti64, ti16, ti16, ti32, tptr, tptr, ti64, ti64, tptr]))vec![ti64, ti16, ti16, ti32, tptr, tptr, ti64, ti64, tptr];
192        cx.set_struct_body(offload_entry_ty, &entry_elements, false);
193        offload_entry_ty
194    }
195
196    fn new<'ll>(
197        cx: &CodegenCx<'ll, '_>,
198        region_id: &'ll Value,
199        llglobal: &'ll Value,
200    ) -> [&'ll Value; 9] {
201        let reserved = cx.get_const_i64(0);
202        let version = cx.get_const_i16(1);
203        let kind = cx.get_const_i16(1);
204        let flags = cx.get_const_i32(0);
205        let size = cx.get_const_i64(0);
206        let data = cx.get_const_i64(0);
207        let aux_addr = cx.const_null(cx.type_ptr());
208        [reserved, version, kind, flags, region_id, llglobal, size, data, aux_addr]
209    }
210}
211
212// Taken from the LLVM APITypes.h declaration:
213struct KernelArgsTy {
214    //  uint32_t Version = 0; // Version of this struct for ABI compatibility.
215    //  uint32_t NumArgs = 0; // Number of arguments in each input pointer.
216    //  void **ArgBasePtrs =
217    //      nullptr;                 // Base pointer of each argument (e.g. a struct).
218    //  void **ArgPtrs = nullptr;    // Pointer to the argument data.
219    //  int64_t *ArgSizes = nullptr; // Size of the argument data in bytes.
220    //  int64_t *ArgTypes = nullptr; // Type of the data (e.g. to / from).
221    //  void **ArgNames = nullptr;   // Name of the data for debugging, possibly null.
222    //  void **ArgMappers = nullptr; // User-defined mappers, possibly null.
223    //  uint64_t Tripcount =
224    // 0; // Tripcount for the teams / distribute loop, 0 otherwise.
225    // struct {
226    //    uint64_t NoWait : 1; // Was this kernel spawned with a `nowait` clause.
227    //    uint64_t IsCUDA : 1; // Was this kernel spawned via CUDA.
228    //    uint64_t Unused : 62;
229    //  } Flags = {0, 0, 0}; // totals to 64 Bit, 8 Byte
230    //  // The number of teams (for x,y,z dimension).
231    //  uint32_t NumTeams[3] = {0, 0, 0};
232    //  // The number of threads (for x,y,z dimension).
233    //  uint32_t ThreadLimit[3] = {0, 0, 0};
234    //  uint32_t DynCGroupMem = 0; // Amount of dynamic cgroup memory requested.
235}
236
237impl KernelArgsTy {
238    const OFFLOAD_VERSION: u64 = 3;
239    const FLAGS: u64 = 1 << 6; // Enable StrictBlocksAndThreads
240    const TRIPCOUNT: u64 = 0;
241    fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Type {
242        let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments");
243        let tptr = cx.type_ptr();
244        let ti64 = cx.type_i64();
245        let ti32 = cx.type_i32();
246        let tarr = cx.type_array(ti32, 3);
247
248        let kernel_elements =
249            ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [ti32, ti32, tptr, tptr, tptr, tptr, tptr, tptr, ti64, ti64, tarr,
                tarr, ti32]))vec![ti32, ti32, tptr, tptr, tptr, tptr, tptr, tptr, ti64, ti64, tarr, tarr, ti32];
250
251        cx.set_struct_body(kernel_arguments_ty, &kernel_elements, false);
252        kernel_arguments_ty
253    }
254
255    fn new<'ll, 'tcx>(
256        cx: &CodegenCx<'ll, 'tcx>,
257        num_args: u64,
258        memtransfer_types: &'ll Value,
259        geps: [&'ll Value; 3],
260        workgroup_dims: &'ll Value,
261        thread_dims: &'ll Value,
262        dyn_cache: &'ll Value,
263    ) -> [(Align, &'ll str, &'ll Value); 13] {
264        let four = Align::from_bytes(4).expect("4 Byte alignment should work");
265        let eight = Align::EIGHT;
266
267        [
268            (four, "Version", cx.get_const_i32(KernelArgsTy::OFFLOAD_VERSION)),
269            (four, "NumArgs", cx.get_const_i32(num_args)),
270            (eight, "ArgBasePtrs", geps[0]),
271            (eight, "ArgPtrs", geps[1]),
272            (eight, "ArgSizes", geps[2]),
273            (eight, "ArgTypes", memtransfer_types),
274            // The next two are debug infos. FIXME(offload): set them
275            (eight, "ArgNames", cx.const_null(cx.type_ptr())), // dbg
276            (eight, "ArgMappers", cx.const_null(cx.type_ptr())), // dbg
277            (eight, "Tripcount", cx.get_const_i64(KernelArgsTy::TRIPCOUNT)),
278            (eight, "Flags", cx.get_const_i64(KernelArgsTy::FLAGS)),
279            (four, "NumTeams", workgroup_dims),
280            (four, "ThreadLimit", thread_dims),
281            (four, "DynCGroupMem", dyn_cache),
282        ]
283    }
284}
285
286// Contains LLVM values needed to manage offloading for a single kernel.
287#[derive(#[automatically_derived]impl<'ll> ::core::marker::Copy for
    OffloadKernelGlobals<'ll> {
}Copy, #[automatically_derived]#[doc(hidden)]unsafe impl<'ll>
    ::core::clone::TrivialClone for OffloadKernelGlobals<'ll> {
}
#[automatically_derived]impl<'ll> ::core::clone::Clone for
    OffloadKernelGlobals<'ll> {
    #[inline]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<&'ll llvm::Value>;
        let _: ::core::clone::AssertParamIsClone<&'ll llvm::Value>;
        let _: ::core::clone::AssertParamIsClone<&'ll llvm::Value>;
        let _: ::core::clone::AssertParamIsClone<&'ll llvm::Value>;
        let _: ::core::clone::AssertParamIsClone<&'ll llvm::Value>;
        *self
    }
}Clone)]
288pub(crate) struct OffloadKernelGlobals<'ll> {
289    pub offload_sizes: &'ll llvm::Value,
290    pub memtransfer_begin: &'ll llvm::Value,
291    pub memtransfer_kernel: &'ll llvm::Value,
292    pub memtransfer_end: &'ll llvm::Value,
293    pub region_id: &'ll llvm::Value,
294}
295
296fn gen_tgt_data_mappers<'ll>(
297    cx: &CodegenCx<'ll, '_>,
298) -> (&'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Type) {
299    let tptr = cx.type_ptr();
300    let ti64 = cx.type_i64();
301    let ti32 = cx.type_i32();
302
303    let args = ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [tptr, ti64, ti32, tptr, tptr, tptr, tptr, tptr, tptr]))vec![tptr, ti64, ti32, tptr, tptr, tptr, tptr, tptr, tptr];
304    let mapper_fn_ty = cx.type_func(&args, cx.type_void());
305    let mapper_begin = "__tgt_target_data_begin_mapper";
306    let mapper_update = "__tgt_target_data_update_mapper";
307    let mapper_end = "__tgt_target_data_end_mapper";
308    let begin_mapper_decl = declare_offload_fn(&cx, mapper_begin, mapper_fn_ty);
309    let update_mapper_decl = declare_offload_fn(&cx, mapper_update, mapper_fn_ty);
310    let end_mapper_decl = declare_offload_fn(&cx, mapper_end, mapper_fn_ty);
311
312    let nounwind = llvm::AttributeKind::NoUnwind.create_attr(cx.llcx);
313    attributes::apply_to_llfn(begin_mapper_decl, Function, &[nounwind]);
314    attributes::apply_to_llfn(update_mapper_decl, Function, &[nounwind]);
315    attributes::apply_to_llfn(end_mapper_decl, Function, &[nounwind]);
316
317    (begin_mapper_decl, update_mapper_decl, end_mapper_decl, mapper_fn_ty)
318}
319
320fn add_priv_unnamed_arr<'ll>(cx: &SimpleCx<'ll>, name: &str, vals: &[u64]) -> &'ll llvm::Value {
321    let ti64 = cx.type_i64();
322    let mut size_val = Vec::with_capacity(vals.len());
323    for &val in vals {
324        size_val.push(cx.get_const_i64(val));
325    }
326    let initializer = cx.const_array(ti64, &size_val);
327    add_unnamed_global(cx, name, initializer, PrivateLinkage)
328}
329
330pub(crate) fn add_unnamed_global<'ll>(
331    cx: &SimpleCx<'ll>,
332    name: &str,
333    initializer: &'ll llvm::Value,
334    l: Linkage,
335) -> &'ll llvm::Value {
336    let llglobal = add_global(cx, name, initializer, l);
337    llvm::LLVMSetUnnamedAddress(llglobal, llvm::UnnamedAddr::Global);
338    llglobal
339}
340
341pub(crate) fn add_global<'ll>(
342    cx: &SimpleCx<'ll>,
343    name: &str,
344    initializer: &'ll llvm::Value,
345    l: Linkage,
346) -> &'ll llvm::Value {
347    let c_name = CString::new(name).unwrap();
348    let llglobal: &'ll llvm::Value = llvm::add_global(cx.llmod, cx.val_ty(initializer), &c_name);
349    llvm::set_global_constant(llglobal, true);
350    llvm::set_linkage(llglobal, l);
351    llvm::set_initializer(llglobal, initializer);
352    llglobal
353}
354
355// This function returns a memtransfer value which encodes how arguments to this kernel shall be
356// mapped to/from the gpu. It also returns a region_id with the name of this kernel, to be
357// concatenated into the list of region_ids.
358pub(crate) fn gen_define_handling<'ll>(
359    cx: &CodegenCx<'ll, '_>,
360    metadata: &[OffloadMetadata],
361    symbol: String,
362    offload_globals: &OffloadGlobals<'ll>,
363) -> OffloadKernelGlobals<'ll> {
364    if let Some(entry) = cx.offload_kernel_cache.borrow().get(&symbol) {
365        return *entry;
366    }
367
368    let offload_entry_ty = offload_globals.offload_entry_ty;
369
370    let (sizes, transfer): (Vec<_>, Vec<_>) =
371        metadata.iter().map(|m| (m.payload_size, m.mode)).unzip();
372    // Our begin mapper should only see simplified information about which args have to be
373    // transferred to the device, the end mapper only about which args should be transferred back.
374    // Any information beyond that makes it harder for LLVM's opt pass to evaluate whether it can
375    // safely move (=optimize) the LLVM-IR location of this data transfer. Only the mapping types
376    // mentioned below are handled, so make sure that we don't generate any other ones.
377    let handled_mappings = MappingFlags::TO
378        | MappingFlags::FROM
379        | MappingFlags::TARGET_PARAM
380        | MappingFlags::LITERAL
381        | MappingFlags::IMPLICIT;
382    for arg in &transfer {
383        if true {
    if !!arg.contains_unknown_bits() {
        ::core::panicking::panic("assertion failed: !arg.contains_unknown_bits()")
    };
};debug_assert!(!arg.contains_unknown_bits());
384        if true {
    if !handled_mappings.contains(*arg) {
        ::core::panicking::panic("assertion failed: handled_mappings.contains(*arg)")
    };
};debug_assert!(handled_mappings.contains(*arg));
385    }
386
387    let valid_begin_mappings = MappingFlags::TO | MappingFlags::LITERAL | MappingFlags::IMPLICIT;
388    let transfer_to: Vec<u64> =
389        transfer.iter().map(|m| m.intersection(valid_begin_mappings).bits()).collect();
390    let transfer_from: Vec<u64> =
391        transfer.iter().map(|m| m.intersection(MappingFlags::FROM).bits()).collect();
392    let valid_kernel_mappings = MappingFlags::LITERAL | MappingFlags::IMPLICIT;
393    // FIXME(offload): add `OMP_MAP_TARGET_PARAM = 0x20` only if necessary
394    let transfer_kernel: Vec<u64> = transfer
395        .iter()
396        .map(|m| (m.intersection(valid_kernel_mappings) | MappingFlags::TARGET_PARAM).bits())
397        .collect();
398
399    let actual_sizes = sizes
400        .iter()
401        .map(|s| match s {
402            OffloadSize::Static(sz) => *sz,
403            // NOTE(Sa4dUs): set `.offload_sizes` entry to 0 for sizes that we determine at runtime, just like clang
404            _ => 0,
405        })
406        .collect::<Vec<_>>();
407    let offload_sizes =
408        add_priv_unnamed_arr(&cx, &::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".offload_sizes.{0}", symbol))
    })format!(".offload_sizes.{symbol}"), &actual_sizes);
409    let memtransfer_begin =
410        add_priv_unnamed_arr(&cx, &::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".offload_maptypes.{0}.begin",
                symbol))
    })format!(".offload_maptypes.{symbol}.begin"), &transfer_to);
411    let memtransfer_kernel =
412        add_priv_unnamed_arr(&cx, &::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".offload_maptypes.{0}.kernel",
                symbol))
    })format!(".offload_maptypes.{symbol}.kernel"), &transfer_kernel);
413    let memtransfer_end =
414        add_priv_unnamed_arr(&cx, &::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".offload_maptypes.{0}.end",
                symbol))
    })format!(".offload_maptypes.{symbol}.end"), &transfer_from);
415
416    // Next: For each function, generate these three entries. A weak constant,
417    // the llvm.rodata entry name, and  the llvm_offload_entries value
418
419    let name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".{0}.region_id", symbol))
    })format!(".{symbol}.region_id");
420    let initializer = cx.get_const_i8(0);
421    let region_id = add_global(&cx, &name, initializer, WeakAnyLinkage);
422
423    let c_entry_name = CString::new(symbol.clone()).unwrap();
424    let c_val = c_entry_name.as_bytes_with_nul();
425    let offload_entry_name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".offloading.entry_name.{0}",
                symbol))
    })format!(".offloading.entry_name.{symbol}");
426
427    let initializer = crate::common::bytes_in_context(cx.llcx, c_val);
428    let llglobal = add_unnamed_global(&cx, &offload_entry_name, initializer, InternalLinkage);
429    llvm::set_alignment(llglobal, Align::ONE);
430    llvm::set_section(llglobal, c".llvm.rodata.offloading");
431
432    let name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!(".offloading.entry.{0}", symbol))
    })format!(".offloading.entry.{symbol}");
433
434    // See the __tgt_offload_entry documentation above.
435    let elems = TgtOffloadEntry::new(&cx, region_id, llglobal);
436
437    let initializer = crate::common::named_struct(offload_entry_ty, &elems);
438    let c_name = CString::new(name).unwrap();
439    let offload_entry = llvm::add_global(cx.llmod, offload_entry_ty, &c_name);
440    llvm::set_global_constant(offload_entry, true);
441    llvm::set_linkage(offload_entry, WeakAnyLinkage);
442    llvm::set_initializer(offload_entry, initializer);
443    llvm::set_alignment(offload_entry, Align::EIGHT);
444    let c_section_name = CString::new("llvm_offload_entries").unwrap();
445    llvm::set_section(offload_entry, &c_section_name);
446
447    cx.add_compiler_used_global(offload_entry);
448
449    let result = OffloadKernelGlobals {
450        offload_sizes,
451        memtransfer_begin,
452        memtransfer_kernel,
453        memtransfer_end,
454        region_id,
455    };
456
457    cx.offload_kernel_cache.borrow_mut().insert(symbol, result);
458
459    result
460}
461
462fn declare_offload_fn<'ll>(
463    cx: &CodegenCx<'ll, '_>,
464    name: &str,
465    ty: &'ll llvm::Type,
466) -> &'ll llvm::Value {
467    crate::declare::declare_simple_fn(
468        cx,
469        name,
470        llvm::CallConv::CCallConv,
471        llvm::UnnamedAddr::No,
472        llvm::Visibility::Default,
473        ty,
474    )
475}
476
477pub(crate) fn scalar_width<'ll>(cx: &SimpleCx<'ll>, ty: &'ll Type) -> u64 {
478    if let Some(float) = cx.float_type(ty) {
479        float.size().bits()
480    } else if let Some(width) = cx.int_width(ty) {
481        width
482    } else if cx.type_kind(ty) == TypeKind::Pointer {
483        ::rustc_span::macros::bug_impl(None,
    format_args!("scalar_width doesn\'t support pointer type {0:?}", ty),
    Location::caller())bug!("scalar_width doesn't support pointer type {ty:?}")
484    } else {
485        ::rustc_span::macros::bug_impl(None,
    format_args!("scalar_width was called on a non scalar type {0:?}", ty),
    Location::caller())bug!("scalar_width was called on a non scalar type {ty:?}")
486    }
487}
488
489fn get_runtime_size<'ll, 'tcx>(
490    builder: &mut Builder<'_, 'll, 'tcx>,
491    args: &[&'ll Value],
492    index: usize,
493    meta: &OffloadMetadata,
494) -> &'ll Value {
495    match meta.payload_size {
496        OffloadSize::Slice { element_size } => {
497            let length_idx = index + 1;
498            let length = args[length_idx];
499            let length_i64 = builder.intcast(length, builder.cx.type_i64(), false);
500            builder.mul(length_i64, builder.cx.get_const_i64(element_size))
501        }
502        _ => ::rustc_span::macros::bug_impl(None,
    format_args!("unexpected offload size {0:?}", meta.payload_size),
    Location::caller())bug!("unexpected offload size {:?}", meta.payload_size),
503    }
504}
505
506// For each kernel *call*, we now use some of our previous declared globals to move data to and from
507// the gpu. For now, we only handle the data transfer part of it.
508// If two consecutive kernels use the same memory, we still move it to the host and back to the gpu.
509// Since in our frontend users (by default) don't have to specify data transfer, this is something
510// we should optimize in the future! In some cases we can directly zero-allocate on the device and
511// only move data back, or if something is immutable, we might only copy it to the device.
512//
513// Current steps:
514// 0. Alloca some variables for the following steps
515// 1. set insert point before kernel call.
516// 2. generate all the GEPS and stores, to be used in 3)
517// 3. generate __tgt_target_data_begin calls to move data to the GPU
518//
519// unchanged: keep kernel call. Later move the kernel to the GPU
520//
521// 4. set insert point after kernel call.
522// 5. generate all the GEPS and stores, to be used in 6)
523// 6. generate __tgt_target_data_end calls to move data from the GPU
524pub(crate) fn gen_call_handling<'ll, 'tcx>(
525    builder: &mut Builder<'_, 'll, 'tcx>,
526    offload_data: &OffloadKernelGlobals<'ll>,
527    args: &[&'ll Value],
528    types: &[&Type],
529    metadata: &[OffloadMetadata],
530    offload_globals: &OffloadGlobals<'ll>,
531    offload_dims: &OffloadKernelDims<'ll>,
532    dyn_cache: &'ll Value,
533    device_id: &'ll Value,
534) {
535    let cx = builder.cx;
536    let OffloadKernelGlobals {
537        offload_sizes,
538        memtransfer_begin,
539        memtransfer_kernel,
540        memtransfer_end,
541        region_id,
542    } = offload_data;
543    let OffloadKernelDims { num_workgroups, threads_per_block, workgroup_dims, thread_dims } =
544        offload_dims;
545
546    let has_dynamic = metadata.iter().any(|m| !#[allow(non_exhaustive_omitted_patterns)] match m.payload_size {
    OffloadSize::Static(_) => true,
    _ => false,
}matches!(m.payload_size, OffloadSize::Static(_)));
547
548    let tgt_decl = offload_globals.launcher_fn;
549    let tgt_target_kernel_ty = offload_globals.launcher_ty;
550
551    let tgt_kernel_decl = offload_globals.kernel_args_ty;
552    let begin_mapper_decl = offload_globals.begin_mapper;
553    let end_mapper_decl = offload_globals.end_mapper;
554    let fn_ty = offload_globals.mapper_fn_ty;
555
556    let num_args = types.len() as u64;
557    let bb = builder.llbb();
558
559    // Step 0)
560    unsafe {
561        llvm::LLVMRustPositionBuilderPastAllocas(builder.llbuilder(), builder.llfn());
562    }
563
564    let ty = cx.type_array(cx.type_ptr(), num_args);
565    // Baseptr are just the input pointer to the kernel, stored in a local alloca
566    let a1 = builder.direct_alloca(ty, Align::EIGHT, ".offload_baseptrs");
567    // Ptrs are the result of a gep into the baseptr, at least for our trivial types.
568    let a2 = builder.direct_alloca(ty, Align::EIGHT, ".offload_ptrs");
569    // These represent the sizes in bytes, e.g. the entry for `&[f64; 16]` will be 8*16.
570    let ty2 = cx.type_array(cx.type_i64(), num_args);
571
572    let a4 = if has_dynamic {
573        let alloc = builder.direct_alloca(ty2, Align::EIGHT, ".offload_sizes");
574
575        builder.memcpy(
576            alloc,
577            Align::EIGHT,
578            offload_sizes,
579            Align::EIGHT,
580            cx.get_const_i64(8 * args.len() as u64),
581            MemFlags::empty(),
582            None,
583        );
584
585        alloc
586    } else {
587        offload_sizes
588    };
589
590    //%kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
591    let a5 = builder.direct_alloca(tgt_kernel_decl, Align::EIGHT, "kernel_args");
592
593    // Step 1)
594    unsafe {
595        llvm::LLVMPositionBuilderAtEnd(builder.llbuilder(), bb);
596    }
597
598    // Now we allocate once per function param, a copy to be passed to one of our maps.
599    let mut vals = ::alloc::vec::Vec::new()vec![];
600    let mut geps = ::alloc::vec::Vec::new()vec![];
601    let i32_0 = cx.get_const_i32(0);
602    for &v in args {
603        let ty = cx.val_ty(v);
604        let ty_kind = cx.type_kind(ty);
605        let (base_val, gep_base) = match ty_kind {
606            TypeKind::Pointer => (v, v),
607            TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::Integer => {
608                // FIXME(Sa4dUs): check for `f128` support, latest NVIDIA cards support it
609                let num_bits = scalar_width(cx, ty);
610
611                let bb = builder.llbb();
612                unsafe {
613                    llvm::LLVMRustPositionBuilderPastAllocas(builder.llbuilder(), builder.llfn());
614                }
615                let addr = builder.direct_alloca(cx.type_i64(), Align::EIGHT, "addr");
616                unsafe {
617                    llvm::LLVMPositionBuilderAtEnd(builder.llbuilder(), bb);
618                }
619
620                let cast = builder.bitcast(v, cx.type_ix(num_bits));
621                let value = builder.zext(cast, cx.type_i64());
622                builder.store(value, addr, Align::EIGHT);
623                (value, addr)
624            }
625            other => ::rustc_span::macros::bug_impl(None,
    format_args!("offload does not support {0:?}", other), Location::caller())bug!("offload does not support {other:?}"),
626        };
627
628        let gep = builder.typed_inbounds_gep(cx.type_f32(), gep_base, &[i32_0]);
629
630        vals.push(base_val);
631        geps.push(gep);
632    }
633
634    for i in 0..num_args {
635        let idx = cx.get_const_i32(i);
636        let gep1 = builder.typed_inbounds_gep(ty, a1, &[i32_0, idx]);
637        builder.store(vals[i as usize], gep1, Align::EIGHT);
638        let gep2 = builder.typed_inbounds_gep(ty, a2, &[i32_0, idx]);
639        builder.store(geps[i as usize], gep2, Align::EIGHT);
640
641        if !#[allow(non_exhaustive_omitted_patterns)] match metadata[i as
                usize].payload_size {
    OffloadSize::Static(_) => true,
    _ => false,
}matches!(metadata[i as usize].payload_size, OffloadSize::Static(_)) {
642            let gep3 = builder.typed_inbounds_gep(ty2, a4, &[i32_0, idx]);
643            let size_val = get_runtime_size(builder, args, i as usize, &metadata[i as usize]);
644            builder.store(size_val, gep3, Align::EIGHT);
645        }
646    }
647
648    // For now we have a very simplistic indexing scheme into our
649    // offload_{baseptrs,ptrs,sizes}. We will probably improve this along with our gpu frontend pr.
650    fn get_geps<'ll, 'tcx>(
651        builder: &mut Builder<'_, 'll, 'tcx>,
652        ty: &'ll Type,
653        ty2: &'ll Type,
654        a1: &'ll Value,
655        a2: &'ll Value,
656        a4: &'ll Value,
657        is_dynamic: bool,
658    ) -> [&'ll Value; 3] {
659        let cx = builder.cx;
660        let i32_0 = cx.get_const_i32(0);
661
662        let gep1 = builder.typed_inbounds_gep(ty, a1, &[i32_0, i32_0]);
663        let gep2 = builder.typed_inbounds_gep(ty, a2, &[i32_0, i32_0]);
664        let gep3 =
665            if is_dynamic { builder.typed_inbounds_gep(ty2, a4, &[i32_0, i32_0]) } else { a4 };
666        [gep1, gep2, gep3]
667    }
668
669    fn generate_mapper_call<'ll, 'tcx>(
670        builder: &mut Builder<'_, 'll, 'tcx>,
671        geps: [&'ll Value; 3],
672        o_type: &'ll Value,
673        fn_to_call: &'ll Value,
674        fn_ty: &'ll Type,
675        num_args: u64,
676        s_ident_t: &'ll Value,
677    ) {
678        let cx = builder.cx;
679        let nullptr = cx.const_null(cx.type_ptr());
680        let i64_max = cx.get_const_i64(u64::MAX);
681        let num_args = cx.get_const_i32(num_args);
682        let args =
683            ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type,
                nullptr, nullptr]))vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr];
684        builder.call(fn_ty, None, None, fn_to_call, ReturnSlot::Direct, &args, None, None);
685    }
686
687    // Step 2)
688    let s_ident_t = offload_globals.ident_t_global;
689    let geps = get_geps(builder, ty, ty2, a1, a2, a4, has_dynamic);
690    generate_mapper_call(
691        builder,
692        geps,
693        memtransfer_begin,
694        begin_mapper_decl,
695        fn_ty,
696        num_args,
697        s_ident_t,
698    );
699    let values = KernelArgsTy::new(
700        &cx,
701        num_args,
702        memtransfer_kernel,
703        geps,
704        workgroup_dims,
705        thread_dims,
706        dyn_cache,
707    );
708
709    // Step 3)
710    // Here we fill the KernelArgsTy, see the documentation above
711    for (i, value) in values.iter().enumerate() {
712        let ptr =
713            builder.typed_inbounds_gep(tgt_kernel_decl, a5, &[i32_0, cx.get_const_i32(i as u64)]);
714        let name = std::ffi::CString::new(value.1).unwrap();
715        llvm::set_value_name(ptr, &name.as_bytes());
716
717        builder.store(value.2, ptr, value.0);
718    }
719
720    let device_id = builder.sext(device_id, cx.type_i64());
721    let args = ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [s_ident_t, device_id, num_workgroups, threads_per_block, region_id,
                a5]))vec![s_ident_t, device_id, num_workgroups, threads_per_block, region_id, a5];
722    builder.call(tgt_target_kernel_ty, None, None, tgt_decl, ReturnSlot::Direct, &args, None, None);
723    // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args)
724
725    // Step 4)
726    let geps = get_geps(builder, ty, ty2, a1, a2, a4, has_dynamic);
727    generate_mapper_call(
728        builder,
729        geps,
730        memtransfer_end,
731        end_mapper_decl,
732        fn_ty,
733        num_args,
734        s_ident_t,
735    );
736}