Skip to main content

rustc_codegen_llvm/llvm/
mod.rs

1#![allow(non_snake_case)]
2
3use std::ffi::{CStr, CString};
4use std::num::NonZero;
5use std::ptr;
6use std::string::FromUtf8Error;
7
8use libc::c_uint;
9use rustc_abi::{AddressSpace, Align, Size, WrappingRange};
10use rustc_llvm::RustString;
11
12pub(crate) use self::CallConv::*;
13pub(crate) use self::CodeGenOptSize::*;
14pub(crate) use self::conversions::*;
15pub(crate) use self::ffi::*;
16pub(crate) use self::metadata_kind::*;
17
18mod conversions;
19pub(crate) mod diagnostic;
20pub(crate) mod enzyme_ffi;
21mod ffi;
22mod metadata_kind;
23pub(crate) mod offload_ffi;
24
25pub(crate) use self::enzyme_ffi::*;
26pub(crate) use self::offload_ffi::*;
27
28impl LLVMRustResult {
29    pub(crate) fn into_result(self) -> Result<(), ()> {
30        match self {
31            LLVMRustResult::Success => Ok(()),
32            LLVMRustResult::Failure => Err(()),
33        }
34    }
35}
36
37pub(crate) fn AddFunctionAttributes<'ll>(
38    llfn: &'ll Value,
39    idx: AttributePlace,
40    attrs: &[&'ll Attribute],
41) {
42    unsafe {
43        LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len());
44    }
45}
46
47pub(crate) fn HasStringAttribute<'ll>(llfn: &'ll Value, name: &str) -> bool {
48    unsafe { LLVMRustHasFnAttribute(llfn, name.as_ptr(), name.len()) }
49}
50
51pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: &str) {
52    unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_ptr(), name.len()) }
53}
54
55pub(crate) fn AddCallSiteAttributes<'ll>(
56    callsite: &'ll Value,
57    idx: AttributePlace,
58    attrs: &[&'ll Attribute],
59) {
60    unsafe {
61        LLVMRustAddCallSiteAttributes(callsite, idx.as_uint(), attrs.as_ptr(), attrs.len());
62    }
63}
64
65pub(crate) fn CreateAttrStringValue<'ll>(
66    llcx: Context<'ll>,
67    attr: &str,
68    value: &str,
69) -> &'ll Attribute {
70    unsafe {
71        LLVMCreateStringAttribute(
72            llcx,
73            attr.as_ptr(),
74            attr.len().try_into().unwrap(),
75            value.as_ptr(),
76            value.len().try_into().unwrap(),
77        )
78    }
79}
80pub(crate) fn CreateAttrStringValueFromCStr<'ll>(
81    llcx: Context<'ll>,
82    attr: &std::ffi::CStr,
83    value: &std::ffi::CStr,
84) -> &'ll Attribute {
85    let attr = attr.to_bytes();
86    let value = value.to_bytes();
87    unsafe {
88        LLVMCreateStringAttribute(
89            llcx,
90            attr.as_ptr(),
91            attr.len() as c_uint,
92            value.as_ptr(),
93            value.len() as c_uint,
94        )
95    }
96}
97
98pub(crate) fn CreateAttrString<'ll>(llcx: Context<'ll>, attr: &str) -> &'ll Attribute {
99    unsafe {
100        LLVMCreateStringAttribute(
101            llcx,
102            attr.as_ptr(),
103            attr.len().try_into().unwrap(),
104            std::ptr::null(),
105            0,
106        )
107    }
108}
109
110pub(crate) fn CreateAlignmentAttr<'ll>(llcx: Context<'ll>, bytes: u64) -> &'ll Attribute {
111    unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) }
112}
113
114pub(crate) fn CreateDereferenceableAttr<'ll>(llcx: Context<'ll>, bytes: u64) -> &'ll Attribute {
115    unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) }
116}
117
118pub(crate) fn CreateDereferenceableOrNullAttr<'ll>(
119    llcx: Context<'ll>,
120    bytes: u64,
121) -> &'ll Attribute {
122    unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) }
123}
124
125pub(crate) fn CreateByValAttr<'ll>(llcx: Context<'ll>, ty: &'ll Type) -> &'ll Attribute {
126    unsafe { LLVMRustCreateByValAttr(llcx, ty) }
127}
128
129pub(crate) fn CreateByRefAttr<'ll>(llcx: Context<'ll>, ty: &'ll Type) -> &'ll Attribute {
130    unsafe { LLVMRustCreateByRefAttr(llcx, ty) }
131}
132
133pub(crate) fn CreateStructRetAttr<'ll>(llcx: Context<'ll>, ty: &'ll Type) -> &'ll Attribute {
134    unsafe { LLVMRustCreateStructRetAttr(llcx, ty) }
135}
136
137pub(crate) fn CreateUWTableAttr<'ll>(llcx: Context<'ll>, async_: bool) -> &'ll Attribute {
138    unsafe { LLVMRustCreateUWTableAttr(llcx, async_) }
139}
140
141pub(crate) fn CreateAllocSizeAttr<'ll>(llcx: Context<'ll>, size_arg: u32) -> &'ll Attribute {
142    unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) }
143}
144
145pub(crate) fn CreateAllocKindAttr<'ll>(
146    llcx: Context<'ll>,
147    kind_arg: AllocKindFlags,
148) -> &'ll Attribute {
149    unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) }
150}
151
152pub(crate) fn CreateRangeAttr<'ll>(
153    llcx: Context<'ll>,
154    size: Size,
155    range: WrappingRange,
156) -> &'ll Attribute {
157    let lower = range.start;
158    // LLVM treats the upper bound as exclusive, but allows wrapping.
159    let upper = range.end.wrapping_add(1);
160
161    // Pass each `u128` endpoint value as a `[u64; 2]` array, least-significant part first.
162    let as_u64_array = |x: u128| [x as u64, (x >> 64) as u64];
163    let lower_words: [u64; 2] = as_u64_array(lower);
164    let upper_words: [u64; 2] = as_u64_array(upper);
165
166    // To ensure that LLVM doesn't try to read beyond the `[u64; 2]` arrays,
167    // we must explicitly check that `size_bits` does not exceed 128.
168    let size_bits = size.bits();
169    if !(size_bits <= 128) {
    ::core::panicking::panic("assertion failed: size_bits <= 128")
};assert!(size_bits <= 128);
170    // More robust assertions that are redundant with `size_bits <= 128` and
171    // should be optimized away.
172    if !(size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap()) {
    ::core::panicking::panic("assertion failed: size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap()")
};assert!(size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap());
173    if !(size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap()) {
    ::core::panicking::panic("assertion failed: size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap()")
};assert!(size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap());
174    let size_bits = c_uint::try_from(size_bits).unwrap();
175
176    unsafe {
177        LLVMRustCreateRangeAttribute(llcx, size_bits, lower_words.as_ptr(), upper_words.as_ptr())
178    }
179}
180
181#[derive(#[automatically_derived]impl ::core::marker::Copy for AttributePlace { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for AttributePlace {
}
#[automatically_derived]impl ::core::clone::Clone for AttributePlace {
    #[inline]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<u32>;
        *self
    }
}Clone)]
182pub(crate) enum AttributePlace {
183    ReturnValue,
184    Argument(u32),
185    Function,
186}
187
188impl AttributePlace {
189    pub(crate) fn as_uint(self) -> c_uint {
190        match self {
191            AttributePlace::ReturnValue => 0,
192            AttributePlace::Argument(i) => 1 + i,
193            AttributePlace::Function => !0,
194        }
195    }
196}
197
198#[derive(#[automatically_derived]impl ::core::marker::Copy for CodeGenOptSize { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for CodeGenOptSize {
}
#[automatically_derived]impl ::core::clone::Clone for CodeGenOptSize {
    #[inline]
    fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    CodeGenOptSize {
}
#[automatically_derived]impl ::core::cmp::PartialEq for CodeGenOptSize {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
            ::core::intrinsics::discriminant_value(other)
    }
}PartialEq)]
199#[repr(C)]
200pub(crate) enum CodeGenOptSize {
201    CodeGenOptSizeNone = 0,
202    CodeGenOptSizeDefault = 1,
203    CodeGenOptSizeAggressive = 2,
204}
205
206pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) {
207    unsafe {
208        LLVMSetInstructionCallConv(instr, cc as c_uint);
209    }
210}
211pub(crate) fn SetFunctionCallConv(fn_: &Value, cc: CallConv) {
212    unsafe {
213        LLVMSetFunctionCallConv(fn_, cc as c_uint);
214    }
215}
216
217// Externally visible symbols that might appear in multiple codegen units need to appear in
218// their own comdat section so that the duplicates can be discarded at link time. This can for
219// example happen for generics when using multiple codegen units. This function simply uses the
220// value's name as the comdat value to make sure that it is in a 1-to-1 relationship to the
221// function.
222// For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52
223pub(crate) fn SetUniqueComdat<'ll>(llmod: Module<'ll>, val: &'ll Value) {
224    let name_buf = get_value_name(val);
225    let name =
226        CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap();
227    set_comdat(llmod, val, &name);
228}
229
230pub(crate) fn set_unnamed_address(global: &Value, unnamed: UnnamedAddr) {
231    LLVMSetUnnamedAddress(global, unnamed);
232}
233
234pub(crate) fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) {
235    unsafe {
236        LLVMSetThreadLocalMode(global, mode);
237    }
238}
239
240impl AttributeKind {
241    /// Create an LLVM Attribute with no associated value.
242    pub(crate) fn create_attr<'ll>(self, llcx: Context<'ll>) -> &'ll Attribute {
243        unsafe { LLVMRustCreateAttrNoValue(llcx, self) }
244    }
245}
246
247impl MemoryEffects {
248    /// Create an LLVM Attribute with these memory effects.
249    pub(crate) fn create_attr<'ll>(self, llcx: Context<'ll>) -> &'ll Attribute {
250        unsafe { LLVMRustCreateMemoryEffectsAttr(llcx, self) }
251    }
252}
253
254pub(crate) fn set_section(llglobal: &Value, section_name: &CStr) {
255    unsafe {
256        LLVMSetSection(llglobal, section_name.as_ptr());
257    }
258}
259
260pub(crate) fn add_global<'ll>(llmod: Module<'ll>, ty: &'ll Type, name_cstr: &CStr) -> &'ll Value {
261    unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) }
262}
263
264pub(crate) fn set_initializer(llglobal: &Value, constant_val: &Value) {
265    unsafe {
266        {
    match (&crate::llvm::LLVMGlobalGetValueType(llglobal),
            &crate::llvm::LLVMTypeOf(constant_val)) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(
267            crate::llvm::LLVMGlobalGetValueType(llglobal),
268            crate::llvm::LLVMTypeOf(constant_val)
269        );
270    }
271    unsafe {
272        LLVMSetInitializer(llglobal, constant_val);
273    }
274}
275
276pub(crate) fn set_global_constant(llglobal: &Value, is_constant: bool) {
277    LLVMSetGlobalConstant(llglobal, is_constant.to_llvm_bool());
278}
279
280pub(crate) fn get_linkage(llglobal: &Value) -> Linkage {
281    unsafe { LLVMGetLinkage(llglobal) }.to_rust()
282}
283
284pub(crate) fn set_linkage(llglobal: &Value, linkage: Linkage) {
285    unsafe {
286        LLVMSetLinkage(llglobal, linkage);
287    }
288}
289
290pub(crate) fn is_declaration(llglobal: &Value) -> bool {
291    unsafe { LLVMIsDeclaration(llglobal) }.is_true()
292}
293
294pub(crate) fn get_visibility(llglobal: &Value) -> Visibility {
295    unsafe { LLVMGetVisibility(llglobal) }.to_rust()
296}
297
298pub(crate) fn set_visibility(llglobal: &Value, visibility: Visibility) {
299    unsafe {
300        LLVMSetVisibility(llglobal, visibility);
301    }
302}
303
304pub(crate) fn set_alignment(llglobal: &Value, align: Align) {
305    unsafe {
306        ffi::LLVMSetAlignment(llglobal, align.bytes() as c_uint);
307    }
308}
309
310pub(crate) fn set_externally_initialized(llglobal: &Value, is_ext_init: bool) {
311    LLVMSetExternallyInitialized(llglobal, is_ext_init.to_llvm_bool());
312}
313
314/// Get the `name`d comdat from `llmod` and assign it to `llglobal`.
315///
316/// Inserts the comdat into `llmod` if it does not exist.
317/// It is an error to call this if the target does not support comdat.
318pub(crate) fn set_comdat<'ll>(llmod: Module<'ll>, llglobal: &'ll Value, name: &CStr) {
319    unsafe {
320        let comdat = LLVMGetOrInsertComdat(llmod, name.as_ptr());
321        LLVMSetComdat(llglobal, comdat);
322    }
323}
324
325pub(crate) fn count_params(llfn: &Value) -> c_uint {
326    LLVMCountParams(llfn)
327}
328
329/// Safe wrapper around `LLVMGetParam`, because segfaults are no fun.
330pub(crate) fn get_param(llfn: &Value, index: c_uint) -> &Value {
331    unsafe {
332        if !(index < LLVMCountParams(llfn)) {
    {
        ::core::panicking::panic_fmt(format_args!("out of bounds argument access: {0} out of {1} arguments",
                index, LLVMCountParams(llfn)));
    }
};assert!(
333            index < LLVMCountParams(llfn),
334            "out of bounds argument access: {} out of {} arguments",
335            index,
336            LLVMCountParams(llfn)
337        );
338        LLVMGetParam(llfn, index)
339    }
340}
341
342/// Safe wrapper for `LLVMGetValueName2`
343/// Needs to allocate the value, because `set_value_name` will invalidate
344/// the pointer.
345pub(crate) fn get_value_name(value: &Value) -> Vec<u8> {
346    unsafe {
347        let mut len = 0;
348        let data = LLVMGetValueName2(value, &mut len);
349        std::slice::from_raw_parts(data.cast(), len).to_vec()
350    }
351}
352
353#[derive(#[automatically_derived]impl ::core::fmt::Debug for Intrinsic {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field1_finish(f, "Intrinsic id",
            &&self.id)
    }
}Debug, #[automatically_derived]impl ::core::marker::Copy for Intrinsic { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for Intrinsic {
}
#[automatically_derived]impl ::core::clone::Clone for Intrinsic {
    #[inline]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<NonZero<c_uint>>;
        *self
    }
}Clone)]
354pub(crate) struct Intrinsic {
355    id: NonZero<c_uint>,
356}
357
358impl Intrinsic {
359    pub(crate) fn lookup(name: &[u8]) -> Option<Self> {
360        let id = unsafe { LLVMLookupIntrinsicID(name.as_ptr(), name.len()) };
361        NonZero::new(id).map(|id| Self { id })
362    }
363
364    pub(crate) fn is_overloaded(self) -> bool {
365        unsafe { LLVMIntrinsicIsOverloaded(self.id).is_true() }
366    }
367
368    pub(crate) fn is_target_specific(self) -> bool {
369        unsafe { LLVMRustIsTargetIntrinsic(self.id) }
370    }
371
372    pub(crate) fn get_declaration<'ll>(
373        self,
374        llmod: Module<'ll>,
375        type_params: &[&'ll Type],
376    ) -> &'ll Value {
377        unsafe {
378            LLVMGetIntrinsicDeclaration(llmod, self.id, type_params.as_ptr(), type_params.len())
379        }
380    }
381}
382
383/// Safe wrapper for `LLVMSetValueName2` from a byte slice
384pub(crate) fn set_value_name(value: &Value, name: &[u8]) {
385    unsafe { LLVMSetValueName2(value, name.as_ptr(), name.len()) };
386}
387
388pub(crate) fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error> {
389    String::from_utf8(RustString::build_byte_buffer(f))
390}
391
392pub(crate) fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> {
393    RustString::build_byte_buffer(f)
394}
395
396pub(crate) fn twine_to_string(tr: &Twine) -> String {
397    unsafe {
398        build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM")
399    }
400}
401
402pub(crate) fn last_error() -> Option<String> {
403    unsafe {
404        let cstr = LLVMRustGetLastError();
405        if cstr.is_null() {
406            None
407        } else {
408            let err = CStr::from_ptr(cstr).to_bytes();
409            let err = String::from_utf8_lossy(err).to_string();
410            libc::free(cstr as *mut _);
411            Some(err)
412        }
413    }
414}
415
416/// Owning pointer to an [`OperandBundle`] that will dispose of the bundle
417/// when dropped.
418pub(crate) struct OperandBundleBox<'a> {
419    raw: ptr::NonNull<OperandBundle<'a>>,
420}
421
422impl<'a> OperandBundleBox<'a> {
423    pub(crate) fn new(name: &str, vals: &[&'a Value]) -> Self {
424        let raw = unsafe {
425            LLVMCreateOperandBundle(name.as_ptr(), name.len(), vals.as_ptr(), vals.len() as c_uint)
426        };
427        Self { raw: ptr::NonNull::new(raw).unwrap() }
428    }
429
430    /// Dereferences to the underlying `&OperandBundle`.
431    ///
432    /// This can't be a `Deref` implementation because `OperandBundle` transitively
433    /// contains an extern type, which is incompatible with `Deref::Target: ?Sized`.
434    pub(crate) fn as_ref(&self) -> &OperandBundle<'a> {
435        // SAFETY: The returned reference is opaque and can only used for FFI.
436        // It is valid for as long as `&self` is.
437        unsafe { self.raw.as_ref() }
438    }
439}
440
441impl Drop for OperandBundleBox<'_> {
442    fn drop(&mut self) {
443        unsafe {
444            LLVMDisposeOperandBundle(self.raw);
445        }
446    }
447}
448
449pub(crate) fn add_module_flag_u32<'ll>(
450    module: Module<'ll>,
451    merge_behavior: ModuleFlagMergeBehavior,
452    key: &str,
453    value: u32,
454) {
455    unsafe {
456        LLVMRustAddModuleFlagU32(module, merge_behavior, key.as_ptr(), key.len(), value);
457    }
458}
459
460pub(crate) fn add_module_flag_str<'ll>(
461    module: Module<'ll>,
462    merge_behavior: ModuleFlagMergeBehavior,
463    key: &str,
464    value: &str,
465) {
466    unsafe {
467        LLVMRustAddModuleFlagString(
468            module,
469            merge_behavior,
470            key.as_ptr(),
471            key.len(),
472            value.as_ptr(),
473            value.len(),
474        );
475    }
476}
477
478pub(crate) fn set_dllimport_storage_class<'ll>(v: &'ll Value) {
479    unsafe {
480        LLVMSetDLLStorageClass(v, DLLStorageClass::DllImport);
481    }
482}
483
484pub(crate) fn set_dso_local<'ll>(v: &'ll Value) {
485    unsafe {
486        LLVMRustSetDSOLocal(v, true);
487    }
488}
489
490/// Safe wrapper for `LLVMRustAppendModuleInlineAsm`, which delegates to
491/// `Module::appendModuleInlineAsm`.
492pub(crate) fn append_module_inline_asm<'ll>(
493    llmod: Module<'ll>,
494    asm: &[u8],
495    target_features: &str,
496    target_cpu: &str,
497) {
498    unsafe {
499        LLVMRustAppendModuleInlineAsm(
500            llmod,
501            asm.as_ptr(),
502            asm.len(),
503            target_features.as_ptr(),
504            target_features.len(),
505            target_cpu.as_ptr(),
506            target_cpu.len(),
507        );
508    }
509}
510
511/// Safe wrapper for `LLVMAddAlias2`
512pub(crate) fn add_alias<'ll>(
513    module: Module<'ll>,
514    ty: &Type,
515    address_space: AddressSpace,
516    aliasee: &Value,
517    name: &CStr,
518) -> &'ll Value {
519    unsafe { LLVMAddAlias2(module, ty, address_space.0, aliasee, name.as_ptr()) }
520}
521
522/// Safe wrapper for `LLVMRustConstPtrAuth`.
523pub(crate) fn const_ptr_auth<'ll>(
524    ptr: &'ll Value,
525    key: u32,
526    disc: u64,
527    addr_diversity: Option<&'ll Value>,
528) -> &'ll Value {
529    unsafe {
530        let addr_div_ptr = addr_diversity.map_or(std::ptr::null(), |v| v as *const Value);
531        let deactivation_symbol = std::ptr::null();
532        let result =
533            LLVMRustConstPtrAuth(ptr as *const Value, key, disc, addr_div_ptr, deactivation_symbol);
534        &*result
535    }
536}