rustc_codegen_llvm/llvm/
mod.rs

1#![allow(non_snake_case)]
2
3use std::ffi::{CStr, CString};
4use std::num::NonZero;
5use std::ptr;
6use std::string::FromUtf8Error;
7
8use libc::c_uint;
9use rustc_abi::{AddressSpace, Align, Size, WrappingRange};
10use rustc_llvm::RustString;
11
12pub(crate) use self::CallConv::*;
13pub(crate) use self::conversions::*;
14pub(crate) use self::ffi::*;
15pub(crate) use self::metadata_kind::*;
16
17mod conversions;
18pub(crate) mod diagnostic;
19pub(crate) mod enzyme_ffi;
20mod ffi;
21mod metadata_kind;
22pub(crate) mod offload_ffi;
23pub(crate) mod tpde_ffi;
24
25pub(crate) use self::enzyme_ffi::*;
26pub(crate) use self::offload_ffi::*;
27pub(crate) use self::tpde_ffi::*;
28
29impl LLVMRustResult {
30    pub(crate) fn into_result(self) -> Result<(), ()> {
31        match self {
32            LLVMRustResult::Success => Ok(()),
33            LLVMRustResult::Failure => Err(()),
34        }
35    }
36}
37
38pub(crate) fn AddFunctionAttributes<'ll>(
39    llfn: Value<'ll>,
40    idx: AttributePlace,
41    attrs: &[&'ll Attribute],
42) {
43    unsafe {
44        LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len());
45    }
46}
47
48pub(crate) fn HasStringAttribute<'ll>(llfn: Value<'ll>, name: &str) -> bool {
49    unsafe { LLVMRustHasFnAttribute(llfn, name.as_ptr(), name.len()) }
50}
51
52pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: Value<'ll>, name: &str) {
53    unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_ptr(), name.len()) }
54}
55
56pub(crate) fn AddCallSiteAttributes<'ll>(
57    callsite: Value<'ll>,
58    idx: AttributePlace,
59    attrs: &[&'ll Attribute],
60) {
61    unsafe {
62        LLVMRustAddCallSiteAttributes(callsite, idx.as_uint(), attrs.as_ptr(), attrs.len());
63    }
64}
65
66pub(crate) fn CreateAttrStringValue<'ll>(
67    llcx: Context<'ll>,
68    attr: &str,
69    value: &str,
70) -> &'ll Attribute {
71    unsafe {
72        LLVMCreateStringAttribute(
73            llcx,
74            attr.as_ptr(),
75            attr.len().try_into().unwrap(),
76            value.as_ptr(),
77            value.len().try_into().unwrap(),
78        )
79    }
80}
81pub(crate) fn CreateAttrStringValueFromCStr<'ll>(
82    llcx: Context<'ll>,
83    attr: &std::ffi::CStr,
84    value: &std::ffi::CStr,
85) -> &'ll Attribute {
86    let attr = attr.to_bytes();
87    let value = value.to_bytes();
88    unsafe {
89        LLVMCreateStringAttribute(
90            llcx,
91            attr.as_ptr(),
92            attr.len() as c_uint,
93            value.as_ptr(),
94            value.len() as c_uint,
95        )
96    }
97}
98
99pub(crate) fn CreateAttrString<'ll>(llcx: Context<'ll>, attr: &str) -> &'ll Attribute {
100    unsafe {
101        LLVMCreateStringAttribute(
102            llcx,
103            attr.as_ptr(),
104            attr.len().try_into().unwrap(),
105            std::ptr::null(),
106            0,
107        )
108    }
109}
110
111pub(crate) fn CreateAlignmentAttr<'ll>(llcx: Context<'ll>, bytes: u64) -> &'ll Attribute {
112    unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) }
113}
114
115pub(crate) fn CreateDereferenceableAttr<'ll>(llcx: Context<'ll>, bytes: u64) -> &'ll Attribute {
116    unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) }
117}
118
119pub(crate) fn CreateDereferenceableOrNullAttr<'ll>(
120    llcx: Context<'ll>,
121    bytes: u64,
122) -> &'ll Attribute {
123    unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) }
124}
125
126pub(crate) fn CreateByValAttr<'ll>(llcx: Context<'ll>, ty: Type<'ll>) -> &'ll Attribute {
127    unsafe { LLVMRustCreateByValAttr(llcx, ty) }
128}
129
130pub(crate) fn CreateByRefAttr<'ll>(llcx: Context<'ll>, ty: Type<'ll>) -> &'ll Attribute {
131    unsafe { LLVMRustCreateByRefAttr(llcx, ty) }
132}
133
134pub(crate) fn CreateStructRetAttr<'ll>(llcx: Context<'ll>, ty: Type<'ll>) -> &'ll Attribute {
135    unsafe { LLVMRustCreateStructRetAttr(llcx, ty) }
136}
137
138pub(crate) fn CreateUWTableAttr<'ll>(llcx: Context<'ll>, async_: bool) -> &'ll Attribute {
139    unsafe { LLVMRustCreateUWTableAttr(llcx, async_) }
140}
141
142pub(crate) fn CreateAllocSizeAttr<'ll>(llcx: Context<'ll>, size_arg: u32) -> &'ll Attribute {
143    unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) }
144}
145
146pub(crate) fn CreateAllocKindAttr<'ll>(
147    llcx: Context<'ll>,
148    kind_arg: AllocKindFlags,
149) -> &'ll Attribute {
150    unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) }
151}
152
153pub(crate) fn CreateRangeAttr<'ll>(
154    llcx: Context<'ll>,
155    size: Size,
156    range: WrappingRange,
157) -> &'ll Attribute {
158    let lower = range.start;
159    // LLVM treats the upper bound as exclusive, but allows wrapping.
160    let upper = range.end.wrapping_add(1);
161
162    // Pass each `u128` endpoint value as a `[u64; 2]` array, least-significant part first.
163    let as_u64_array = |x: u128| [x as u64, (x >> 64) as u64];
164    let lower_words: [u64; 2] = as_u64_array(lower);
165    let upper_words: [u64; 2] = as_u64_array(upper);
166
167    // To ensure that LLVM doesn't try to read beyond the `[u64; 2]` arrays,
168    // we must explicitly check that `size_bits` does not exceed 128.
169    let size_bits = size.bits();
170    if !(size_bits <= 128) {
    ::core::panicking::panic("assertion failed: size_bits <= 128")
};assert!(size_bits <= 128);
171    // More robust assertions that are redundant with `size_bits <= 128` and
172    // should be optimized away.
173    if !(size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap()) {
    ::core::panicking::panic("assertion failed: size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap()")
};assert!(size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap());
174    if !(size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap()) {
    ::core::panicking::panic("assertion failed: size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap()")
};assert!(size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap());
175    let size_bits = c_uint::try_from(size_bits).unwrap();
176
177    unsafe {
178        LLVMRustCreateRangeAttribute(llcx, size_bits, lower_words.as_ptr(), upper_words.as_ptr())
179    }
180}
181
182#[derive(#[automatically_derived]impl ::core::marker::Copy for AttributePlace { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for AttributePlace {
}
#[automatically_derived]impl ::core::clone::Clone for AttributePlace {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<u32>;
        *self
    }
}Clone)]
183pub(crate) enum AttributePlace {
184    ReturnValue,
185    Argument(u32),
186    Function,
187}
188
189impl AttributePlace {
190    pub(crate) fn as_uint(self) -> c_uint {
191        match self {
192            AttributePlace::ReturnValue => 0,
193            AttributePlace::Argument(i) => 1 + i,
194            AttributePlace::Function => !0,
195        }
196    }
197}
198
199pub(crate) fn SetInstructionCallConv<'ll>(instr: Value<'ll>, cc: CallConv) {
200    unsafe {
201        LLVMSetInstructionCallConv(instr, cc as c_uint);
202    }
203}
204pub(crate) fn SetFunctionCallConv<'ll>(fn_: Value<'ll>, cc: CallConv) {
205    unsafe {
206        LLVMSetFunctionCallConv(fn_, cc as c_uint);
207    }
208}
209
210// Externally visible symbols that might appear in multiple codegen units need to appear in
211// their own comdat section so that the duplicates can be discarded at link time. This can for
212// example happen for generics when using multiple codegen units. This function simply uses the
213// value's name as the comdat value to make sure that it is in a 1-to-1 relationship to the
214// function.
215// For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52
216pub(crate) fn SetUniqueComdat<'ll>(llmod: Module<'ll>, val: Value<'ll>) {
217    let name_buf = get_value_name(val);
218    let name =
219        CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap();
220    set_comdat(llmod, val, &name);
221}
222
223pub(crate) fn set_unnamed_address<'ll>(global: Value<'ll>, unnamed: UnnamedAddr) {
224    LLVMSetUnnamedAddress(global, unnamed);
225}
226
227pub(crate) fn set_thread_local_mode<'ll>(global: Value<'ll>, mode: ThreadLocalMode) {
228    unsafe {
229        LLVMSetThreadLocalMode(global, mode);
230    }
231}
232
233impl AttributeKind {
234    /// Create an LLVM Attribute with no associated value.
235    pub(crate) fn create_attr<'ll>(self, llcx: Context<'ll>) -> &'ll Attribute {
236        unsafe { LLVMRustCreateAttrNoValue(llcx, self) }
237    }
238}
239
240impl MemoryEffects {
241    /// Create an LLVM Attribute with these memory effects.
242    pub(crate) fn create_attr<'ll>(self, llcx: Context<'ll>) -> &'ll Attribute {
243        unsafe { LLVMRustCreateMemoryEffectsAttr(llcx, self) }
244    }
245}
246
247pub(crate) fn set_section<'ll>(llglobal: Value<'ll>, section_name: &CStr) {
248    unsafe {
249        LLVMSetSection(llglobal, section_name.as_ptr());
250    }
251}
252
253pub(crate) fn add_global<'ll>(llmod: Module<'ll>, ty: Type<'ll>, name_cstr: &CStr) -> Value<'ll> {
254    unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) }
255}
256
257pub(crate) fn set_initializer<'ll>(llglobal: Value<'ll>, constant_val: Value<'ll>) {
258    unsafe {
259        {
    match (&crate::llvm::LLVMGlobalGetValueType(llglobal),
            &crate::llvm::LLVMTypeOf(constant_val)) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(
260            crate::llvm::LLVMGlobalGetValueType(llglobal),
261            crate::llvm::LLVMTypeOf(constant_val)
262        );
263    }
264    unsafe {
265        LLVMSetInitializer(llglobal, constant_val);
266    }
267}
268
269pub(crate) fn set_global_constant<'ll>(llglobal: Value<'ll>, is_constant: bool) {
270    LLVMSetGlobalConstant(llglobal, is_constant.to_llvm_bool());
271}
272
273pub(crate) fn get_linkage<'ll>(llglobal: Value<'ll>) -> Linkage {
274    unsafe { LLVMGetLinkage(llglobal) }.to_rust()
275}
276
277pub(crate) fn set_linkage<'ll>(llglobal: Value<'ll>, linkage: Linkage) {
278    unsafe {
279        LLVMSetLinkage(llglobal, linkage);
280    }
281}
282
283pub(crate) fn is_declaration<'ll>(llglobal: Value<'ll>) -> bool {
284    unsafe { LLVMIsDeclaration(llglobal) }.is_true()
285}
286
287pub(crate) fn get_visibility<'ll>(llglobal: Value<'ll>) -> Visibility {
288    unsafe { LLVMGetVisibility(llglobal) }.to_rust()
289}
290
291pub(crate) fn set_visibility<'ll>(llglobal: Value<'ll>, visibility: Visibility) {
292    unsafe {
293        LLVMSetVisibility(llglobal, visibility);
294    }
295}
296
297pub(crate) fn set_alignment<'ll>(llglobal: Value<'ll>, align: Align) {
298    unsafe {
299        ffi::LLVMSetAlignment(llglobal, align.bytes() as c_uint);
300    }
301}
302
303pub(crate) fn set_externally_initialized<'ll>(llglobal: Value<'ll>, is_ext_init: bool) {
304    LLVMSetExternallyInitialized(llglobal, is_ext_init.to_llvm_bool());
305}
306
307/// Get the `name`d comdat from `llmod` and assign it to `llglobal`.
308///
309/// Inserts the comdat into `llmod` if it does not exist.
310/// It is an error to call this if the target does not support comdat.
311pub(crate) fn set_comdat<'ll>(llmod: Module<'ll>, llglobal: Value<'ll>, name: &CStr) {
312    unsafe {
313        let comdat = LLVMGetOrInsertComdat(llmod, name.as_ptr());
314        LLVMSetComdat(llglobal, comdat);
315    }
316}
317
318pub(crate) fn count_params<'ll>(llfn: Value<'ll>) -> c_uint {
319    LLVMCountParams(llfn)
320}
321
322/// Safe wrapper around `LLVMGetParam`, because segfaults are no fun.
323pub(crate) fn get_param<'ll>(llfn: Value<'ll>, index: c_uint) -> Value<'ll> {
324    unsafe {
325        if !(index < LLVMCountParams(llfn)) {
    {
        ::core::panicking::panic_fmt(format_args!("out of bounds argument access: {0} out of {1} arguments",
                index, LLVMCountParams(llfn)));
    }
};assert!(
326            index < LLVMCountParams(llfn),
327            "out of bounds argument access: {} out of {} arguments",
328            index,
329            LLVMCountParams(llfn)
330        );
331        LLVMGetParam(llfn, index)
332    }
333}
334
335/// Safe wrapper for `LLVMGetValueName2`
336/// Needs to allocate the value, because `set_value_name` will invalidate
337/// the pointer.
338pub(crate) fn get_value_name<'ll>(value: Value<'ll>) -> Vec<u8> {
339    unsafe {
340        let mut len = 0;
341        let data = LLVMGetValueName2(value, &mut len);
342        std::slice::from_raw_parts(data.cast(), len).to_vec()
343    }
344}
345
346#[derive(#[automatically_derived]impl ::core::fmt::Debug for Intrinsic {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field1_finish(f, "Intrinsic id",
            &&self.id)
    }
}Debug, #[automatically_derived]impl ::core::marker::Copy for Intrinsic { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for Intrinsic {
}
#[automatically_derived]impl ::core::clone::Clone for Intrinsic {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<NonZero<c_uint>>;
        *self
    }
}Clone)]
347pub(crate) struct Intrinsic {
348    id: NonZero<c_uint>,
349}
350
351impl Intrinsic {
352    pub(crate) fn lookup(name: &[u8]) -> Option<Self> {
353        let id = unsafe { LLVMLookupIntrinsicID(name.as_ptr(), name.len()) };
354        NonZero::new(id).map(|id| Self { id })
355    }
356
357    pub(crate) fn is_overloaded(self) -> bool {
358        unsafe { LLVMIntrinsicIsOverloaded(self.id).is_true() }
359    }
360
361    pub(crate) fn is_target_specific(self) -> bool {
362        unsafe { LLVMRustIsTargetIntrinsic(self.id) }
363    }
364
365    pub(crate) fn get_declaration<'ll>(
366        self,
367        llmod: Module<'ll>,
368        type_params: &[Type<'ll>],
369    ) -> Value<'ll> {
370        unsafe {
371            LLVMGetIntrinsicDeclaration(llmod, self.id, type_params.as_ptr(), type_params.len())
372        }
373    }
374}
375
376/// Safe wrapper for `LLVMSetValueName2` from a byte slice
377pub(crate) fn set_value_name<'ll>(value: Value<'ll>, name: &[u8]) {
378    if !!name.contains(&0) {
    ::core::panicking::panic("assertion failed: !name.contains(&0)")
};assert!(!name.contains(&0));
379    unsafe { LLVMSetValueName2(value, name.as_ptr(), name.len()) };
380}
381
382pub(crate) fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error> {
383    String::from_utf8(RustString::build_byte_buffer(f))
384}
385
386pub(crate) fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> {
387    RustString::build_byte_buffer(f)
388}
389
390pub(crate) fn twine_to_string(tr: &Twine) -> String {
391    unsafe {
392        build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM")
393    }
394}
395
396pub(crate) fn last_error() -> Option<String> {
397    unsafe {
398        let cstr = LLVMRustGetLastError();
399        if cstr.is_null() {
400            None
401        } else {
402            let err = CStr::from_ptr(cstr).to_bytes();
403            let err = String::from_utf8_lossy(err).to_string();
404            libc::free(cstr as *mut _);
405            Some(err)
406        }
407    }
408}
409
410/// Owning pointer to an [`OperandBundle`] that will dispose of the bundle
411/// when dropped.
412pub(crate) struct OperandBundleBox<'ll> {
413    raw: ptr::NonNull<OperandBundle<'ll>>,
414}
415
416impl<'ll> OperandBundleBox<'ll> {
417    pub(crate) fn new(name: &str, vals: &[Value<'ll>]) -> Self {
418        let raw = unsafe {
419            LLVMCreateOperandBundle(name.as_ptr(), name.len(), vals.as_ptr(), vals.len() as c_uint)
420        };
421        Self { raw: ptr::NonNull::new(raw).unwrap() }
422    }
423
424    /// Dereferences to the underlying `&OperandBundle`.
425    ///
426    /// This can't be a `Deref` implementation because `OperandBundle` transitively
427    /// contains an extern type, which is incompatible with `Deref::Target: ?Sized`.
428    pub(crate) fn as_ref(&self) -> &OperandBundle<'ll> {
429        // SAFETY: The returned reference is opaque and can only used for FFI.
430        // It is valid for as long as `&self` is.
431        unsafe { self.raw.as_ref() }
432    }
433}
434
435impl Drop for OperandBundleBox<'_> {
436    fn drop(&mut self) {
437        unsafe {
438            LLVMDisposeOperandBundle(self.raw);
439        }
440    }
441}
442
443pub(crate) fn add_module_flag_u32<'ll>(
444    module: Module<'ll>,
445    merge_behavior: ModuleFlagMergeBehavior,
446    key: &str,
447    value: u32,
448) {
449    unsafe {
450        LLVMRustAddModuleFlagU32(module, merge_behavior, key.as_ptr(), key.len(), value);
451    }
452}
453
454pub(crate) fn add_module_flag_str<'ll>(
455    module: Module<'ll>,
456    merge_behavior: ModuleFlagMergeBehavior,
457    key: &str,
458    value: &str,
459) {
460    unsafe {
461        LLVMRustAddModuleFlagString(
462            module,
463            merge_behavior,
464            key.as_ptr(),
465            key.len(),
466            value.as_ptr(),
467            value.len(),
468        );
469    }
470}
471
472pub(crate) fn set_dllimport_storage_class<'ll>(v: Value<'ll>) {
473    unsafe {
474        LLVMSetDLLStorageClass(v, DLLStorageClass::DllImport);
475    }
476}
477
478pub(crate) fn set_dso_local<'ll>(v: Value<'ll>) {
479    unsafe {
480        LLVMRustSetDSOLocal(v, true);
481    }
482}
483
484/// Safe wrapper for `LLVMRustAppendModuleInlineAsm`, which delegates to
485/// `Module::appendModuleInlineAsm`.
486pub(crate) fn append_module_inline_asm<'ll>(
487    llmod: Module<'ll>,
488    asm: &[u8],
489    target_features: &str,
490    target_cpu: &str,
491) {
492    unsafe {
493        LLVMRustAppendModuleInlineAsm(
494            llmod,
495            asm.as_ptr(),
496            asm.len(),
497            target_features.as_ptr(),
498            target_features.len(),
499            target_cpu.as_ptr(),
500            target_cpu.len(),
501        );
502    }
503}
504
505/// Safe wrapper for `LLVMAddAlias2`
506pub(crate) fn add_alias<'ll>(
507    module: Module<'ll>,
508    ty: Type<'ll>,
509    address_space: AddressSpace,
510    aliasee: Value<'ll>,
511    name: &CStr,
512) -> Value<'ll> {
513    unsafe { LLVMAddAlias2(module, ty, address_space.0, aliasee, name.as_ptr()) }
514}
515
516/// Safe wrapper for `LLVMRustConstPtrAuth`.
517pub(crate) fn const_ptr_auth<'ll>(
518    ptr: Value<'ll>,
519    key: u32,
520    disc: u64,
521    addr_diversity: Option<Value<'ll>>,
522) -> Value<'ll> {
523    unsafe { LLVMRustConstPtrAuth(ptr, key, disc, addr_diversity, None) }
524}
rustc_codegen_llvm/llvm/mod.rs

rustc_codegen_llvm/llvm/
mod.rs
