1use std::ffi::{CStr, CString};
2use std::io::{self, Write};
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use std::{fs, ptr, slice, str};
6
7use libc::{c_char, c_int, c_void, size_t};
8use rustc_codegen_ssa::back::link::ensure_removed;
9use rustc_codegen_ssa::back::write::{
10 BitcodeSection, CodegenContext, EmitObj, InlineAsmError, ModuleConfig, SharedEmitter,
11 TargetMachineFactoryConfig, TargetMachineFactoryFn,
12};
13use rustc_codegen_ssa::base::wants_wasm_eh;
14use rustc_codegen_ssa::traits::*;
15use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind};
16use rustc_data_structures::profiling::SelfProfilerRef;
17use rustc_data_structures::small_c_str::SmallCStr;
18use rustc_errors::{DiagCtxt, DiagCtxtHandle, Level};
19use rustc_fs_util::{link_or_copy, path_to_c_string};
20use rustc_middle::ty::TyCtxt;
21use rustc_session::Session;
22use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath};
23use rustc_span::{BytePos, DUMMY_SP, InnerSpan, Pos, RemapPathScopeComponents};
24use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
25use tracing::{debug, trace};
26
27use crate::back::llvm_backend::ModuleRefs;
28use crate::back::lto::{Buffer, ModuleBuffer};
29use crate::back::owned_target_machine::OwnedTargetMachine;
30use crate::back::profiling::{
31 LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
32};
33use crate::builder::SBuilder;
34use crate::builder::gpu_offload::scalar_width;
35use crate::context::SimpleCx;
36use crate::diagnostics::{
37 CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, ParseTargetMachineConfig,
38 UnsupportedCompression, WithLlvmError, WriteBytecode,
39};
40use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
41use crate::llvm::{self, DiagnosticInfo, TypeKind};
42use crate::type_::llvm_type_ptr;
43use crate::{LlvmCodegenBackend, ModuleLlvm, attributes, base, common, llvm_util};
44
45pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! {
46 match llvm::last_error() {
47 Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)),
48 None => dcx.emit_fatal(err),
49 }
50}
51
52fn write_output_file<'ll>(
53 dcx: DiagCtxtHandle<'_>,
54 target: &'ll llvm::TargetMachine,
55 no_builtins: bool,
56 m: llvm::Module<'ll>,
57 output: &Path,
58 dwo_output: Option<&Path>,
59 file_type: llvm::FileType,
60 self_profiler_ref: &SelfProfilerRef,
61 verify_llvm_ir: bool,
62) {
63 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:63",
"rustc_codegen_llvm::back::write", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
::tracing_core::__macro_support::Option::Some(63u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("write_output_file output={0:?} dwo_output={1:?}",
output, dwo_output) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output);
64 let output_c = path_to_c_string(output);
65 let dwo_output_c;
66 let dwo_output_ptr = if let Some(dwo_output) = dwo_output {
67 dwo_output_c = path_to_c_string(dwo_output);
68 dwo_output_c.as_ptr()
69 } else {
70 std::ptr::null()
71 };
72 let result = unsafe {
73 llvm::LLVMRustWriteOutputFile(
74 target,
75 m,
76 output_c.as_ptr(),
77 dwo_output_ptr,
78 file_type,
79 verify_llvm_ir,
80 no_builtins,
81 )
82 };
83
84 if result == llvm::LLVMRustResult::Success {
86 let artifact_kind = match file_type {
87 llvm::FileType::ObjectFile => "object_file",
88 llvm::FileType::AssemblyFile => "assembly_file",
89 };
90 record_artifact_size(self_profiler_ref, artifact_kind, output);
91 if let Some(dwo_file) = dwo_output {
92 record_artifact_size(self_profiler_ref, "dwo_file", dwo_file);
93 }
94 }
95
96 result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
97}
98
99pub(crate) fn create_informational_target_machine(sess: &Session) -> OwnedTargetMachine {
100 let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None };
101 target_machine_factory(sess, config::OptLevel::No)(sess.dcx(), config)
102}
103
104pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
105 let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() {
106 tcx.output_filenames(()).split_dwarf_path(
107 tcx.sess.split_debuginfo(),
108 tcx.sess.opts.unstable_opts.split_dwarf_kind,
109 mod_name,
110 )
111 } else {
112 None
113 };
114
115 let output_obj_file =
116 Some(tcx.output_filenames(()).temp_path_for_cgu(OutputType::Object, mod_name));
117 let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
118
119 target_machine_factory(tcx.sess, tcx.backend_optimization_level(()))(tcx.dcx(), config)
120}
121
122fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) {
123 use self::config::OptLevel::*;
124 match cfg {
125 No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone),
126 Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone),
127 More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone),
128 Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone),
129 Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault),
130 SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive),
131 }
132}
133
134fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
135 use config::OptLevel::*;
136 match cfg {
137 No => llvm::PassBuilderOptLevel::O0,
138 Less => llvm::PassBuilderOptLevel::O1,
139 More => llvm::PassBuilderOptLevel::O2,
140 Aggressive => llvm::PassBuilderOptLevel::O3,
141 Size => llvm::PassBuilderOptLevel::Os,
142 SizeMin => llvm::PassBuilderOptLevel::Oz,
143 }
144}
145
146fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
147 match relocation_model {
148 RelocModel::Static => llvm::RelocModel::Static,
149 RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
152 RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
153 RelocModel::Ropi => llvm::RelocModel::ROPI,
154 RelocModel::Rwpi => llvm::RelocModel::RWPI,
155 RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI,
156 }
157}
158
159pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel {
160 match code_model {
161 Some(CodeModel::Tiny) => llvm::CodeModel::Tiny,
162 Some(CodeModel::Small) => llvm::CodeModel::Small,
163 Some(CodeModel::Kernel) => llvm::CodeModel::Kernel,
164 Some(CodeModel::Medium) => llvm::CodeModel::Medium,
165 Some(CodeModel::Large) => llvm::CodeModel::Large,
166 None => llvm::CodeModel::None,
167 }
168}
169
170fn to_llvm_float_abi(float_abi: Option<FloatAbi>) -> llvm::FloatAbi {
171 match float_abi {
172 None => llvm::FloatAbi::Default,
173 Some(FloatAbi::Soft) => llvm::FloatAbi::Soft,
174 Some(FloatAbi::Hard) => llvm::FloatAbi::Hard,
175 }
176}
177
178pub(crate) fn target_machine_factory(
179 sess: &Session,
180 optlvl: config::OptLevel,
181) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
182 let _prof_timer = sess.prof.generic_activity("target_machine_factory");
184
185 let reloc_model = to_llvm_relocation_model(sess.relocation_model());
186
187 let (opt_level, _) = to_llvm_opt_settings(optlvl);
188 let float_abi = to_llvm_float_abi(sess.target.llvm_floatabi);
189
190 let ffunction_sections =
191 sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
192 let fdata_sections = ffunction_sections;
193 let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names;
194
195 let code_model = to_llvm_code_model(sess.code_model());
196
197 let singlethread = sess.target.singlethread(&sess.internal_target_features);
198
199 let triple = SmallCStr::new(&llvm_util::llvm_target(sess));
200 let cpu = SmallCStr::new(llvm_util::target_cpu(sess));
201 let features = CString::new(sess.global_backend_features.join(",")).unwrap();
202 let abi = SmallCStr::new(sess.target.llvm_abiname.desc());
203 let trap_unreachable =
204 sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable);
205 let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes;
206
207 let verbose_asm = sess.opts.unstable_opts.verbose_asm;
208 let relax_x86_elf_relocations =
209 sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations);
210
211 let use_init_array =
212 !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
213
214 let path_mapping = sess.source_map().path_mapping().clone();
215 let working_dir = sess.source_map().working_dir().clone();
216
217 let use_emulated_tls = #[allow(non_exhaustive_omitted_patterns)] match sess.tls_model() {
TlsModel::Emulated => true,
_ => false,
}matches!(sess.tls_model(), TlsModel::Emulated);
218
219 let debuginfo_compression = match sess.opts.unstable_opts.debuginfo_compression {
220 config::DebugInfoCompression::None => llvm::CompressionKind::None,
221 config::DebugInfoCompression::Zlib => {
222 if llvm::LLVMRustLLVMHasZlibCompression() {
223 llvm::CompressionKind::Zlib
224 } else {
225 sess.dcx().emit_warn(UnsupportedCompression { algorithm: "zlib" });
226 llvm::CompressionKind::None
227 }
228 }
229 config::DebugInfoCompression::Zstd => {
230 if llvm::LLVMRustLLVMHasZstdCompression() {
231 llvm::CompressionKind::Zstd
232 } else {
233 sess.dcx().emit_warn(UnsupportedCompression { algorithm: "zstd" });
234 llvm::CompressionKind::None
235 }
236 }
237 };
238
239 let use_wasm_eh = wants_wasm_eh(&sess.target);
240
241 let large_data_threshold = sess.opts.unstable_opts.large_data_threshold.unwrap_or(0);
242
243 let prof = SelfProfilerRef::clone(&sess.prof);
244 Arc::new(move |dcx: DiagCtxtHandle<'_>, config: TargetMachineFactoryConfig| {
245 let _prof_timer = prof.generic_activity("target_machine_factory_inner");
247
248 let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
249 let path = path.unwrap_or_default();
250 let path = path_mapping
251 .to_real_filename(&working_dir, path)
252 .path(RemapPathScopeComponents::DEBUGINFO)
253 .to_string_lossy()
254 .into_owned();
255 CString::new(path).unwrap()
256 };
257
258 let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file);
259 let output_obj_file = path_to_cstring_helper(config.output_obj_file);
260
261 OwnedTargetMachine::new(
262 &triple,
263 &cpu,
264 &features,
265 &abi,
266 code_model,
267 reloc_model,
268 opt_level,
269 float_abi,
270 ffunction_sections,
271 fdata_sections,
272 funique_section_names,
273 trap_unreachable,
274 singlethread,
275 verbose_asm,
276 emit_stack_size_section,
277 relax_x86_elf_relocations,
278 use_init_array,
279 &split_dwarf_file,
280 &output_obj_file,
281 debuginfo_compression,
282 use_emulated_tls,
283 use_wasm_eh,
284 large_data_threshold,
285 )
286 .unwrap_or_else(|err| dcx.emit_fatal(ParseTargetMachineConfig(err)))
287 })
288}
289
290pub(crate) fn save_temp_bitcode(
291 cgcx: &CodegenContext,
292 module: &ModuleCodegen<ModuleLlvm>,
293 name: &str,
294) {
295 if !cgcx.save_temps {
296 return;
297 }
298 let ext = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0}.bc", name))
})format!("{name}.bc");
299 let path = cgcx.output_filenames.temp_path_ext_for_cgu(&ext, &module.name);
300 write_bitcode_to_file(&module.module_llvm, &path)
301}
302
303fn write_bitcode_to_file(module: &ModuleLlvm, path: &Path) {
304 let path = path_to_c_string(&path);
305 module.with_module_refs(|ModuleRefs { llmod, .. }| {
306 unsafe { llvm::LLVMWriteBitcodeToFile(llmod, path.as_ptr()) };
307 });
308}
309
310pub(crate) enum CodegenDiagnosticsStage {
312 Opt,
314 LTO,
316 Codegen,
318}
319
320#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl<'a>
::core::clone::TrivialClone for DiagnosticHandlerCallbackData<'a> {
}
#[automatically_derived]impl<'a> ::core::clone::Clone for
DiagnosticHandlerCallbackData<'a> {
#[inline]
fn clone(&self) -> Self {
let _: ::core::clone::AssertParamIsClone<&'a CodegenContext>;
let _: ::core::clone::AssertParamIsClone<&'a SharedEmitter>;
*self
}
}Clone, #[automatically_derived]impl<'a> ::core::marker::Copy for
DiagnosticHandlerCallbackData<'a> {
}Copy)]
322struct DiagnosticHandlerCallbackData<'a> {
323 cgcx: &'a CodegenContext,
324 shared_emitter: &'a SharedEmitter,
325}
326
327pub(crate) struct DiagnosticHandlers<'a, 'll> {
328 llcx: llvm::Context<'ll>,
329 old_handler: Option<ptr::NonNull<llvm::DiagnosticHandler>>,
330
331 diagnostic_handler_callback_data: *mut DiagnosticHandlerCallbackData<'a>,
336}
337
338impl<'a, 'll> DiagnosticHandlers<'a, 'll> {
339 pub(crate) fn new(
340 cgcx: &'a CodegenContext,
341 shared_emitter: &'a SharedEmitter,
342 llcx: llvm::Context<'ll>,
343 module: &ModuleCodegen<ModuleLlvm>,
344 stage: CodegenDiagnosticsStage,
345 ) -> Self {
346 let remark_passes_all: bool;
347 let remark_passes: Vec<CString>;
348 match &cgcx.remark {
349 Passes::All => {
350 remark_passes_all = true;
351 remark_passes = Vec::new();
352 }
353 Passes::Some(passes) => {
354 remark_passes_all = false;
355 remark_passes =
356 passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect();
357 }
358 };
359 let remark_passes: Vec<*const c_char> =
360 remark_passes.iter().map(|name: &CString| name.as_ptr()).collect();
361 let remark_file = cgcx
362 .remark_dir
363 .as_ref()
364 .map(|dir| {
366 let stage_suffix = match stage {
367 CodegenDiagnosticsStage::Codegen => "codegen",
368 CodegenDiagnosticsStage::Opt => "opt",
369 CodegenDiagnosticsStage::LTO => "lto",
370 };
371 dir.join(::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0}.{1}.opt.yaml", module.name,
stage_suffix))
})format!("{}.{stage_suffix}.opt.yaml", module.name))
372 })
373 .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
374
375 let pgo_available = cgcx.module_config.pgo_use.is_some();
376 let diagnostic_handler_callback_data =
377 Box::into_raw(Box::new(DiagnosticHandlerCallbackData { cgcx, shared_emitter }));
378 unsafe {
379 let old_handler = llvm::LLVMRustContextTakeDiagnosticHandler(llcx);
380 llvm::LLVMRustContextConfigureDiagnosticHandler(
381 llcx,
382 diagnostic_handler_callback,
383 diagnostic_handler_callback_data.cast::<c_void>(),
384 remark_passes_all,
385 remark_passes.as_ptr(),
386 remark_passes.len(),
387 remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()),
390 pgo_available,
391 );
392 DiagnosticHandlers { llcx, old_handler, diagnostic_handler_callback_data }
393 }
394 }
395}
396
397impl<'a, 'll> Drop for DiagnosticHandlers<'a, 'll> {
398 fn drop(&mut self) {
399 unsafe {
400 llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler);
401 drop(Box::from_raw(self.diagnostic_handler_callback_data));
402 }
403 }
404}
405
406fn report_inline_asm(
407 cgcx: &CodegenContext,
408 msg: String,
409 severity: llvm::DiagnosticSeverity,
410 cookie: u64,
411 source: Option<(String, Vec<InnerSpan>)>,
412) -> InlineAsmError {
413 let (lo, hi) = if cookie == 0 || #[allow(non_exhaustive_omitted_patterns)] match cgcx.lto {
Lto::Fat | Lto::Thin => true,
_ => false,
}matches!(cgcx.lto, Lto::Fat | Lto::Thin) {
417 (DUMMY_SP.lo(), DUMMY_SP.hi())
418 } else {
419 let lo = BytePos::from_u32(cookie as u32);
420 let hi = BytePos::from_u32((cookie >> 32) as u32);
421 (lo, hi)
422 };
423 let level = match severity {
424 llvm::DiagnosticSeverity::Error => Level::Error,
425 llvm::DiagnosticSeverity::Warning => Level::Warning(None),
426 llvm::DiagnosticSeverity::Note | llvm::DiagnosticSeverity::Remark => Level::Note,
427 };
428 let msg = msg.trim_prefix("error: ").to_string();
429 InlineAsmError { lo, hi, msg, level, source }
430}
431
432unsafe extern "C" fn diagnostic_handler_callback(info: &DiagnosticInfo, data: *mut c_void) {
433 if data.is_null() {
434 return;
435 }
436 diagnostic_handler_callback_inner(info, unsafe {
438 &*data.cast::<DiagnosticHandlerCallbackData<'_>>()
439 });
440}
441
442fn diagnostic_handler_callback_inner(
443 info: &DiagnosticInfo,
444 data: &DiagnosticHandlerCallbackData<'_>,
445) {
446 let &DiagnosticHandlerCallbackData { cgcx, shared_emitter } = data;
447
448 let dcx = DiagCtxt::new(Box::new(shared_emitter.clone()));
449 let dcx = dcx.handle();
450
451 match unsafe { llvm::diagnostic::Diagnostic::unpack(info) } {
452 llvm::diagnostic::InlineAsm(inline) => {
453 shared_emitter.inline_asm_error(report_inline_asm(
455 cgcx,
456 inline.message,
457 inline.severity,
458 inline.cookie,
459 inline.source,
460 ));
461 }
462
463 llvm::diagnostic::Optimization(opt) => {
464 dcx.emit_note(FromLlvmOptimizationDiag {
465 filename: &opt.filename,
466 line: opt.line,
467 column: opt.column,
468 pass_name: &opt.pass_name,
469 kind: match opt.kind {
470 OptimizationRemark => "success",
471 OptimizationMissed | OptimizationFailure => "missed",
472 OptimizationAnalysis
473 | OptimizationAnalysisFPCommute
474 | OptimizationAnalysisAliasing => "analysis",
475 OptimizationRemarkOther => "other",
476 },
477 message: &opt.message,
478 });
479 }
480
481 llvm::diagnostic::StackSize => {}
483
484 llvm::diagnostic::Other(diagnostic_ref) => {
485 let message = llvm::build_string(|s| unsafe {
486 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
487 })
488 .expect("non-UTF8 diagnostic");
489 match diagnostic_ref.severity() {
490 llvm::DiagnosticSeverity::Error => {
491 dcx.emit_err(FromLlvmDiag { message });
492 }
493 llvm::DiagnosticSeverity::Warning => {
494 dcx.emit_warn(FromLlvmDiag { message });
495 }
496 _ => {}
497 };
498 }
499 }
500}
501
502fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
503 match config.pgo_gen {
504 SwitchWithOptPath::Enabled(ref opt_dir_path) => {
505 let path = if let Some(dir_path) = opt_dir_path {
506 dir_path.join("default_%m.profraw")
507 } else {
508 PathBuf::from("default_%m.profraw")
509 };
510
511 Some(CString::new(::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0}", path.display()))
})format!("{}", path.display())).unwrap())
512 }
513 SwitchWithOptPath::Disabled => None,
514 }
515}
516
517fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
518 config
519 .pgo_use
520 .as_ref()
521 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
522}
523
524fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
525 config
526 .pgo_sample_use
527 .as_ref()
528 .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
529}
530
531fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
532 config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
533}
534
535#[derive(#[automatically_derived]impl ::core::fmt::Debug for AutodiffStage {
#[inline]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::write_str(f,
match self {
AutodiffStage::PreAD => "PreAD",
AutodiffStage::DuringAD => "DuringAD",
AutodiffStage::PostAD => "PostAD",
})
}
}Debug, #[automatically_derived]impl ::core::cmp::Eq for AutodiffStage { }Eq, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
AutodiffStage {
}
#[automatically_derived]impl ::core::cmp::PartialEq for AutodiffStage {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other)
}
}PartialEq)]
539pub(crate) enum AutodiffStage {
540 PreAD,
541 DuringAD,
542 PostAD,
543}
544
545pub(crate) unsafe fn llvm_optimize<'ll>(
546 cgcx: &CodegenContext,
547 prof: &SelfProfilerRef,
548 dcx: DiagCtxtHandle<'_>,
549 module_refs: ModuleRefs<'ll>,
550 thin_lto_buffer: Option<&mut Option<Buffer>>,
551 thin_lto_summary_buffer: Option<&mut Option<Buffer>>,
552 config: &ModuleConfig,
553 opt_level: config::OptLevel,
554 opt_stage: llvm::OptStage,
555 autodiff_stage: AutodiffStage,
556) {
557 let ModuleRefs { llcx, llmod, tm, .. } = module_refs;
558
559 let consider_ad = config.autodiff.contains(&config::AutoDiff::Enable);
568 let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
569 let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore);
570 let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter);
571 let print_passes = config.autodiff.contains(&config::AutoDiff::PrintPasses);
572 let passes_after_enzyme = if autodiff_stage == AutodiffStage::PostAD {
573 config.autodiff_post_passes.as_deref()
574 } else {
575 None
576 };
577
578 let merge_functions;
579 let unroll_loops;
580 let vectorize_slp;
581 let vectorize_loop;
582
583 if consider_ad && autodiff_stage != AutodiffStage::PostAD {
592 merge_functions = false;
593 unroll_loops = false;
594 vectorize_slp = false;
595 vectorize_loop = false;
596 } else {
597 unroll_loops =
598 opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
599 merge_functions = config.merge_functions;
600 vectorize_slp = config.vectorize_slp;
601 vectorize_loop = config.vectorize_loop;
602 }
603 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:603",
"rustc_codegen_llvm::back::write", ::tracing::Level::TRACE,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
::tracing_core::__macro_support::Option::Some(603u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
::tracing_core::field::FieldSet::new(&[{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("unroll_loops")
}> =
::tracing::__macro_support::FieldName::new("unroll_loops");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("vectorize_slp")
}> =
::tracing::__macro_support::FieldName::new("vectorize_slp");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("vectorize_loop")
}> =
::tracing::__macro_support::FieldName::new("vectorize_loop");
NAME.as_str()
},
{
const NAME:
::tracing::__macro_support::FieldName<{
::tracing::__macro_support::FieldName::len("run_enzyme")
}> =
::tracing::__macro_support::FieldName::new("run_enzyme");
NAME.as_str()
}], ::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::TRACE <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&unroll_loops)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&vectorize_slp)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&vectorize_loop)
as &dyn ::tracing::field::Value)),
(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&run_enzyme)
as &dyn ::tracing::field::Value))])
});
} else { ; }
};trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
604 if thin_lto_buffer.is_some() {
605 if !#[allow(non_exhaustive_omitted_patterns)] match opt_stage {
llvm::OptStage::PreLinkNoLTO | llvm::OptStage::PreLinkFatLTO |
llvm::OptStage::PreLinkThinLTO => true,
_ => false,
} {
{
::core::panicking::panic_fmt(format_args!("the bitcode for LTO can only be obtained at the pre-link stage"));
}
};assert!(
606 matches!(
607 opt_stage,
608 llvm::OptStage::PreLinkNoLTO
609 | llvm::OptStage::PreLinkFatLTO
610 | llvm::OptStage::PreLinkThinLTO
611 ),
612 "the bitcode for LTO can only be obtained at the pre-link stage"
613 );
614 }
615 let pgo_gen_path = get_pgo_gen_path(config);
616 let pgo_use_path = get_pgo_use_path(config);
617 let pgo_sample_use_path = get_pgo_sample_use_path(config);
618 let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
619 let is_final_stage =
620 !#[allow(non_exhaustive_omitted_patterns)] match opt_stage {
llvm::OptStage::PreLinkFatLTO | llvm::OptStage::PreLinkThinLTO => true,
_ => false,
}matches!(opt_stage, llvm::OptStage::PreLinkFatLTO | llvm::OptStage::PreLinkThinLTO);
621 let instr_profile_output_path = get_instr_profile_output_path(config);
622 let sanitize_dataflow_abilist: Vec<_> = config
623 .sanitizer_dataflow_abilist
624 .iter()
625 .map(|file| CString::new(file.as_str()).unwrap())
626 .collect();
627 let sanitize_dataflow_abilist_ptrs: Vec<_> =
628 sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect();
629 let sanitizer_options = if !is_lto {
631 Some(llvm::SanitizerOptions {
632 sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
633 sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
634 sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI),
635 sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW),
636 sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(),
637 sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(),
638 sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI),
639 sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
640 sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
641 sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
642 sanitize_realtime: config.sanitizer.contains(SanitizerSet::REALTIME),
643 sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
644 sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
645 sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
646 sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
647 sanitize_kernel_address_recover: config
648 .sanitizer_recover
649 .contains(SanitizerSet::KERNELADDRESS),
650 sanitize_kernel_hwaddress: config.sanitizer.contains(SanitizerSet::KERNELHWADDRESS),
651 sanitize_kernel_hwaddress_recover: config
652 .sanitizer_recover
653 .contains(SanitizerSet::KERNELHWADDRESS),
654 })
655 } else {
656 None
657 };
658
659 fn handle_offload<'ll>(
662 cx: &SimpleCx<'ll>,
663 old_fn: &'ll llvm::Value,
664 ) -> Option<&'ll llvm::Value> {
665 let old_fn_ty = cx.get_type_of_global(old_fn);
666 let old_param_types = cx.func_params_types(old_fn_ty);
667 let old_param_count = old_param_types.len();
668 if old_param_count == 0 {
669 return None;
670 }
671
672 let mut new_param_types = Vec::with_capacity(old_param_count as usize + 1);
674 new_param_types.push(cx.type_ptr());
675
676 for &old_ty in &old_param_types {
678 let new_ty = match cx.type_kind(old_ty) {
679 TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::Integer => {
680 cx.type_i64()
681 }
682 _ => old_ty,
683 };
684 new_param_types.push(new_ty);
685 }
686
687 let ret_ty = unsafe { llvm::LLVMGetReturnType(old_fn_ty) };
689 let new_fn_ty = cx.type_func(&new_param_types, ret_ty);
690
691 let old_fn_name = String::from_utf8(llvm::get_value_name(old_fn)).unwrap();
693 let new_fn_name = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!("{0}.offload", &old_fn_name))
})format!("{}.offload", &old_fn_name);
694 let new_fn = cx.add_func(&new_fn_name, new_fn_ty);
695 let a0 = llvm::get_param(new_fn, 0);
696 llvm::set_value_name(a0, CString::new("dyn_ptr").unwrap().as_bytes());
697
698 let bb = SBuilder::append_block(cx, new_fn, "entry");
699 let mut builder = SBuilder::build(cx, bb);
700
701 let mut old_args_rebuilt = Vec::with_capacity(old_param_types.len());
702
703 for (i, &old_ty) in old_param_types.iter().enumerate() {
704 let new_arg = llvm::get_param(new_fn, (i + 1) as u32);
705
706 let rebuilt = match cx.type_kind(old_ty) {
707 TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::Integer => {
708 let num_bits = scalar_width(cx, old_ty);
709
710 let trunc = builder.trunc(new_arg, cx.type_ix(num_bits));
711 builder.bitcast(trunc, old_ty)
712 }
713 _ => new_arg,
714 };
715
716 old_args_rebuilt.push(rebuilt);
717 }
718
719 builder.ret_void();
720
721 unsafe {
724 llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_wrapper(
725 old_fn,
726 new_fn,
727 old_args_rebuilt.as_slice(),
728 );
729 }
730
731 llvm::set_linkage(new_fn, llvm::get_linkage(old_fn));
732 llvm::set_visibility(new_fn, llvm::get_visibility(old_fn));
733
734 unsafe {
736 llvm::LLVMReplaceAllUsesWith(old_fn, new_fn);
737 }
738 let name = llvm::get_value_name(old_fn);
739 unsafe {
740 llvm::LLVMDeleteFunction(old_fn);
741 }
742 llvm::set_value_name(new_fn, &name);
744 Some(new_fn)
745 }
746
747 if cgcx.target_is_like_gpu
748 && config.offload.iter().any(|o| #[allow(non_exhaustive_omitted_patterns)] match o {
config::Offload::Device(_) => true,
_ => false,
}matches!(o, config::Offload::Device(_)))
749 {
750 let cx = SimpleCx::new(llmod, llcx, cgcx.pointer_size);
751 for func in cx.get_functions() {
752 let offload_kernel = "offload-kernel";
753 if !attributes::has_string_attr(func, offload_kernel) {
754 continue;
755 }
756 match handle_offload(&cx, func) {
757 Some(new_fn) => attributes::remove_string_attr_from_llfn(new_fn, offload_kernel),
758 None => attributes::remove_string_attr_from_llfn(func, offload_kernel),
759 }
760 }
761 }
762
763 let mut llvm_profiler = prof
764 .llvm_recording_enabled()
765 .then(|| LlvmSelfProfiler::new(prof.get_self_profiler().unwrap()));
766
767 let llvm_selfprofiler =
768 llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
769
770 let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() };
771
772 let llvm_plugins = config.llvm_plugins.join(",");
773
774 let enzyme_fn = if consider_ad {
775 let wrapper = llvm::EnzymeWrapper::get_instance();
776 wrapper.registerEnzymeAndPassPipeline
777 } else {
778 std::ptr::null()
779 };
780
781 let result = unsafe {
782 llvm::LLVMRustOptimize(
783 llmod,
784 tm,
785 to_pass_builder_opt_level(opt_level),
786 opt_stage,
787 cgcx.use_linker_plugin_lto,
788 config.no_prepopulate_passes,
789 config.verify_llvm_ir,
790 config.lint_llvm_ir,
791 thin_lto_buffer,
792 thin_lto_summary_buffer,
793 merge_functions,
794 unroll_loops,
795 vectorize_slp,
796 vectorize_loop,
797 config.no_builtins,
798 config.emit_lifetime_markers,
799 enzyme_fn,
800 print_before_enzyme,
801 print_after_enzyme,
802 print_passes,
803 sanitizer_options.as_ref(),
804 pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
805 pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
806 config.instrument_coverage,
807 instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
808 pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
809 config.debug_info_for_profiling,
810 llvm_selfprofiler,
811 selfprofile_before_pass_callback,
812 selfprofile_after_pass_callback,
813 passes_after_enzyme.map(|s| s.as_ptr()).unwrap_or(std::ptr::null()),
814 passes_after_enzyme.map(|s| s.len()).unwrap_or(0),
815 extra_passes.as_ptr(),
816 extra_passes.len(),
817 llvm_plugins.as_ptr(),
818 llvm_plugins.len(),
819 )
820 };
821
822 if !cgcx.target_is_like_gpu && is_final_stage {
827 if let Some(device_path) = config
828 .offload
829 .iter()
830 .find_map(|o| if let config::Offload::Host(path) = o { Some(path) } else { None })
831 {
832 let device_pathbuf = PathBuf::from(device_path);
833 if device_pathbuf.is_relative() {
834 dcx.emit_err(crate::diagnostics::OffloadWithoutAbsPath);
835 } else if !#[allow(non_exhaustive_omitted_patterns)] match device_pathbuf.extension().and_then(|e|
e.to_str()) {
Some("ptx" | "elf") => true,
_ => false,
}matches!(
836 device_pathbuf.extension().and_then(|e| e.to_str()),
837 Some("ptx" | "elf")
838 ) {
839 dcx.emit_err(crate::diagnostics::OffloadWrongFileName);
840 } else if !device_pathbuf.exists() {
841 dcx.emit_err(crate::diagnostics::OffloadNonexistingPath);
842 }
843 let device_image_c = path_to_c_string(device_pathbuf.as_path());
844 let ok = unsafe {
845 llvm::RustOffloadWrapper::get_instance()
846 .llvm_rust_offload_wrap_images(llmod, device_image_c.as_c_str())
847 };
848 if !ok {
849 dcx.emit_err(crate::diagnostics::OffloadWrapImagesFailed);
850 }
851 }
852 }
853 result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
854}
855
856pub(crate) fn optimize(
857 cgcx: &CodegenContext,
858 prof: &SelfProfilerRef,
859 shared_emitter: &SharedEmitter,
860 module: &ModuleCodegen<ModuleLlvm>,
861 config: &ModuleConfig,
862) {
863 module.module_llvm.with_module_refs(|module_refs| {
868 optimize_inner(cgcx, prof, shared_emitter, module, module_refs, config);
869 });
870}
871
872fn optimize_inner<'ll>(
873 cgcx: &CodegenContext,
874 prof: &SelfProfilerRef,
875 shared_emitter: &SharedEmitter,
876 module: &ModuleCodegen<ModuleLlvm>,
877 module_refs: ModuleRefs<'ll>, config: &ModuleConfig,
879) {
880 let _timer = prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name);
881
882 let dcx = DiagCtxt::new(Box::new(shared_emitter.clone()));
883 let dcx = dcx.handle();
884
885 let _handlers = DiagnosticHandlers::new(
886 cgcx,
887 shared_emitter,
888 module_refs.llcx,
889 module,
890 CodegenDiagnosticsStage::Opt,
891 );
892
893 if module.kind == ModuleKind::Regular {
894 save_temp_bitcode(cgcx, module, "no-opt");
895 }
896
897 if let Some(opt_level) = config.opt_level {
900 let opt_stage = match cgcx.lto {
901 Lto::Fat => llvm::OptStage::PreLinkFatLTO,
902 Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
903 _ if cgcx.use_linker_plugin_lto => llvm::OptStage::PreLinkThinLTO,
904 _ => llvm::OptStage::PreLinkNoLTO,
905 };
906
907 let consider_ad = config.autodiff.contains(&config::AutoDiff::Enable);
910 let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
911 let (mut thin_lto_buffer, mut thin_lto_summary_buffer) = if (module.kind
916 == ModuleKind::Regular
917 && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full))
918 || config.emit_thin_lto_summary
919 {
920 (Some(None), config.emit_thin_lto_summary.then_some(None))
921 } else {
922 (None, None)
923 };
924 unsafe {
925 llvm_optimize(
926 cgcx,
927 prof,
928 dcx,
929 module_refs,
930 thin_lto_buffer.as_mut(),
931 thin_lto_summary_buffer.as_mut(),
932 config,
933 opt_level,
934 opt_stage,
935 autodiff_stage,
936 )
937 };
938 if let Some(thin_lto_buffer) = thin_lto_buffer {
939 let thin_lto_buffer = thin_lto_buffer.unwrap();
940 module
941 .module_llvm
942 .thin_lto_buffer
943 .set(thin_lto_buffer.data().to_vec())
944 .expect("thin_lto_buffer should be unset");
945 let bc_summary_out =
946 cgcx.output_filenames.temp_path_for_cgu(OutputType::ThinLinkBitcode, &module.name);
947 if let Some(thin_lto_summary_buffer) = thin_lto_summary_buffer
948 && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name()
949 {
950 let thin_lto_summary_buffer = thin_lto_summary_buffer.unwrap();
951 let summary_data = thin_lto_summary_buffer.data();
952 prof.artifact_size(
953 "llvm_bitcode_summary",
954 thin_link_bitcode_filename.to_string_lossy(),
955 summary_data.len() as u64,
956 );
957 let _timer = prof.generic_activity_with_arg(
958 "LLVM_module_codegen_emit_bitcode_summary",
959 &*module.name,
960 );
961 if let Err(err) = fs::write(&bc_summary_out, summary_data) {
962 dcx.emit_err(WriteBytecode { path: &bc_summary_out, err });
963 }
964 }
965 }
966 }
967}
968
969pub(crate) fn codegen(
970 cgcx: &CodegenContext,
971 prof: &SelfProfilerRef,
972 shared_emitter: &SharedEmitter,
973 module: ModuleCodegen<ModuleLlvm>,
974 config: &ModuleConfig,
975) -> CompiledModule {
976 let _timer = prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
977
978 let dcx = DiagCtxt::new(Box::new(shared_emitter.clone()));
979 let dcx = dcx.handle();
980
981 module.module_llvm.with_module_refs(|ModuleRefs { llcx, llmod, tm, .. }| {
982 let _handlers = DiagnosticHandlers::new(
983 cgcx,
984 shared_emitter,
985 llcx,
986 &module,
987 CodegenDiagnosticsStage::Codegen,
988 );
989
990 if cgcx.msvc_imps_needed {
991 create_msvc_imps(cgcx, llcx, llmod);
992 }
993
994 let bc_out = cgcx.output_filenames.temp_path_for_cgu(OutputType::Bitcode, &module.name);
999 let obj_out = cgcx.output_filenames.temp_path_for_cgu(OutputType::Object, &module.name);
1000
1001 if config.bitcode_needed() {
1002 if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
1003 let thin = {
1004 let _timer = prof.generic_activity_with_arg(
1005 "LLVM_module_codegen_make_bitcode",
1006 &*module.name,
1007 );
1008 ModuleBuffer::new(llmod, cgcx.lto != Lto::Fat)
1009 };
1010 let data = thin.data();
1011 let _timer = prof
1012 .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
1013 if let Some(bitcode_filename) = bc_out.file_name() {
1014 prof.artifact_size(
1015 "llvm_bitcode",
1016 bitcode_filename.to_string_lossy(),
1017 data.len() as u64,
1018 );
1019 }
1020 if let Err(err) = fs::write(&bc_out, data) {
1021 dcx.emit_err(WriteBytecode { path: &bc_out, err });
1022 }
1023 }
1024
1025 if config.embed_bitcode() && module.kind == ModuleKind::Regular {
1026 let _timer = prof
1027 .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name);
1028 let thin_bc =
1029 module.module_llvm.thin_lto_buffer.get().expect("cannot find embedded bitcode");
1030 embed_bitcode(cgcx, llcx, llmod, &thin_bc);
1031 }
1032 }
1033
1034 if config.emit_ir {
1035 let _timer =
1036 prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name);
1037 let out =
1038 cgcx.output_filenames.temp_path_for_cgu(OutputType::LlvmAssembly, &module.name);
1039 let out_c = path_to_c_string(&out);
1040
1041 extern "C" fn demangle_callback(
1042 input_ptr: *const c_char,
1043 input_len: size_t,
1044 output_ptr: *mut c_char,
1045 output_len: size_t,
1046 ) -> size_t {
1047 let input =
1048 unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) };
1049
1050 let Ok(input) = str::from_utf8(input) else { return 0 };
1051
1052 let output = unsafe {
1053 slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize)
1054 };
1055 let mut cursor = io::Cursor::new(output);
1056
1057 let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 };
1058
1059 if cursor.write_fmt(format_args!("{0:#}", demangled))write!(cursor, "{demangled:#}").is_err() {
1060 return 0;
1062 }
1063
1064 cursor.position() as size_t
1065 }
1066
1067 let result =
1068 unsafe { llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback) };
1069
1070 if result == llvm::LLVMRustResult::Success {
1071 record_artifact_size(prof, "llvm_ir", &out);
1072 }
1073
1074 result
1075 .into_result()
1076 .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }));
1077 }
1078
1079 if config.emit_asm {
1080 let _timer =
1081 prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
1082 let path = cgcx.output_filenames.temp_path_for_cgu(OutputType::Assembly, &module.name);
1083
1084 let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj {
1089 llvm::LLVMCloneModule(llmod)
1090 } else {
1091 llmod
1092 };
1093 write_output_file(
1094 dcx,
1095 tm,
1096 config.no_builtins,
1097 llmod,
1098 &path,
1099 None,
1100 llvm::FileType::AssemblyFile,
1101 prof,
1102 config.verify_llvm_ir,
1103 );
1104 }
1105
1106 match config.emit_obj {
1107 EmitObj::ObjectCode(_) => {
1108 let _timer =
1109 prof.generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
1110
1111 let dwo_out = cgcx.output_filenames.temp_path_dwo_for_cgu(&module.name);
1112 let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) {
1113 (SplitDebuginfo::Off, _) => None,
1115 _ if !cgcx.target_can_use_split_dwarf => None,
1118 (_, SplitDwarfKind::Single) => None,
1121 (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()),
1124 };
1125
1126 write_output_file(
1127 dcx,
1128 tm,
1129 config.no_builtins,
1130 llmod,
1131 &obj_out,
1132 dwo_out,
1133 llvm::FileType::ObjectFile,
1134 prof,
1135 config.verify_llvm_ir,
1136 );
1137 }
1138
1139 EmitObj::Bitcode => {
1140 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:1140",
"rustc_codegen_llvm::back::write", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
::tracing_core::__macro_support::Option::Some(1140u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("copying bitcode {0:?} to obj {1:?}",
bc_out, obj_out) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
1141 if let Err(err) = link_or_copy(&bc_out, &obj_out) {
1142 dcx.emit_err(CopyBitcode { err });
1143 }
1144
1145 if !config.emit_bc {
1146 {
use ::tracing::__macro_support::Callsite as _;
static __CALLSITE: ::tracing::callsite::DefaultCallsite =
{
static META: ::tracing::Metadata<'static> =
{
::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs:1146",
"rustc_codegen_llvm::back::write", ::tracing::Level::DEBUG,
::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/back/write.rs"),
::tracing_core::__macro_support::Option::Some(1146u32),
::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::back::write"),
::tracing_core::field::FieldSet::new(&["message"],
::tracing_core::callsite::Identifier(&__CALLSITE)),
::tracing::metadata::Kind::EVENT)
};
::tracing::callsite::DefaultCallsite::new(&META)
};
let enabled =
::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
&&
::tracing::Level::DEBUG <=
::tracing::level_filters::LevelFilter::current() &&
{
let interest = __CALLSITE.interest();
!interest.is_never() &&
::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
interest)
};
if enabled {
(|value_set: ::tracing::field::ValueSet|
{
let meta = __CALLSITE.metadata();
::tracing::Event::dispatch(meta, &value_set);
;
})({
#[allow(unused_imports)]
use ::tracing::field::{debug, display, Value};
__CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("removing_bitcode {0:?}",
bc_out) as &dyn ::tracing::field::Value))])
});
} else { ; }
};debug!("removing_bitcode {:?}", bc_out);
1147 ensure_removed(dcx, &bc_out);
1148 }
1149 }
1150
1151 EmitObj::None => {}
1152 }
1153
1154 record_llvm_cgu_instructions_stats(prof, &module.name, llmod);
1155 });
1156
1157 let dwarf_object_emitted = #[allow(non_exhaustive_omitted_patterns)] match config.emit_obj {
EmitObj::ObjectCode(_) => true,
_ => false,
}matches!(config.emit_obj, EmitObj::ObjectCode(_))
1166 && cgcx.target_can_use_split_dwarf
1167 && cgcx.split_debuginfo != SplitDebuginfo::Off
1168 && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
1169 module.into_compiled_module(
1170 config.emit_obj != EmitObj::None,
1171 dwarf_object_emitted,
1172 config.emit_bc,
1173 config.emit_asm,
1174 config.emit_ir,
1175 &cgcx.output_filenames,
1176 )
1177}
1178
1179fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> {
1180 let mut asm = ::alloc::__export::must_use({
::alloc::fmt::format(format_args!(".section {0},\"{1}\"\n",
section_name, section_flags))
})format!(".section {section_name},\"{section_flags}\"\n").into_bytes();
1181 asm.extend_from_slice(b".ascii \"");
1182 asm.reserve(data.len());
1183 for &byte in data {
1184 if byte == b'\\' || byte == b'"' {
1185 asm.push(b'\\');
1186 asm.push(byte);
1187 } else if byte < 0x20 || byte >= 0x80 {
1188 asm.push(b'\\');
1191 asm.push(b'0' + ((byte >> 6) & 0x7));
1192 asm.push(b'0' + ((byte >> 3) & 0x7));
1193 asm.push(b'0' + ((byte >> 0) & 0x7));
1194 } else {
1195 asm.push(byte);
1196 }
1197 }
1198 asm.extend_from_slice(b"\"\n");
1199 asm
1200}
1201
1202pub(crate) fn bitcode_section_name(cgcx: &CodegenContext) -> &'static CStr {
1203 if cgcx.target_is_like_darwin {
1204 c"__LLVM,__bitcode"
1205 } else if cgcx.target_is_like_aix {
1206 c".ipa"
1207 } else {
1208 c".llvmbc"
1209 }
1210}
1211
1212fn embed_bitcode<'ll>(
1214 cgcx: &CodegenContext,
1215 llcx: llvm::Context<'ll>,
1216 llmod: llvm::Module<'ll>,
1217 bitcode: &[u8],
1218) {
1219 if cgcx.target_is_like_darwin
1258 || cgcx.target_is_like_aix
1259 || cgcx.target_arch == "wasm32"
1260 || cgcx.target_arch == "wasm64"
1261 {
1262 let llconst = common::bytes_in_context(llcx, bitcode);
1264 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.module");
1265 llvm::set_initializer(llglobal, llconst);
1266
1267 llvm::set_section(llglobal, bitcode_section_name(cgcx));
1268 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1269 llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE);
1270
1271 let llconst = common::bytes_in_context(llcx, &[]);
1272 let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline");
1273 llvm::set_initializer(llglobal, llconst);
1274 let section = if cgcx.target_is_like_darwin {
1275 c"__LLVM,__cmdline"
1276 } else if cgcx.target_is_like_aix {
1277 c".info"
1278 } else {
1279 c".llvmcmd"
1280 };
1281 llvm::set_section(llglobal, section);
1282 llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
1283 } else {
1284 let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
1286 let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
1287 llvm::append_module_inline_asm(llmod, &asm, "", "");
1288 let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]);
1289 llvm::append_module_inline_asm(llmod, &asm, "", "");
1290 }
1291}
1292
1293fn create_msvc_imps<'ll>(
1300 cgcx: &CodegenContext,
1301 llcx: llvm::Context<'ll>,
1302 llmod: llvm::Module<'ll>,
1303) {
1304 if !cgcx.msvc_imps_needed {
1305 return;
1306 }
1307 let prefix: &[u8] = if cgcx.target_arch == "x86" { b"\x01__imp__" } else { b"\x01__imp_" };
1312
1313 let ptr_ty = llvm_type_ptr(llcx);
1314 let symbols = std::iter::chain(
1315 base::iter_globals(llmod),
1316 base::iter_global_aliases(llmod).filter(|&val| {
1317 llvm::LLVMGetTypeKind(unsafe { llvm::LLVMGlobalGetValueType(val) }).to_rust()
1318 != llvm::TypeKind::Function
1319 }),
1320 )
1321 .map(|val| (val, llvm::get_linkage(val)))
1322 .filter(|&(val, linkage)| {
1323 #[allow(non_exhaustive_omitted_patterns)] match linkage {
llvm::Linkage::ExternalLinkage | llvm::Linkage::WeakAnyLinkage => true,
_ => false,
}matches!(linkage, llvm::Linkage::ExternalLinkage | llvm::Linkage::WeakAnyLinkage)
1324 && !llvm::is_declaration(val)
1325 })
1326 .collect::<Vec<_>>();
1327
1328 for (val, linkage) in symbols {
1329 let name = llvm::get_value_name(val);
1330 if ignored(&name) {
1332 continue;
1333 }
1334
1335 let mut imp_name = prefix.to_vec();
1336 imp_name.extend(name);
1337 let imp_name = CString::new(imp_name).unwrap();
1338
1339 let imp = llvm::add_global(llmod, ptr_ty, &imp_name);
1340
1341 llvm::set_initializer(imp, val);
1342 llvm::set_linkage(imp, linkage);
1343 }
1344
1345 fn ignored(symbol_name: &[u8]) -> bool {
1347 symbol_name.starts_with(b"__llvm_profile_")
1349 }
1350}
1351
1352fn record_artifact_size(
1353 self_profiler_ref: &SelfProfilerRef,
1354 artifact_kind: &'static str,
1355 path: &Path,
1356) {
1357 if !self_profiler_ref.enabled() {
1359 return;
1360 }
1361
1362 if let Some(artifact_name) = path.file_name() {
1363 let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
1364 self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
1365 }
1366}
1367
1368fn record_llvm_cgu_instructions_stats<'ll>(
1369 prof: &SelfProfilerRef,
1370 name: &str,
1371 llmod: llvm::Module<'ll>,
1372) {
1373 if !prof.enabled() {
1374 return;
1375 }
1376
1377 let total = unsafe { llvm::LLVMRustModuleInstructionStats(llmod) };
1378 prof.artifact_size("cgu_instructions", name, total);
1379}