1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_ast::attr;
10use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
11use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
12use rustc_data_structures::jobserver::{self, Acquired};
13use rustc_data_structures::memmap::Mmap;
14use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
15use rustc_errors::emitter::Emitter;
16use rustc_errors::translation::Translate;
17use rustc_errors::{
18 Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FluentBundle, Level, MultiSpan,
19 Style, Suggestions,
20};
21use rustc_fs_util::link_or_copy;
22use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
23use rustc_incremental::{
24 copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
25};
26use rustc_metadata::EncodedMetadata;
27use rustc_metadata::fs::copy_to_stdout;
28use rustc_middle::bug;
29use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
30use rustc_middle::middle::exported_symbols::SymbolExportInfo;
31use rustc_middle::ty::TyCtxt;
32use rustc_session::Session;
33use rustc_session::config::{
34 self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
35};
36use rustc_span::source_map::SourceMap;
37use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
38use rustc_target::spec::{MergeFunctions, SanitizerSet};
39use tracing::debug;
40
41use super::link::{self, ensure_removed};
42use super::lto::{self, SerializedModule};
43use super::symbol_export::symbol_name_for_instance_in_crate;
44use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
45use crate::traits::*;
46use crate::{
47 CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
48 errors,
49};
50
51const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
52
53#[derive(Clone, Copy, PartialEq)]
55pub enum EmitObj {
56 None,
58
59 Bitcode,
62
63 ObjectCode(BitcodeSection),
65}
66
67#[derive(Clone, Copy, PartialEq)]
69pub enum BitcodeSection {
70 None,
72
73 Full,
75}
76
77pub struct ModuleConfig {
79 pub passes: Vec<String>,
81 pub opt_level: Option<config::OptLevel>,
84
85 pub opt_size: Option<config::OptLevel>,
87
88 pub pgo_gen: SwitchWithOptPath,
89 pub pgo_use: Option<PathBuf>,
90 pub pgo_sample_use: Option<PathBuf>,
91 pub debug_info_for_profiling: bool,
92 pub instrument_coverage: bool,
93
94 pub sanitizer: SanitizerSet,
95 pub sanitizer_recover: SanitizerSet,
96 pub sanitizer_dataflow_abilist: Vec<String>,
97 pub sanitizer_memory_track_origins: usize,
98
99 pub emit_pre_lto_bc: bool,
101 pub emit_no_opt_bc: bool,
102 pub emit_bc: bool,
103 pub emit_ir: bool,
104 pub emit_asm: bool,
105 pub emit_obj: EmitObj,
106 pub emit_thin_lto: bool,
107 pub emit_thin_lto_summary: bool,
108 pub bc_cmdline: String,
109
110 pub verify_llvm_ir: bool,
113 pub lint_llvm_ir: bool,
114 pub no_prepopulate_passes: bool,
115 pub no_builtins: bool,
116 pub time_module: bool,
117 pub vectorize_loop: bool,
118 pub vectorize_slp: bool,
119 pub merge_functions: bool,
120 pub emit_lifetime_markers: bool,
121 pub llvm_plugins: Vec<String>,
122 pub autodiff: Vec<config::AutoDiff>,
123}
124
125impl ModuleConfig {
126 fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
127 macro_rules! if_regular {
130 ($regular: expr, $other: expr) => {
131 if let ModuleKind::Regular = kind { $regular } else { $other }
132 };
133 }
134
135 let sess = tcx.sess;
136 let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
137
138 let save_temps = sess.opts.cg.save_temps;
139
140 let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
141 || match kind {
142 ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
143 ModuleKind::Allocator => false,
144 ModuleKind::Metadata => sess.opts.output_types.contains_key(&OutputType::Metadata),
145 };
146
147 let emit_obj = if !should_emit_obj {
148 EmitObj::None
149 } else if sess.target.obj_is_bitcode
150 || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
151 {
152 EmitObj::Bitcode
167 } else if need_bitcode_in_object(tcx) {
168 EmitObj::ObjectCode(BitcodeSection::Full)
169 } else {
170 EmitObj::ObjectCode(BitcodeSection::None)
171 };
172
173 ModuleConfig {
174 passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
175
176 opt_level: opt_level_and_size,
177 opt_size: opt_level_and_size,
178
179 pgo_gen: if_regular!(
180 sess.opts.cg.profile_generate.clone(),
181 SwitchWithOptPath::Disabled
182 ),
183 pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
184 pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
185 debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
186 instrument_coverage: if_regular!(sess.instrument_coverage(), false),
187
188 sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
189 sanitizer_dataflow_abilist: if_regular!(
190 sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
191 Vec::new()
192 ),
193 sanitizer_recover: if_regular!(
194 sess.opts.unstable_opts.sanitizer_recover,
195 SanitizerSet::empty()
196 ),
197 sanitizer_memory_track_origins: if_regular!(
198 sess.opts.unstable_opts.sanitizer_memory_track_origins,
199 0
200 ),
201
202 emit_pre_lto_bc: if_regular!(
203 save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
204 false
205 ),
206 emit_no_opt_bc: if_regular!(save_temps, false),
207 emit_bc: if_regular!(
208 save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
209 save_temps
210 ),
211 emit_ir: if_regular!(
212 sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
213 false
214 ),
215 emit_asm: if_regular!(
216 sess.opts.output_types.contains_key(&OutputType::Assembly),
217 false
218 ),
219 emit_obj,
220 emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
221 emit_thin_lto_summary: if_regular!(
222 sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
223 false
224 ),
225 bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
226
227 verify_llvm_ir: sess.verify_llvm_ir(),
228 lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
229 no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
230 no_builtins: no_builtins || sess.target.no_builtins,
231
232 time_module: if_regular!(true, false),
235
236 vectorize_loop: !sess.opts.cg.no_vectorize_loops
239 && (sess.opts.optimize == config::OptLevel::More
240 || sess.opts.optimize == config::OptLevel::Aggressive),
241 vectorize_slp: !sess.opts.cg.no_vectorize_slp
242 && sess.opts.optimize == config::OptLevel::Aggressive,
243
244 merge_functions: match sess
254 .opts
255 .unstable_opts
256 .merge_functions
257 .unwrap_or(sess.target.merge_functions)
258 {
259 MergeFunctions::Disabled => false,
260 MergeFunctions::Trampolines | MergeFunctions::Aliases => {
261 use config::OptLevel::*;
262 match sess.opts.optimize {
263 Aggressive | More | SizeMin | Size => true,
264 Less | No => false,
265 }
266 }
267 },
268
269 emit_lifetime_markers: sess.emit_lifetime_markers(),
270 llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
271 autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
272 }
273 }
274
275 pub fn bitcode_needed(&self) -> bool {
276 self.emit_bc
277 || self.emit_thin_lto_summary
278 || self.emit_obj == EmitObj::Bitcode
279 || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
280 }
281}
282
283pub struct TargetMachineFactoryConfig {
285 pub split_dwarf_file: Option<PathBuf>,
289
290 pub output_obj_file: Option<PathBuf>,
293}
294
295impl TargetMachineFactoryConfig {
296 pub fn new(
297 cgcx: &CodegenContext<impl WriteBackendMethods>,
298 module_name: &str,
299 ) -> TargetMachineFactoryConfig {
300 let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
301 cgcx.output_filenames.split_dwarf_path(
302 cgcx.split_debuginfo,
303 cgcx.split_dwarf_kind,
304 Some(module_name),
305 )
306 } else {
307 None
308 };
309
310 let output_obj_file =
311 Some(cgcx.output_filenames.temp_path(OutputType::Object, Some(module_name)));
312 TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
313 }
314}
315
316pub type TargetMachineFactoryFn<B> = Arc<
317 dyn Fn(
318 TargetMachineFactoryConfig,
319 ) -> Result<
320 <B as WriteBackendMethods>::TargetMachine,
321 <B as WriteBackendMethods>::TargetMachineError,
322 > + Send
323 + Sync,
324>;
325
326type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
327
328#[derive(Clone)]
330pub struct CodegenContext<B: WriteBackendMethods> {
331 pub prof: SelfProfilerRef,
333 pub lto: Lto,
334 pub save_temps: bool,
335 pub fewer_names: bool,
336 pub time_trace: bool,
337 pub exported_symbols: Option<Arc<ExportedSymbols>>,
338 pub opts: Arc<config::Options>,
339 pub crate_types: Vec<CrateType>,
340 pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
341 pub output_filenames: Arc<OutputFilenames>,
342 pub regular_module_config: Arc<ModuleConfig>,
343 pub metadata_module_config: Arc<ModuleConfig>,
344 pub allocator_module_config: Arc<ModuleConfig>,
345 pub tm_factory: TargetMachineFactoryFn<B>,
346 pub msvc_imps_needed: bool,
347 pub is_pe_coff: bool,
348 pub target_can_use_split_dwarf: bool,
349 pub target_arch: String,
350 pub target_is_like_osx: bool,
351 pub target_is_like_aix: bool,
352 pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
353 pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
354
355 pub expanded_args: Vec<String>,
360
361 pub diag_emitter: SharedEmitter,
363 pub remark: Passes,
365 pub remark_dir: Option<PathBuf>,
368 pub incr_comp_session_dir: Option<PathBuf>,
371 pub coordinator_send: Sender<Box<dyn Any + Send>>,
373 pub parallel: bool,
377}
378
379impl<B: WriteBackendMethods> CodegenContext<B> {
380 pub fn create_dcx(&self) -> DiagCtxt {
381 DiagCtxt::new(Box::new(self.diag_emitter.clone()))
382 }
383
384 pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
385 match kind {
386 ModuleKind::Regular => &self.regular_module_config,
387 ModuleKind::Metadata => &self.metadata_module_config,
388 ModuleKind::Allocator => &self.allocator_module_config,
389 }
390 }
391}
392
393fn generate_lto_work<B: ExtraBackendMethods>(
394 cgcx: &CodegenContext<B>,
395 autodiff: Vec<AutoDiffItem>,
396 needs_fat_lto: Vec<FatLtoInput<B>>,
397 needs_thin_lto: Vec<(String, B::ThinBuffer)>,
398 import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
399) -> Vec<(WorkItem<B>, u64)> {
400 let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
401
402 if !needs_fat_lto.is_empty() {
403 assert!(needs_thin_lto.is_empty());
404 let mut module =
405 B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
406 if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
407 let config = cgcx.config(ModuleKind::Regular);
408 module = unsafe { module.autodiff(cgcx, autodiff, config).unwrap() };
409 }
410 vec![(WorkItem::LTO(module), 0)]
412 } else {
413 if !autodiff.is_empty() {
414 let dcx = cgcx.create_dcx();
415 dcx.handle().emit_fatal(AutodiffWithoutLto {});
416 }
417 assert!(needs_fat_lto.is_empty());
418 let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
419 .unwrap_or_else(|e| e.raise());
420 lto_modules
421 .into_iter()
422 .map(|module| {
423 let cost = module.cost();
424 (WorkItem::LTO(module), cost)
425 })
426 .chain(copy_jobs.into_iter().map(|wp| {
427 (
428 WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
429 name: wp.cgu_name.clone(),
430 source: wp,
431 }),
432 0, )
434 }))
435 .collect()
436 }
437}
438
439struct CompiledModules {
440 modules: Vec<CompiledModule>,
441 allocator_module: Option<CompiledModule>,
442}
443
444fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
445 let sess = tcx.sess;
446 sess.opts.cg.embed_bitcode
447 && tcx.crate_types().contains(&CrateType::Rlib)
448 && sess.opts.output_types.contains_key(&OutputType::Exe)
449}
450
451fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
452 if sess.opts.incremental.is_none() {
453 return false;
454 }
455
456 match sess.lto() {
457 Lto::No => false,
458 Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
459 }
460}
461
462pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
463 backend: B,
464 tcx: TyCtxt<'_>,
465 target_cpu: String,
466 metadata: EncodedMetadata,
467 metadata_module: Option<CompiledModule>,
468) -> OngoingCodegen<B> {
469 let (coordinator_send, coordinator_receive) = channel();
470
471 let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
472 let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
473
474 let crate_info = CrateInfo::new(tcx, target_cpu);
475
476 let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
477 let metadata_config = ModuleConfig::new(ModuleKind::Metadata, tcx, no_builtins);
478 let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
479
480 let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
481 let (codegen_worker_send, codegen_worker_receive) = channel();
482
483 let coordinator_thread = start_executing_work(
484 backend.clone(),
485 tcx,
486 &crate_info,
487 shared_emitter,
488 codegen_worker_send,
489 coordinator_receive,
490 Arc::new(regular_config),
491 Arc::new(metadata_config),
492 Arc::new(allocator_config),
493 coordinator_send.clone(),
494 );
495
496 OngoingCodegen {
497 backend,
498 metadata,
499 metadata_module,
500 crate_info,
501
502 codegen_worker_receive,
503 shared_emitter_main,
504 coordinator: Coordinator {
505 sender: coordinator_send,
506 future: Some(coordinator_thread),
507 phantom: PhantomData,
508 },
509 output_filenames: Arc::clone(tcx.output_filenames(())),
510 }
511}
512
513fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
514 sess: &Session,
515 compiled_modules: &CompiledModules,
516) -> FxIndexMap<WorkProductId, WorkProduct> {
517 let mut work_products = FxIndexMap::default();
518
519 if sess.opts.incremental.is_none() {
520 return work_products;
521 }
522
523 let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
524
525 for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
526 let mut files = Vec::new();
527 if let Some(object_file_path) = &module.object {
528 files.push((OutputType::Object.extension(), object_file_path.as_path()));
529 }
530 if let Some(dwarf_object_file_path) = &module.dwarf_object {
531 files.push(("dwo", dwarf_object_file_path.as_path()));
532 }
533 if let Some(path) = &module.assembly {
534 files.push((OutputType::Assembly.extension(), path.as_path()));
535 }
536 if let Some(path) = &module.llvm_ir {
537 files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
538 }
539 if let Some(path) = &module.bytecode {
540 files.push((OutputType::Bitcode.extension(), path.as_path()));
541 }
542 if let Some((id, product)) =
543 copy_cgu_workproduct_to_incr_comp_cache_dir(sess, &module.name, files.as_slice())
544 {
545 work_products.insert(id, product);
546 }
547 }
548
549 work_products
550}
551
552fn produce_final_output_artifacts(
553 sess: &Session,
554 compiled_modules: &CompiledModules,
555 crate_output: &OutputFilenames,
556) {
557 let mut user_wants_bitcode = false;
558 let mut user_wants_objects = false;
559
560 let copy_gracefully = |from: &Path, to: &OutFileName| match to {
562 OutFileName::Stdout => {
563 if let Err(e) = copy_to_stdout(from) {
564 sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
565 }
566 }
567 OutFileName::Real(path) => {
568 if let Err(e) = fs::copy(from, path) {
569 sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
570 }
571 }
572 };
573
574 let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
575 if compiled_modules.modules.len() == 1 {
576 let module_name = Some(&compiled_modules.modules[0].name[..]);
579 let path = crate_output.temp_path(output_type, module_name);
580 let output = crate_output.path(output_type);
581 if !output_type.is_text_output() && output.is_tty() {
582 sess.dcx()
583 .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
584 } else {
585 copy_gracefully(&path, &output);
586 }
587 if !sess.opts.cg.save_temps && !keep_numbered {
588 ensure_removed(sess.dcx(), &path);
590 }
591 } else {
592 let extension = crate_output
593 .temp_path(output_type, None)
594 .extension()
595 .unwrap()
596 .to_str()
597 .unwrap()
598 .to_owned();
599
600 if crate_output.outputs.contains_explicit_name(&output_type) {
601 sess.dcx().emit_warn(errors::IgnoringEmitPath { extension });
604 } else if crate_output.single_output_file.is_some() {
605 sess.dcx().emit_warn(errors::IgnoringOutput { extension });
608 } else {
609 }
613 }
614 };
615
616 for output_type in crate_output.outputs.keys() {
620 match *output_type {
621 OutputType::Bitcode => {
622 user_wants_bitcode = true;
623 copy_if_one_unit(OutputType::Bitcode, true);
627 }
628 OutputType::ThinLinkBitcode => {
629 copy_if_one_unit(OutputType::ThinLinkBitcode, false);
630 }
631 OutputType::LlvmAssembly => {
632 copy_if_one_unit(OutputType::LlvmAssembly, false);
633 }
634 OutputType::Assembly => {
635 copy_if_one_unit(OutputType::Assembly, false);
636 }
637 OutputType::Object => {
638 user_wants_objects = true;
639 copy_if_one_unit(OutputType::Object, true);
640 }
641 OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
642 }
643 }
644
645 if !sess.opts.cg.save_temps {
658 let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
674
675 let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
676
677 let keep_numbered_objects =
678 needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
679
680 for module in compiled_modules.modules.iter() {
681 if let Some(ref path) = module.object {
682 if !keep_numbered_objects {
683 ensure_removed(sess.dcx(), path);
684 }
685 }
686
687 if let Some(ref path) = module.dwarf_object {
688 if !keep_numbered_objects {
689 ensure_removed(sess.dcx(), path);
690 }
691 }
692
693 if let Some(ref path) = module.bytecode {
694 if !keep_numbered_bitcode {
695 ensure_removed(sess.dcx(), path);
696 }
697 }
698 }
699
700 if !user_wants_bitcode {
701 if let Some(ref allocator_module) = compiled_modules.allocator_module {
702 if let Some(ref path) = allocator_module.bytecode {
703 ensure_removed(sess.dcx(), path);
704 }
705 }
706 }
707 }
708
709 if sess.opts.json_artifact_notifications {
710 if compiled_modules.modules.len() == 1 {
711 compiled_modules.modules[0].for_each_output(|_path, ty| {
712 if sess.opts.output_types.contains_key(&ty) {
713 let descr = ty.shorthand();
714 let path = crate_output.path(ty);
717 sess.dcx().emit_artifact_notification(path.as_path(), descr);
718 }
719 });
720 } else {
721 for module in &compiled_modules.modules {
722 module.for_each_output(|path, ty| {
723 if sess.opts.output_types.contains_key(&ty) {
724 let descr = ty.shorthand();
725 sess.dcx().emit_artifact_notification(&path, descr);
726 }
727 });
728 }
729 }
730 }
731
732 }
738
739pub(crate) enum WorkItem<B: WriteBackendMethods> {
740 Optimize(ModuleCodegen<B::Module>),
742 CopyPostLtoArtifacts(CachedModuleCodegen),
745 LTO(lto::LtoModuleCodegen<B>),
747}
748
749impl<B: WriteBackendMethods> WorkItem<B> {
750 fn module_kind(&self) -> ModuleKind {
751 match *self {
752 WorkItem::Optimize(ref m) => m.kind,
753 WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
754 }
755 }
756
757 fn short_description(&self) -> String {
759 #[cfg(not(windows))]
763 fn desc(short: &str, _long: &str, name: &str) -> String {
764 assert_eq!(short.len(), 3);
784 let name = if let Some(index) = name.find("-cgu.") {
785 &name[index + 1..] } else {
787 name
788 };
789 format!("{short} {name}")
790 }
791
792 #[cfg(windows)]
794 fn desc(_short: &str, long: &str, name: &str) -> String {
795 format!("{long} {name}")
796 }
797
798 match self {
799 WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
800 WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
801 WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
802 }
803 }
804}
805
806pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
808 Finished(CompiledModule),
810
811 NeedsLink(ModuleCodegen<B::Module>),
814
815 NeedsFatLto(FatLtoInput<B>),
818
819 NeedsThinLto(String, B::ThinBuffer),
822}
823
824pub enum FatLtoInput<B: WriteBackendMethods> {
825 Serialized { name: String, buffer: B::ModuleBuffer },
826 InMemory(ModuleCodegen<B::Module>),
827}
828
829pub(crate) enum ComputedLtoType {
831 No,
832 Thin,
833 Fat,
834}
835
836pub(crate) fn compute_per_cgu_lto_type(
837 sess_lto: &Lto,
838 opts: &config::Options,
839 sess_crate_types: &[CrateType],
840 module_kind: ModuleKind,
841) -> ComputedLtoType {
842 if module_kind == ModuleKind::Metadata {
845 return ComputedLtoType::No;
846 }
847
848 let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
852
853 let is_allocator = module_kind == ModuleKind::Allocator;
858
859 let is_rlib = sess_crate_types.len() == 1 && sess_crate_types[0] == CrateType::Rlib;
868
869 match sess_lto {
870 Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
871 Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
872 Lto::Fat if !is_rlib => ComputedLtoType::Fat,
873 _ => ComputedLtoType::No,
874 }
875}
876
877fn execute_optimize_work_item<B: ExtraBackendMethods>(
878 cgcx: &CodegenContext<B>,
879 module: ModuleCodegen<B::Module>,
880 module_config: &ModuleConfig,
881) -> Result<WorkItemResult<B>, FatalError> {
882 let dcx = cgcx.create_dcx();
883 let dcx = dcx.handle();
884
885 unsafe {
886 B::optimize(cgcx, dcx, &module, module_config)?;
887 }
888
889 let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
895
896 let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
899 let filename = pre_lto_bitcode_filename(&module.name);
900 cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
901 } else {
902 None
903 };
904
905 match lto_type {
906 ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
907 ComputedLtoType::Thin => {
908 let (name, thin_buffer) = B::prepare_thin(module, false);
909 if let Some(path) = bitcode {
910 fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
911 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
912 });
913 }
914 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
915 }
916 ComputedLtoType::Fat => match bitcode {
917 Some(path) => {
918 let (name, buffer) = B::serialize_module(module);
919 fs::write(&path, buffer.data()).unwrap_or_else(|e| {
920 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
921 });
922 Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
923 }
924 None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
925 },
926 }
927}
928
929fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
930 cgcx: &CodegenContext<B>,
931 module: CachedModuleCodegen,
932 module_config: &ModuleConfig,
933) -> WorkItemResult<B> {
934 let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
935
936 let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
937 let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
938 debug!(
939 "copying preexisting module `{}` from {:?} to {}",
940 module.name,
941 source_file,
942 output_path.display()
943 );
944 match link_or_copy(&source_file, &output_path) {
945 Ok(_) => Some(output_path),
946 Err(error) => {
947 cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
948 source_file,
949 output_path,
950 error,
951 });
952 None
953 }
954 }
955 };
956
957 let dwarf_object =
958 module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
959 let dwarf_obj_out = cgcx
960 .output_filenames
961 .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name))
962 .expect(
963 "saved dwarf object in work product but `split_dwarf_path` returned `None`",
964 );
965 load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
966 });
967
968 let load_from_incr_cache = |perform, output_type: OutputType| {
969 if perform {
970 let saved_file = module.source.saved_files.get(output_type.extension())?;
971 let output_path = cgcx.output_filenames.temp_path(output_type, Some(&module.name));
972 load_from_incr_comp_dir(output_path, &saved_file)
973 } else {
974 None
975 }
976 };
977
978 let should_emit_obj = module_config.emit_obj != EmitObj::None;
979 let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
980 let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
981 let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
982 let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
983 if should_emit_obj && object.is_none() {
984 cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
985 }
986
987 WorkItemResult::Finished(CompiledModule {
988 name: module.name,
989 kind: ModuleKind::Regular,
990 object,
991 dwarf_object,
992 bytecode,
993 assembly,
994 llvm_ir,
995 })
996}
997
998fn execute_lto_work_item<B: ExtraBackendMethods>(
999 cgcx: &CodegenContext<B>,
1000 module: lto::LtoModuleCodegen<B>,
1001 module_config: &ModuleConfig,
1002) -> Result<WorkItemResult<B>, FatalError> {
1003 let module = unsafe { module.optimize(cgcx)? };
1004 finish_intra_module_work(cgcx, module, module_config)
1005}
1006
1007fn finish_intra_module_work<B: ExtraBackendMethods>(
1008 cgcx: &CodegenContext<B>,
1009 module: ModuleCodegen<B::Module>,
1010 module_config: &ModuleConfig,
1011) -> Result<WorkItemResult<B>, FatalError> {
1012 let dcx = cgcx.create_dcx();
1013 let dcx = dcx.handle();
1014
1015 if !cgcx.opts.unstable_opts.combine_cgu
1016 || module.kind == ModuleKind::Metadata
1017 || module.kind == ModuleKind::Allocator
1018 {
1019 let module = unsafe { B::codegen(cgcx, dcx, module, module_config)? };
1020 Ok(WorkItemResult::Finished(module))
1021 } else {
1022 Ok(WorkItemResult::NeedsLink(module))
1023 }
1024}
1025
1026pub(crate) enum Message<B: WriteBackendMethods> {
1028 Token(io::Result<Acquired>),
1031
1032 WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1035
1036 AddAutoDiffItems(Vec<AutoDiffItem>),
1038
1039 CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1043
1044 AddImportOnlyModule {
1047 module_data: SerializedModule<B::ModuleBuffer>,
1048 work_product: WorkProduct,
1049 },
1050
1051 CodegenComplete,
1054
1055 CodegenAborted,
1058}
1059
1060pub struct CguMessage;
1063
1064struct Diagnostic {
1074 level: Level,
1075 messages: Vec<(DiagMessage, Style)>,
1076 code: Option<ErrCode>,
1077 children: Vec<Subdiagnostic>,
1078 args: DiagArgMap,
1079}
1080
1081pub(crate) struct Subdiagnostic {
1085 level: Level,
1086 messages: Vec<(DiagMessage, Style)>,
1087}
1088
1089#[derive(PartialEq, Clone, Copy, Debug)]
1090enum MainThreadState {
1091 Idle,
1093
1094 Codegenning,
1096
1097 Lending,
1099}
1100
1101fn start_executing_work<B: ExtraBackendMethods>(
1102 backend: B,
1103 tcx: TyCtxt<'_>,
1104 crate_info: &CrateInfo,
1105 shared_emitter: SharedEmitter,
1106 codegen_worker_send: Sender<CguMessage>,
1107 coordinator_receive: Receiver<Box<dyn Any + Send>>,
1108 regular_config: Arc<ModuleConfig>,
1109 metadata_config: Arc<ModuleConfig>,
1110 allocator_config: Arc<ModuleConfig>,
1111 tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1112) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1113 let coordinator_send = tx_to_llvm_workers;
1114 let sess = tcx.sess;
1115
1116 let mut each_linked_rlib_for_lto = Vec::new();
1117 drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1118 if link::ignored_for_lto(sess, crate_info, cnum) {
1119 return;
1120 }
1121 each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1122 }));
1123
1124 let exported_symbols = {
1126 let mut exported_symbols = FxHashMap::default();
1127
1128 let copy_symbols = |cnum| {
1129 let symbols = tcx
1130 .exported_symbols(cnum)
1131 .iter()
1132 .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1133 .collect();
1134 Arc::new(symbols)
1135 };
1136
1137 match sess.lto() {
1138 Lto::No => None,
1139 Lto::ThinLocal => {
1140 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1141 Some(Arc::new(exported_symbols))
1142 }
1143 Lto::Fat | Lto::Thin => {
1144 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1145 for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1146 exported_symbols.insert(cnum, copy_symbols(cnum));
1147 }
1148 Some(Arc::new(exported_symbols))
1149 }
1150 }
1151 };
1152
1153 let coordinator_send2 = coordinator_send.clone();
1159 let helper = jobserver::client()
1160 .into_helper_thread(move |token| {
1161 drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1162 })
1163 .expect("failed to spawn helper thread");
1164
1165 let ol =
1166 if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1167 config::OptLevel::No
1169 } else {
1170 tcx.backend_optimization_level(())
1171 };
1172 let backend_features = tcx.global_backend_features(());
1173
1174 let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1175 let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1176 match result {
1177 Ok(dir) => Some(dir),
1178 Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1179 }
1180 } else {
1181 None
1182 };
1183
1184 let cgcx = CodegenContext::<B> {
1185 crate_types: tcx.crate_types().to_vec(),
1186 each_linked_rlib_for_lto,
1187 lto: sess.lto(),
1188 fewer_names: sess.fewer_names(),
1189 save_temps: sess.opts.cg.save_temps,
1190 time_trace: sess.opts.unstable_opts.llvm_time_trace,
1191 opts: Arc::new(sess.opts.clone()),
1192 prof: sess.prof.clone(),
1193 exported_symbols,
1194 remark: sess.opts.cg.remark.clone(),
1195 remark_dir,
1196 incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1197 coordinator_send,
1198 expanded_args: tcx.sess.expanded_args.clone(),
1199 diag_emitter: shared_emitter.clone(),
1200 output_filenames: Arc::clone(tcx.output_filenames(())),
1201 regular_module_config: regular_config,
1202 metadata_module_config: metadata_config,
1203 allocator_module_config: allocator_config,
1204 tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1205 msvc_imps_needed: msvc_imps_needed(tcx),
1206 is_pe_coff: tcx.sess.target.is_like_windows,
1207 target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1208 target_arch: tcx.sess.target.arch.to_string(),
1209 target_is_like_osx: tcx.sess.target.is_like_osx,
1210 target_is_like_aix: tcx.sess.target.is_like_aix,
1211 split_debuginfo: tcx.sess.split_debuginfo(),
1212 split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1213 parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1214 };
1215
1216 return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1352 let mut worker_id_counter = 0;
1353 let mut free_worker_ids = Vec::new();
1354 let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1355 if let Some(id) = free_worker_ids.pop() {
1356 id
1357 } else {
1358 let id = worker_id_counter;
1359 worker_id_counter += 1;
1360 id
1361 }
1362 };
1363
1364 let mut autodiff_items = Vec::new();
1367 let mut compiled_modules = vec![];
1368 let mut compiled_allocator_module = None;
1369 let mut needs_link = Vec::new();
1370 let mut needs_fat_lto = Vec::new();
1371 let mut needs_thin_lto = Vec::new();
1372 let mut lto_import_only_modules = Vec::new();
1373 let mut started_lto = false;
1374
1375 #[derive(Debug, PartialEq)]
1380 enum CodegenState {
1381 Ongoing,
1382 Completed,
1383 Aborted,
1384 }
1385 use CodegenState::*;
1386 let mut codegen_state = Ongoing;
1387
1388 let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1390
1391 let mut tokens = Vec::new();
1394
1395 let mut main_thread_state = MainThreadState::Idle;
1396
1397 let mut running_with_own_token = 0;
1400
1401 let running_with_any_token = |main_thread_state, running_with_own_token| {
1404 running_with_own_token
1405 + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1406 };
1407
1408 let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1409
1410 loop {
1416 if codegen_state == Ongoing {
1420 if main_thread_state == MainThreadState::Idle {
1421 let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1429 let additional_running = std::cmp::min(extra_tokens, work_items.len());
1430 let anticipated_running = running_with_own_token + additional_running + 1;
1431
1432 if !queue_full_enough(work_items.len(), anticipated_running) {
1433 if codegen_worker_send.send(CguMessage).is_err() {
1435 panic!("Could not send CguMessage to main thread")
1436 }
1437 main_thread_state = MainThreadState::Codegenning;
1438 } else {
1439 let (item, _) =
1443 work_items.pop().expect("queue empty - queue_full_enough() broken?");
1444 main_thread_state = MainThreadState::Lending;
1445 spawn_work(
1446 &cgcx,
1447 &mut llvm_start_time,
1448 get_worker_id(&mut free_worker_ids),
1449 item,
1450 );
1451 }
1452 }
1453 } else if codegen_state == Completed {
1454 if running_with_any_token(main_thread_state, running_with_own_token) == 0
1455 && work_items.is_empty()
1456 {
1457 if needs_fat_lto.is_empty()
1459 && needs_thin_lto.is_empty()
1460 && lto_import_only_modules.is_empty()
1461 {
1462 break;
1464 }
1465
1466 assert!(!started_lto);
1472 started_lto = true;
1473
1474 let needs_fat_lto = mem::take(&mut needs_fat_lto);
1475 let needs_thin_lto = mem::take(&mut needs_thin_lto);
1476 let import_only_modules = mem::take(&mut lto_import_only_modules);
1477
1478 for (work, cost) in generate_lto_work(
1479 &cgcx,
1480 autodiff_items.clone(),
1481 needs_fat_lto,
1482 needs_thin_lto,
1483 import_only_modules,
1484 ) {
1485 let insertion_index = work_items
1486 .binary_search_by_key(&cost, |&(_, cost)| cost)
1487 .unwrap_or_else(|e| e);
1488 work_items.insert(insertion_index, (work, cost));
1489 if cgcx.parallel {
1490 helper.request_token();
1491 }
1492 }
1493 }
1494
1495 match main_thread_state {
1499 MainThreadState::Idle => {
1500 if let Some((item, _)) = work_items.pop() {
1501 main_thread_state = MainThreadState::Lending;
1502 spawn_work(
1503 &cgcx,
1504 &mut llvm_start_time,
1505 get_worker_id(&mut free_worker_ids),
1506 item,
1507 );
1508 } else {
1509 assert!(running_with_own_token > 0);
1516 running_with_own_token -= 1;
1517 main_thread_state = MainThreadState::Lending;
1518 }
1519 }
1520 MainThreadState::Codegenning => bug!(
1521 "codegen worker should not be codegenning after \
1522 codegen was already completed"
1523 ),
1524 MainThreadState::Lending => {
1525 }
1527 }
1528 } else {
1529 assert!(codegen_state == Aborted);
1532 if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1533 break;
1534 }
1535 }
1536
1537 if codegen_state != Aborted {
1540 while !work_items.is_empty() && running_with_own_token < tokens.len() {
1541 let (item, _) = work_items.pop().unwrap();
1542 spawn_work(
1543 &cgcx,
1544 &mut llvm_start_time,
1545 get_worker_id(&mut free_worker_ids),
1546 item,
1547 );
1548 running_with_own_token += 1;
1549 }
1550 }
1551
1552 tokens.truncate(running_with_own_token);
1554
1555 let mut free_worker = |worker_id| {
1561 if main_thread_state == MainThreadState::Lending {
1562 main_thread_state = MainThreadState::Idle;
1563 } else {
1564 running_with_own_token -= 1;
1565 }
1566
1567 free_worker_ids.push(worker_id);
1568 };
1569
1570 let msg = coordinator_receive.recv().unwrap();
1571 match *msg.downcast::<Message<B>>().ok().unwrap() {
1572 Message::Token(token) => {
1576 match token {
1577 Ok(token) => {
1578 tokens.push(token);
1579
1580 if main_thread_state == MainThreadState::Lending {
1581 main_thread_state = MainThreadState::Idle;
1586 running_with_own_token += 1;
1587 }
1588 }
1589 Err(e) => {
1590 let msg = &format!("failed to acquire jobserver token: {e}");
1591 shared_emitter.fatal(msg);
1592 codegen_state = Aborted;
1593 }
1594 }
1595 }
1596
1597 Message::CodegenDone { llvm_work_item, cost } => {
1598 let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1607 let insertion_index = match insertion_index {
1608 Ok(idx) | Err(idx) => idx,
1609 };
1610 work_items.insert(insertion_index, (llvm_work_item, cost));
1611
1612 if cgcx.parallel {
1613 helper.request_token();
1614 }
1615 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1616 main_thread_state = MainThreadState::Idle;
1617 }
1618
1619 Message::AddAutoDiffItems(mut items) => {
1620 autodiff_items.append(&mut items);
1621 }
1622
1623 Message::CodegenComplete => {
1624 if codegen_state != Aborted {
1625 codegen_state = Completed;
1626 }
1627 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1628 main_thread_state = MainThreadState::Idle;
1629 }
1630
1631 Message::CodegenAborted => {
1639 codegen_state = Aborted;
1640 }
1641
1642 Message::WorkItem { result, worker_id } => {
1643 free_worker(worker_id);
1644
1645 match result {
1646 Ok(WorkItemResult::Finished(compiled_module)) => {
1647 match compiled_module.kind {
1648 ModuleKind::Regular => {
1649 assert!(needs_link.is_empty());
1650 compiled_modules.push(compiled_module);
1651 }
1652 ModuleKind::Allocator => {
1653 assert!(compiled_allocator_module.is_none());
1654 compiled_allocator_module = Some(compiled_module);
1655 }
1656 ModuleKind::Metadata => bug!("Should be handled separately"),
1657 }
1658 }
1659 Ok(WorkItemResult::NeedsLink(module)) => {
1660 assert!(compiled_modules.is_empty());
1661 needs_link.push(module);
1662 }
1663 Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1664 assert!(!started_lto);
1665 assert!(needs_thin_lto.is_empty());
1666 needs_fat_lto.push(fat_lto_input);
1667 }
1668 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1669 assert!(!started_lto);
1670 assert!(needs_fat_lto.is_empty());
1671 needs_thin_lto.push((name, thin_buffer));
1672 }
1673 Err(Some(WorkerFatalError)) => {
1674 codegen_state = Aborted;
1676 }
1677 Err(None) => {
1678 bug!("worker thread panicked");
1681 }
1682 }
1683 }
1684
1685 Message::AddImportOnlyModule { module_data, work_product } => {
1686 assert!(!started_lto);
1687 assert_eq!(codegen_state, Ongoing);
1688 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1689 lto_import_only_modules.push((module_data, work_product));
1690 main_thread_state = MainThreadState::Idle;
1691 }
1692 }
1693 }
1694
1695 if codegen_state == Aborted {
1696 return Err(());
1697 }
1698
1699 let needs_link = mem::take(&mut needs_link);
1700 if !needs_link.is_empty() {
1701 assert!(compiled_modules.is_empty());
1702 let dcx = cgcx.create_dcx();
1703 let dcx = dcx.handle();
1704 let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1705 let module = unsafe {
1706 B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?
1707 };
1708 compiled_modules.push(module);
1709 }
1710
1711 drop(llvm_start_time);
1713
1714 compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1718
1719 Ok(CompiledModules {
1720 modules: compiled_modules,
1721 allocator_module: compiled_allocator_module,
1722 })
1723 })
1724 .expect("failed to spawn coordinator thread");
1725
1726 fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1729 let quarter_of_workers = workers_running - 3 * workers_running / 4;
1780 items_in_queue > 0 && items_in_queue >= quarter_of_workers
1781 }
1782}
1783
1784#[must_use]
1786pub(crate) struct WorkerFatalError;
1787
1788fn spawn_work<'a, B: ExtraBackendMethods>(
1789 cgcx: &'a CodegenContext<B>,
1790 llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1791 worker_id: usize,
1792 work: WorkItem<B>,
1793) {
1794 if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1795 *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1796 }
1797
1798 let cgcx = cgcx.clone();
1799
1800 B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1801 struct Bomb<B: ExtraBackendMethods> {
1804 coordinator_send: Sender<Box<dyn Any + Send>>,
1805 result: Option<Result<WorkItemResult<B>, FatalError>>,
1806 worker_id: usize,
1807 }
1808 impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1809 fn drop(&mut self) {
1810 let worker_id = self.worker_id;
1811 let msg = match self.result.take() {
1812 Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1813 Some(Err(FatalError)) => {
1814 Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1815 }
1816 None => Message::WorkItem::<B> { result: Err(None), worker_id },
1817 };
1818 drop(self.coordinator_send.send(Box::new(msg)));
1819 }
1820 }
1821
1822 let mut bomb =
1823 Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1824
1825 bomb.result = {
1832 let module_config = cgcx.config(work.module_kind());
1833
1834 Some(match work {
1835 WorkItem::Optimize(m) => {
1836 let _timer =
1837 cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1838 execute_optimize_work_item(&cgcx, m, module_config)
1839 }
1840 WorkItem::CopyPostLtoArtifacts(m) => {
1841 let _timer = cgcx.prof.generic_activity_with_arg(
1842 "codegen_copy_artifacts_from_incr_cache",
1843 &*m.name,
1844 );
1845 Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1846 }
1847 WorkItem::LTO(m) => {
1848 let _timer =
1849 cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1850 execute_lto_work_item(&cgcx, m, module_config)
1851 }
1852 })
1853 };
1854 })
1855 .expect("failed to spawn work thread");
1856}
1857
1858enum SharedEmitterMessage {
1859 Diagnostic(Diagnostic),
1860 InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1861 Fatal(String),
1862}
1863
1864#[derive(Clone)]
1865pub struct SharedEmitter {
1866 sender: Sender<SharedEmitterMessage>,
1867}
1868
1869pub struct SharedEmitterMain {
1870 receiver: Receiver<SharedEmitterMessage>,
1871}
1872
1873impl SharedEmitter {
1874 fn new() -> (SharedEmitter, SharedEmitterMain) {
1875 let (sender, receiver) = channel();
1876
1877 (SharedEmitter { sender }, SharedEmitterMain { receiver })
1878 }
1879
1880 pub fn inline_asm_error(
1881 &self,
1882 span: SpanData,
1883 msg: String,
1884 level: Level,
1885 source: Option<(String, Vec<InnerSpan>)>,
1886 ) {
1887 drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1888 }
1889
1890 fn fatal(&self, msg: &str) {
1891 drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1892 }
1893}
1894
1895impl Translate for SharedEmitter {
1896 fn fluent_bundle(&self) -> Option<&FluentBundle> {
1897 None
1898 }
1899
1900 fn fallback_fluent_bundle(&self) -> &FluentBundle {
1901 panic!("shared emitter attempted to translate a diagnostic");
1902 }
1903}
1904
1905impl Emitter for SharedEmitter {
1906 fn emit_diagnostic(
1907 &mut self,
1908 mut diag: rustc_errors::DiagInner,
1909 _registry: &rustc_errors::registry::Registry,
1910 ) {
1911 assert_eq!(diag.span, MultiSpan::new());
1914 assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1915 assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1916 assert_eq!(diag.is_lint, None);
1917 let args = mem::replace(&mut diag.args, DiagArgMap::default());
1920 drop(
1921 self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1922 level: diag.level(),
1923 messages: diag.messages,
1924 code: diag.code,
1925 children: diag
1926 .children
1927 .into_iter()
1928 .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1929 .collect(),
1930 args,
1931 })),
1932 );
1933 }
1934
1935 fn source_map(&self) -> Option<&SourceMap> {
1936 None
1937 }
1938}
1939
1940impl SharedEmitterMain {
1941 fn check(&self, sess: &Session, blocking: bool) {
1942 loop {
1943 let message = if blocking {
1944 match self.receiver.recv() {
1945 Ok(message) => Ok(message),
1946 Err(_) => Err(()),
1947 }
1948 } else {
1949 match self.receiver.try_recv() {
1950 Ok(message) => Ok(message),
1951 Err(_) => Err(()),
1952 }
1953 };
1954
1955 match message {
1956 Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1957 let dcx = sess.dcx();
1960 let mut d =
1961 rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1962 d.code = diag.code; d.children = diag
1964 .children
1965 .into_iter()
1966 .map(|sub| rustc_errors::Subdiag {
1967 level: sub.level,
1968 messages: sub.messages,
1969 span: MultiSpan::new(),
1970 })
1971 .collect();
1972 d.args = diag.args;
1973 dcx.emit_diagnostic(d);
1974 sess.dcx().abort_if_errors();
1975 }
1976 Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1977 assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1978 let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1979 if !span.is_dummy() {
1980 err.span(span.span());
1981 }
1982
1983 if let Some((buffer, spans)) = source {
1985 let source = sess
1986 .source_map()
1987 .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
1988 let spans: Vec<_> = spans
1989 .iter()
1990 .map(|sp| {
1991 Span::with_root_ctxt(
1992 source.normalized_byte_pos(sp.start as u32),
1993 source.normalized_byte_pos(sp.end as u32),
1994 )
1995 })
1996 .collect();
1997 err.span_note(spans, "instantiated into assembly here");
1998 }
1999
2000 err.emit();
2001 }
2002 Ok(SharedEmitterMessage::Fatal(msg)) => {
2003 sess.dcx().fatal(msg);
2004 }
2005 Err(_) => {
2006 break;
2007 }
2008 }
2009 }
2010 }
2011}
2012
2013pub struct Coordinator<B: ExtraBackendMethods> {
2014 pub sender: Sender<Box<dyn Any + Send>>,
2015 future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2016 phantom: PhantomData<B>,
2018}
2019
2020impl<B: ExtraBackendMethods> Coordinator<B> {
2021 fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2022 self.future.take().unwrap().join()
2023 }
2024}
2025
2026impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2027 fn drop(&mut self) {
2028 if let Some(future) = self.future.take() {
2029 drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2032 drop(future.join());
2033 }
2034 }
2035}
2036
2037pub struct OngoingCodegen<B: ExtraBackendMethods> {
2038 pub backend: B,
2039 pub metadata: EncodedMetadata,
2040 pub metadata_module: Option<CompiledModule>,
2041 pub crate_info: CrateInfo,
2042 pub codegen_worker_receive: Receiver<CguMessage>,
2043 pub shared_emitter_main: SharedEmitterMain,
2044 pub output_filenames: Arc<OutputFilenames>,
2045 pub coordinator: Coordinator<B>,
2046}
2047
2048impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2049 pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2050 self.shared_emitter_main.check(sess, true);
2051 let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2052 Ok(Ok(compiled_modules)) => compiled_modules,
2053 Ok(Err(())) => {
2054 sess.dcx().abort_if_errors();
2055 panic!("expected abort due to worker thread errors")
2056 }
2057 Err(_) => {
2058 bug!("panic during codegen/LLVM phase");
2059 }
2060 });
2061
2062 sess.dcx().abort_if_errors();
2063
2064 let work_products =
2065 copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2066 produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2067
2068 if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2071 self.backend.print_pass_timings()
2072 }
2073
2074 if sess.print_llvm_stats() {
2075 self.backend.print_statistics()
2076 }
2077
2078 (
2079 CodegenResults {
2080 metadata: self.metadata,
2081 crate_info: self.crate_info,
2082
2083 modules: compiled_modules.modules,
2084 allocator_module: compiled_modules.allocator_module,
2085 metadata_module: self.metadata_module,
2086 },
2087 work_products,
2088 )
2089 }
2090
2091 pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2092 self.wait_for_signal_to_codegen_item();
2093 self.check_for_errors(tcx.sess);
2094 drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2095 }
2096
2097 pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2098 drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2099 }
2100
2101 pub(crate) fn check_for_errors(&self, sess: &Session) {
2102 self.shared_emitter_main.check(sess, false);
2103 }
2104
2105 pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2106 match self.codegen_worker_receive.recv() {
2107 Ok(CguMessage) => {
2108 }
2110 Err(_) => {
2111 }
2114 }
2115 }
2116}
2117
2118pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2119 _backend: &B,
2120 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2121 module: ModuleCodegen<B::Module>,
2122 cost: u64,
2123) {
2124 let llvm_work_item = WorkItem::Optimize(module);
2125 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2126}
2127
2128pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2129 _backend: &B,
2130 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2131 module: CachedModuleCodegen,
2132) {
2133 let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2134 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2135}
2136
2137pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2138 _backend: &B,
2139 tcx: TyCtxt<'_>,
2140 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2141 module: CachedModuleCodegen,
2142) {
2143 let filename = pre_lto_bitcode_filename(&module.name);
2144 let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2145 let file = fs::File::open(&bc_path)
2146 .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2147
2148 let mmap = unsafe {
2149 Mmap::map(file).unwrap_or_else(|e| {
2150 panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2151 })
2152 };
2153 drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2155 module_data: SerializedModule::FromUncompressedFile(mmap),
2156 work_product: module.source,
2157 })));
2158}
2159
2160fn pre_lto_bitcode_filename(module_name: &str) -> String {
2161 format!("{module_name}.{PRE_LTO_BC_EXT}")
2162}
2163
2164fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2165 assert!(
2168 !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2169 && tcx.sess.target.is_like_windows
2170 && tcx.sess.opts.cg.prefer_dynamic)
2171 );
2172
2173 let can_have_static_objects =
2177 tcx.sess.lto() == Lto::Thin || tcx.crate_types().iter().any(|ct| *ct == CrateType::Rlib);
2178
2179 tcx.sess.target.is_like_windows &&
2180 can_have_static_objects &&
2181 !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2185}