rustc_codegen_ssa/back/
write.rs

1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_ast::attr;
10use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
11use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
12use rustc_data_structures::jobserver::{self, Acquired};
13use rustc_data_structures::memmap::Mmap;
14use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
15use rustc_errors::emitter::Emitter;
16use rustc_errors::translation::Translate;
17use rustc_errors::{
18    Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FluentBundle, Level, MultiSpan,
19    Style, Suggestions,
20};
21use rustc_fs_util::link_or_copy;
22use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
23use rustc_incremental::{
24    copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
25};
26use rustc_metadata::EncodedMetadata;
27use rustc_metadata::fs::copy_to_stdout;
28use rustc_middle::bug;
29use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
30use rustc_middle::middle::exported_symbols::SymbolExportInfo;
31use rustc_middle::ty::TyCtxt;
32use rustc_session::Session;
33use rustc_session::config::{
34    self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
35};
36use rustc_span::source_map::SourceMap;
37use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
38use rustc_target::spec::{MergeFunctions, SanitizerSet};
39use tracing::debug;
40
41use super::link::{self, ensure_removed};
42use super::lto::{self, SerializedModule};
43use super::symbol_export::symbol_name_for_instance_in_crate;
44use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
45use crate::traits::*;
46use crate::{
47    CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
48    errors,
49};
50
51const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
52
53/// What kind of object file to emit.
54#[derive(Clone, Copy, PartialEq)]
55pub enum EmitObj {
56    // No object file.
57    None,
58
59    // Just uncompressed llvm bitcode. Provides easy compatibility with
60    // emscripten's ecc compiler, when used as the linker.
61    Bitcode,
62
63    // Object code, possibly augmented with a bitcode section.
64    ObjectCode(BitcodeSection),
65}
66
67/// What kind of llvm bitcode section to embed in an object file.
68#[derive(Clone, Copy, PartialEq)]
69pub enum BitcodeSection {
70    // No bitcode section.
71    None,
72
73    // A full, uncompressed bitcode section.
74    Full,
75}
76
77/// Module-specific configuration for `optimize_and_codegen`.
78pub struct ModuleConfig {
79    /// Names of additional optimization passes to run.
80    pub passes: Vec<String>,
81    /// Some(level) to optimize at a certain level, or None to run
82    /// absolutely no optimizations (used for the metadata module).
83    pub opt_level: Option<config::OptLevel>,
84
85    /// Some(level) to optimize binary size, or None to not affect program size.
86    pub opt_size: Option<config::OptLevel>,
87
88    pub pgo_gen: SwitchWithOptPath,
89    pub pgo_use: Option<PathBuf>,
90    pub pgo_sample_use: Option<PathBuf>,
91    pub debug_info_for_profiling: bool,
92    pub instrument_coverage: bool,
93
94    pub sanitizer: SanitizerSet,
95    pub sanitizer_recover: SanitizerSet,
96    pub sanitizer_dataflow_abilist: Vec<String>,
97    pub sanitizer_memory_track_origins: usize,
98
99    // Flags indicating which outputs to produce.
100    pub emit_pre_lto_bc: bool,
101    pub emit_no_opt_bc: bool,
102    pub emit_bc: bool,
103    pub emit_ir: bool,
104    pub emit_asm: bool,
105    pub emit_obj: EmitObj,
106    pub emit_thin_lto: bool,
107    pub emit_thin_lto_summary: bool,
108    pub bc_cmdline: String,
109
110    // Miscellaneous flags. These are mostly copied from command-line
111    // options.
112    pub verify_llvm_ir: bool,
113    pub lint_llvm_ir: bool,
114    pub no_prepopulate_passes: bool,
115    pub no_builtins: bool,
116    pub time_module: bool,
117    pub vectorize_loop: bool,
118    pub vectorize_slp: bool,
119    pub merge_functions: bool,
120    pub emit_lifetime_markers: bool,
121    pub llvm_plugins: Vec<String>,
122    pub autodiff: Vec<config::AutoDiff>,
123}
124
125impl ModuleConfig {
126    fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
127        // If it's a regular module, use `$regular`, otherwise use `$other`.
128        // `$regular` and `$other` are evaluated lazily.
129        macro_rules! if_regular {
130            ($regular: expr, $other: expr) => {
131                if let ModuleKind::Regular = kind { $regular } else { $other }
132            };
133        }
134
135        let sess = tcx.sess;
136        let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
137
138        let save_temps = sess.opts.cg.save_temps;
139
140        let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
141            || match kind {
142                ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
143                ModuleKind::Allocator => false,
144                ModuleKind::Metadata => sess.opts.output_types.contains_key(&OutputType::Metadata),
145            };
146
147        let emit_obj = if !should_emit_obj {
148            EmitObj::None
149        } else if sess.target.obj_is_bitcode
150            || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
151        {
152            // This case is selected if the target uses objects as bitcode, or
153            // if linker plugin LTO is enabled. In the linker plugin LTO case
154            // the assumption is that the final link-step will read the bitcode
155            // and convert it to object code. This may be done by either the
156            // native linker or rustc itself.
157            //
158            // Note, however, that the linker-plugin-lto requested here is
159            // explicitly ignored for `#![no_builtins]` crates. These crates are
160            // specifically ignored by rustc's LTO passes and wouldn't work if
161            // loaded into the linker. These crates define symbols that LLVM
162            // lowers intrinsics to, and these symbol dependencies aren't known
163            // until after codegen. As a result any crate marked
164            // `#![no_builtins]` is assumed to not participate in LTO and
165            // instead goes on to generate object code.
166            EmitObj::Bitcode
167        } else if need_bitcode_in_object(tcx) {
168            EmitObj::ObjectCode(BitcodeSection::Full)
169        } else {
170            EmitObj::ObjectCode(BitcodeSection::None)
171        };
172
173        ModuleConfig {
174            passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
175
176            opt_level: opt_level_and_size,
177            opt_size: opt_level_and_size,
178
179            pgo_gen: if_regular!(
180                sess.opts.cg.profile_generate.clone(),
181                SwitchWithOptPath::Disabled
182            ),
183            pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
184            pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
185            debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
186            instrument_coverage: if_regular!(sess.instrument_coverage(), false),
187
188            sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
189            sanitizer_dataflow_abilist: if_regular!(
190                sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
191                Vec::new()
192            ),
193            sanitizer_recover: if_regular!(
194                sess.opts.unstable_opts.sanitizer_recover,
195                SanitizerSet::empty()
196            ),
197            sanitizer_memory_track_origins: if_regular!(
198                sess.opts.unstable_opts.sanitizer_memory_track_origins,
199                0
200            ),
201
202            emit_pre_lto_bc: if_regular!(
203                save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
204                false
205            ),
206            emit_no_opt_bc: if_regular!(save_temps, false),
207            emit_bc: if_regular!(
208                save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
209                save_temps
210            ),
211            emit_ir: if_regular!(
212                sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
213                false
214            ),
215            emit_asm: if_regular!(
216                sess.opts.output_types.contains_key(&OutputType::Assembly),
217                false
218            ),
219            emit_obj,
220            emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
221            emit_thin_lto_summary: if_regular!(
222                sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
223                false
224            ),
225            bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
226
227            verify_llvm_ir: sess.verify_llvm_ir(),
228            lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
229            no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
230            no_builtins: no_builtins || sess.target.no_builtins,
231
232            // Exclude metadata and allocator modules from time_passes output,
233            // since they throw off the "LLVM passes" measurement.
234            time_module: if_regular!(true, false),
235
236            // Copy what clang does by turning on loop vectorization at O2 and
237            // slp vectorization at O3.
238            vectorize_loop: !sess.opts.cg.no_vectorize_loops
239                && (sess.opts.optimize == config::OptLevel::More
240                    || sess.opts.optimize == config::OptLevel::Aggressive),
241            vectorize_slp: !sess.opts.cg.no_vectorize_slp
242                && sess.opts.optimize == config::OptLevel::Aggressive,
243
244            // Some targets (namely, NVPTX) interact badly with the
245            // MergeFunctions pass. This is because MergeFunctions can generate
246            // new function calls which may interfere with the target calling
247            // convention; e.g. for the NVPTX target, PTX kernels should not
248            // call other PTX kernels. MergeFunctions can also be configured to
249            // generate aliases instead, but aliases are not supported by some
250            // backends (again, NVPTX). Therefore, allow targets to opt out of
251            // the MergeFunctions pass, but otherwise keep the pass enabled (at
252            // O2 and O3) since it can be useful for reducing code size.
253            merge_functions: match sess
254                .opts
255                .unstable_opts
256                .merge_functions
257                .unwrap_or(sess.target.merge_functions)
258            {
259                MergeFunctions::Disabled => false,
260                MergeFunctions::Trampolines | MergeFunctions::Aliases => {
261                    use config::OptLevel::*;
262                    match sess.opts.optimize {
263                        Aggressive | More | SizeMin | Size => true,
264                        Less | No => false,
265                    }
266                }
267            },
268
269            emit_lifetime_markers: sess.emit_lifetime_markers(),
270            llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
271            autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
272        }
273    }
274
275    pub fn bitcode_needed(&self) -> bool {
276        self.emit_bc
277            || self.emit_thin_lto_summary
278            || self.emit_obj == EmitObj::Bitcode
279            || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
280    }
281}
282
283/// Configuration passed to the function returned by the `target_machine_factory`.
284pub struct TargetMachineFactoryConfig {
285    /// Split DWARF is enabled in LLVM by checking that `TM.MCOptions.SplitDwarfFile` isn't empty,
286    /// so the path to the dwarf object has to be provided when we create the target machine.
287    /// This can be ignored by backends which do not need it for their Split DWARF support.
288    pub split_dwarf_file: Option<PathBuf>,
289
290    /// The name of the output object file. Used for setting OutputFilenames in target options
291    /// so that LLVM can emit the CodeView S_OBJNAME record in pdb files
292    pub output_obj_file: Option<PathBuf>,
293}
294
295impl TargetMachineFactoryConfig {
296    pub fn new(
297        cgcx: &CodegenContext<impl WriteBackendMethods>,
298        module_name: &str,
299    ) -> TargetMachineFactoryConfig {
300        let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
301            cgcx.output_filenames.split_dwarf_path(
302                cgcx.split_debuginfo,
303                cgcx.split_dwarf_kind,
304                Some(module_name),
305            )
306        } else {
307            None
308        };
309
310        let output_obj_file =
311            Some(cgcx.output_filenames.temp_path(OutputType::Object, Some(module_name)));
312        TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
313    }
314}
315
316pub type TargetMachineFactoryFn<B> = Arc<
317    dyn Fn(
318            TargetMachineFactoryConfig,
319        ) -> Result<
320            <B as WriteBackendMethods>::TargetMachine,
321            <B as WriteBackendMethods>::TargetMachineError,
322        > + Send
323        + Sync,
324>;
325
326type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
327
328/// Additional resources used by optimize_and_codegen (not module specific)
329#[derive(Clone)]
330pub struct CodegenContext<B: WriteBackendMethods> {
331    // Resources needed when running LTO
332    pub prof: SelfProfilerRef,
333    pub lto: Lto,
334    pub save_temps: bool,
335    pub fewer_names: bool,
336    pub time_trace: bool,
337    pub exported_symbols: Option<Arc<ExportedSymbols>>,
338    pub opts: Arc<config::Options>,
339    pub crate_types: Vec<CrateType>,
340    pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
341    pub output_filenames: Arc<OutputFilenames>,
342    pub regular_module_config: Arc<ModuleConfig>,
343    pub metadata_module_config: Arc<ModuleConfig>,
344    pub allocator_module_config: Arc<ModuleConfig>,
345    pub tm_factory: TargetMachineFactoryFn<B>,
346    pub msvc_imps_needed: bool,
347    pub is_pe_coff: bool,
348    pub target_can_use_split_dwarf: bool,
349    pub target_arch: String,
350    pub target_is_like_osx: bool,
351    pub target_is_like_aix: bool,
352    pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
353    pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
354
355    /// All commandline args used to invoke the compiler, with @file args fully expanded.
356    /// This will only be used within debug info, e.g. in the pdb file on windows
357    /// This is mainly useful for other tools that reads that debuginfo to figure out
358    /// how to call the compiler with the same arguments.
359    pub expanded_args: Vec<String>,
360
361    /// Emitter to use for diagnostics produced during codegen.
362    pub diag_emitter: SharedEmitter,
363    /// LLVM optimizations for which we want to print remarks.
364    pub remark: Passes,
365    /// Directory into which should the LLVM optimization remarks be written.
366    /// If `None`, they will be written to stderr.
367    pub remark_dir: Option<PathBuf>,
368    /// The incremental compilation session directory, or None if we are not
369    /// compiling incrementally
370    pub incr_comp_session_dir: Option<PathBuf>,
371    /// Channel back to the main control thread to send messages to
372    pub coordinator_send: Sender<Box<dyn Any + Send>>,
373    /// `true` if the codegen should be run in parallel.
374    ///
375    /// Depends on [`CodegenBackend::supports_parallel()`] and `-Zno_parallel_backend`.
376    pub parallel: bool,
377}
378
379impl<B: WriteBackendMethods> CodegenContext<B> {
380    pub fn create_dcx(&self) -> DiagCtxt {
381        DiagCtxt::new(Box::new(self.diag_emitter.clone()))
382    }
383
384    pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
385        match kind {
386            ModuleKind::Regular => &self.regular_module_config,
387            ModuleKind::Metadata => &self.metadata_module_config,
388            ModuleKind::Allocator => &self.allocator_module_config,
389        }
390    }
391}
392
393fn generate_lto_work<B: ExtraBackendMethods>(
394    cgcx: &CodegenContext<B>,
395    autodiff: Vec<AutoDiffItem>,
396    needs_fat_lto: Vec<FatLtoInput<B>>,
397    needs_thin_lto: Vec<(String, B::ThinBuffer)>,
398    import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
399) -> Vec<(WorkItem<B>, u64)> {
400    let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
401
402    if !needs_fat_lto.is_empty() {
403        assert!(needs_thin_lto.is_empty());
404        let mut module =
405            B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
406        if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
407            let config = cgcx.config(ModuleKind::Regular);
408            module = unsafe { module.autodiff(cgcx, autodiff, config).unwrap() };
409        }
410        // We are adding a single work item, so the cost doesn't matter.
411        vec![(WorkItem::LTO(module), 0)]
412    } else {
413        if !autodiff.is_empty() {
414            let dcx = cgcx.create_dcx();
415            dcx.handle().emit_fatal(AutodiffWithoutLto {});
416        }
417        assert!(needs_fat_lto.is_empty());
418        let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
419            .unwrap_or_else(|e| e.raise());
420        lto_modules
421            .into_iter()
422            .map(|module| {
423                let cost = module.cost();
424                (WorkItem::LTO(module), cost)
425            })
426            .chain(copy_jobs.into_iter().map(|wp| {
427                (
428                    WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
429                        name: wp.cgu_name.clone(),
430                        source: wp,
431                    }),
432                    0, // copying is very cheap
433                )
434            }))
435            .collect()
436    }
437}
438
439struct CompiledModules {
440    modules: Vec<CompiledModule>,
441    allocator_module: Option<CompiledModule>,
442}
443
444fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
445    let sess = tcx.sess;
446    sess.opts.cg.embed_bitcode
447        && tcx.crate_types().contains(&CrateType::Rlib)
448        && sess.opts.output_types.contains_key(&OutputType::Exe)
449}
450
451fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
452    if sess.opts.incremental.is_none() {
453        return false;
454    }
455
456    match sess.lto() {
457        Lto::No => false,
458        Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
459    }
460}
461
462pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
463    backend: B,
464    tcx: TyCtxt<'_>,
465    target_cpu: String,
466    metadata: EncodedMetadata,
467    metadata_module: Option<CompiledModule>,
468) -> OngoingCodegen<B> {
469    let (coordinator_send, coordinator_receive) = channel();
470
471    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
472    let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
473
474    let crate_info = CrateInfo::new(tcx, target_cpu);
475
476    let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
477    let metadata_config = ModuleConfig::new(ModuleKind::Metadata, tcx, no_builtins);
478    let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
479
480    let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
481    let (codegen_worker_send, codegen_worker_receive) = channel();
482
483    let coordinator_thread = start_executing_work(
484        backend.clone(),
485        tcx,
486        &crate_info,
487        shared_emitter,
488        codegen_worker_send,
489        coordinator_receive,
490        Arc::new(regular_config),
491        Arc::new(metadata_config),
492        Arc::new(allocator_config),
493        coordinator_send.clone(),
494    );
495
496    OngoingCodegen {
497        backend,
498        metadata,
499        metadata_module,
500        crate_info,
501
502        codegen_worker_receive,
503        shared_emitter_main,
504        coordinator: Coordinator {
505            sender: coordinator_send,
506            future: Some(coordinator_thread),
507            phantom: PhantomData,
508        },
509        output_filenames: Arc::clone(tcx.output_filenames(())),
510    }
511}
512
513fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
514    sess: &Session,
515    compiled_modules: &CompiledModules,
516) -> FxIndexMap<WorkProductId, WorkProduct> {
517    let mut work_products = FxIndexMap::default();
518
519    if sess.opts.incremental.is_none() {
520        return work_products;
521    }
522
523    let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
524
525    for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
526        let mut files = Vec::new();
527        if let Some(object_file_path) = &module.object {
528            files.push((OutputType::Object.extension(), object_file_path.as_path()));
529        }
530        if let Some(dwarf_object_file_path) = &module.dwarf_object {
531            files.push(("dwo", dwarf_object_file_path.as_path()));
532        }
533        if let Some(path) = &module.assembly {
534            files.push((OutputType::Assembly.extension(), path.as_path()));
535        }
536        if let Some(path) = &module.llvm_ir {
537            files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
538        }
539        if let Some(path) = &module.bytecode {
540            files.push((OutputType::Bitcode.extension(), path.as_path()));
541        }
542        if let Some((id, product)) =
543            copy_cgu_workproduct_to_incr_comp_cache_dir(sess, &module.name, files.as_slice())
544        {
545            work_products.insert(id, product);
546        }
547    }
548
549    work_products
550}
551
552fn produce_final_output_artifacts(
553    sess: &Session,
554    compiled_modules: &CompiledModules,
555    crate_output: &OutputFilenames,
556) {
557    let mut user_wants_bitcode = false;
558    let mut user_wants_objects = false;
559
560    // Produce final compile outputs.
561    let copy_gracefully = |from: &Path, to: &OutFileName| match to {
562        OutFileName::Stdout => {
563            if let Err(e) = copy_to_stdout(from) {
564                sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
565            }
566        }
567        OutFileName::Real(path) => {
568            if let Err(e) = fs::copy(from, path) {
569                sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
570            }
571        }
572    };
573
574    let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
575        if compiled_modules.modules.len() == 1 {
576            // 1) Only one codegen unit. In this case it's no difficulty
577            //    to copy `foo.0.x` to `foo.x`.
578            let module_name = Some(&compiled_modules.modules[0].name[..]);
579            let path = crate_output.temp_path(output_type, module_name);
580            let output = crate_output.path(output_type);
581            if !output_type.is_text_output() && output.is_tty() {
582                sess.dcx()
583                    .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
584            } else {
585                copy_gracefully(&path, &output);
586            }
587            if !sess.opts.cg.save_temps && !keep_numbered {
588                // The user just wants `foo.x`, not `foo.#module-name#.x`.
589                ensure_removed(sess.dcx(), &path);
590            }
591        } else {
592            let extension = crate_output
593                .temp_path(output_type, None)
594                .extension()
595                .unwrap()
596                .to_str()
597                .unwrap()
598                .to_owned();
599
600            if crate_output.outputs.contains_explicit_name(&output_type) {
601                // 2) Multiple codegen units, with `--emit foo=some_name`. We have
602                //    no good solution for this case, so warn the user.
603                sess.dcx().emit_warn(errors::IgnoringEmitPath { extension });
604            } else if crate_output.single_output_file.is_some() {
605                // 3) Multiple codegen units, with `-o some_name`. We have
606                //    no good solution for this case, so warn the user.
607                sess.dcx().emit_warn(errors::IgnoringOutput { extension });
608            } else {
609                // 4) Multiple codegen units, but no explicit name. We
610                //    just leave the `foo.0.x` files in place.
611                // (We don't have to do any work in this case.)
612            }
613        }
614    };
615
616    // Flag to indicate whether the user explicitly requested bitcode.
617    // Otherwise, we produced it only as a temporary output, and will need
618    // to get rid of it.
619    for output_type in crate_output.outputs.keys() {
620        match *output_type {
621            OutputType::Bitcode => {
622                user_wants_bitcode = true;
623                // Copy to .bc, but always keep the .0.bc. There is a later
624                // check to figure out if we should delete .0.bc files, or keep
625                // them for making an rlib.
626                copy_if_one_unit(OutputType::Bitcode, true);
627            }
628            OutputType::ThinLinkBitcode => {
629                copy_if_one_unit(OutputType::ThinLinkBitcode, false);
630            }
631            OutputType::LlvmAssembly => {
632                copy_if_one_unit(OutputType::LlvmAssembly, false);
633            }
634            OutputType::Assembly => {
635                copy_if_one_unit(OutputType::Assembly, false);
636            }
637            OutputType::Object => {
638                user_wants_objects = true;
639                copy_if_one_unit(OutputType::Object, true);
640            }
641            OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
642        }
643    }
644
645    // Clean up unwanted temporary files.
646
647    // We create the following files by default:
648    //  - #crate#.#module-name#.bc
649    //  - #crate#.#module-name#.o
650    //  - #crate#.crate.metadata.bc
651    //  - #crate#.crate.metadata.o
652    //  - #crate#.o (linked from crate.##.o)
653    //  - #crate#.bc (copied from crate.##.bc)
654    // We may create additional files if requested by the user (through
655    // `-C save-temps` or `--emit=` flags).
656
657    if !sess.opts.cg.save_temps {
658        // Remove the temporary .#module-name#.o objects. If the user didn't
659        // explicitly request bitcode (with --emit=bc), and the bitcode is not
660        // needed for building an rlib, then we must remove .#module-name#.bc as
661        // well.
662
663        // Specific rules for keeping .#module-name#.bc:
664        //  - If the user requested bitcode (`user_wants_bitcode`), and
665        //    codegen_units > 1, then keep it.
666        //  - If the user requested bitcode but codegen_units == 1, then we
667        //    can toss .#module-name#.bc because we copied it to .bc earlier.
668        //  - If we're not building an rlib and the user didn't request
669        //    bitcode, then delete .#module-name#.bc.
670        // If you change how this works, also update back::link::link_rlib,
671        // where .#module-name#.bc files are (maybe) deleted after making an
672        // rlib.
673        let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
674
675        let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
676
677        let keep_numbered_objects =
678            needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
679
680        for module in compiled_modules.modules.iter() {
681            if let Some(ref path) = module.object {
682                if !keep_numbered_objects {
683                    ensure_removed(sess.dcx(), path);
684                }
685            }
686
687            if let Some(ref path) = module.dwarf_object {
688                if !keep_numbered_objects {
689                    ensure_removed(sess.dcx(), path);
690                }
691            }
692
693            if let Some(ref path) = module.bytecode {
694                if !keep_numbered_bitcode {
695                    ensure_removed(sess.dcx(), path);
696                }
697            }
698        }
699
700        if !user_wants_bitcode {
701            if let Some(ref allocator_module) = compiled_modules.allocator_module {
702                if let Some(ref path) = allocator_module.bytecode {
703                    ensure_removed(sess.dcx(), path);
704                }
705            }
706        }
707    }
708
709    if sess.opts.json_artifact_notifications {
710        if compiled_modules.modules.len() == 1 {
711            compiled_modules.modules[0].for_each_output(|_path, ty| {
712                if sess.opts.output_types.contains_key(&ty) {
713                    let descr = ty.shorthand();
714                    // for single cgu file is renamed to drop cgu specific suffix
715                    // so we regenerate it the same way
716                    let path = crate_output.path(ty);
717                    sess.dcx().emit_artifact_notification(path.as_path(), descr);
718                }
719            });
720        } else {
721            for module in &compiled_modules.modules {
722                module.for_each_output(|path, ty| {
723                    if sess.opts.output_types.contains_key(&ty) {
724                        let descr = ty.shorthand();
725                        sess.dcx().emit_artifact_notification(&path, descr);
726                    }
727                });
728            }
729        }
730    }
731
732    // We leave the following files around by default:
733    //  - #crate#.o
734    //  - #crate#.crate.metadata.o
735    //  - #crate#.bc
736    // These are used in linking steps and will be cleaned up afterward.
737}
738
739pub(crate) enum WorkItem<B: WriteBackendMethods> {
740    /// Optimize a newly codegened, totally unoptimized module.
741    Optimize(ModuleCodegen<B::Module>),
742    /// Copy the post-LTO artifacts from the incremental cache to the output
743    /// directory.
744    CopyPostLtoArtifacts(CachedModuleCodegen),
745    /// Performs (Thin)LTO on the given module.
746    LTO(lto::LtoModuleCodegen<B>),
747}
748
749impl<B: WriteBackendMethods> WorkItem<B> {
750    fn module_kind(&self) -> ModuleKind {
751        match *self {
752            WorkItem::Optimize(ref m) => m.kind,
753            WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
754        }
755    }
756
757    /// Generate a short description of this work item suitable for use as a thread name.
758    fn short_description(&self) -> String {
759        // `pthread_setname()` on *nix ignores anything beyond the first 15
760        // bytes. Use short descriptions to maximize the space available for
761        // the module name.
762        #[cfg(not(windows))]
763        fn desc(short: &str, _long: &str, name: &str) -> String {
764            // The short label is three bytes, and is followed by a space. That
765            // leaves 11 bytes for the CGU name. How we obtain those 11 bytes
766            // depends on the CGU name form.
767            //
768            // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part
769            //   before the `-cgu.0` is the same for every CGU, so use the
770            //   `cgu.0` part. The number suffix will be different for each
771            //   CGU.
772            //
773            // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole
774            //   name because each CGU will have a unique ASCII hash, and the
775            //   first 11 bytes will be enough to identify it.
776            //
777            // - Incremental (with `-Zhuman-readable-cgu-names`), e.g.
778            //   `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole
779            //   name. The first 11 bytes won't be enough to uniquely identify
780            //   it, but no obvious substring will, and this is a rarely used
781            //   option so it doesn't matter much.
782            //
783            assert_eq!(short.len(), 3);
784            let name = if let Some(index) = name.find("-cgu.") {
785                &name[index + 1..] // +1 skips the leading '-'.
786            } else {
787                name
788            };
789            format!("{short} {name}")
790        }
791
792        // Windows has no thread name length limit, so use more descriptive names.
793        #[cfg(windows)]
794        fn desc(_short: &str, long: &str, name: &str) -> String {
795            format!("{long} {name}")
796        }
797
798        match self {
799            WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
800            WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
801            WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
802        }
803    }
804}
805
806/// A result produced by the backend.
807pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
808    /// The backend has finished compiling a CGU, nothing more required.
809    Finished(CompiledModule),
810
811    /// The backend has finished compiling a CGU, which now needs linking
812    /// because `-Zcombine-cgu` was specified.
813    NeedsLink(ModuleCodegen<B::Module>),
814
815    /// The backend has finished compiling a CGU, which now needs to go through
816    /// fat LTO.
817    NeedsFatLto(FatLtoInput<B>),
818
819    /// The backend has finished compiling a CGU, which now needs to go through
820    /// thin LTO.
821    NeedsThinLto(String, B::ThinBuffer),
822}
823
824pub enum FatLtoInput<B: WriteBackendMethods> {
825    Serialized { name: String, buffer: B::ModuleBuffer },
826    InMemory(ModuleCodegen<B::Module>),
827}
828
829/// Actual LTO type we end up choosing based on multiple factors.
830pub(crate) enum ComputedLtoType {
831    No,
832    Thin,
833    Fat,
834}
835
836pub(crate) fn compute_per_cgu_lto_type(
837    sess_lto: &Lto,
838    opts: &config::Options,
839    sess_crate_types: &[CrateType],
840    module_kind: ModuleKind,
841) -> ComputedLtoType {
842    // Metadata modules never participate in LTO regardless of the lto
843    // settings.
844    if module_kind == ModuleKind::Metadata {
845        return ComputedLtoType::No;
846    }
847
848    // If the linker does LTO, we don't have to do it. Note that we
849    // keep doing full LTO, if it is requested, as not to break the
850    // assumption that the output will be a single module.
851    let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
852
853    // When we're automatically doing ThinLTO for multi-codegen-unit
854    // builds we don't actually want to LTO the allocator modules if
855    // it shows up. This is due to various linker shenanigans that
856    // we'll encounter later.
857    let is_allocator = module_kind == ModuleKind::Allocator;
858
859    // We ignore a request for full crate graph LTO if the crate type
860    // is only an rlib, as there is no full crate graph to process,
861    // that'll happen later.
862    //
863    // This use case currently comes up primarily for targets that
864    // require LTO so the request for LTO is always unconditionally
865    // passed down to the backend, but we don't actually want to do
866    // anything about it yet until we've got a final product.
867    let is_rlib = sess_crate_types.len() == 1 && sess_crate_types[0] == CrateType::Rlib;
868
869    match sess_lto {
870        Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
871        Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
872        Lto::Fat if !is_rlib => ComputedLtoType::Fat,
873        _ => ComputedLtoType::No,
874    }
875}
876
877fn execute_optimize_work_item<B: ExtraBackendMethods>(
878    cgcx: &CodegenContext<B>,
879    module: ModuleCodegen<B::Module>,
880    module_config: &ModuleConfig,
881) -> Result<WorkItemResult<B>, FatalError> {
882    let dcx = cgcx.create_dcx();
883    let dcx = dcx.handle();
884
885    unsafe {
886        B::optimize(cgcx, dcx, &module, module_config)?;
887    }
888
889    // After we've done the initial round of optimizations we need to
890    // decide whether to synchronously codegen this module or ship it
891    // back to the coordinator thread for further LTO processing (which
892    // has to wait for all the initial modules to be optimized).
893
894    let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
895
896    // If we're doing some form of incremental LTO then we need to be sure to
897    // save our module to disk first.
898    let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
899        let filename = pre_lto_bitcode_filename(&module.name);
900        cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
901    } else {
902        None
903    };
904
905    match lto_type {
906        ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
907        ComputedLtoType::Thin => {
908            let (name, thin_buffer) = B::prepare_thin(module, false);
909            if let Some(path) = bitcode {
910                fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
911                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
912                });
913            }
914            Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
915        }
916        ComputedLtoType::Fat => match bitcode {
917            Some(path) => {
918                let (name, buffer) = B::serialize_module(module);
919                fs::write(&path, buffer.data()).unwrap_or_else(|e| {
920                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
921                });
922                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
923            }
924            None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
925        },
926    }
927}
928
929fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
930    cgcx: &CodegenContext<B>,
931    module: CachedModuleCodegen,
932    module_config: &ModuleConfig,
933) -> WorkItemResult<B> {
934    let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
935
936    let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
937        let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
938        debug!(
939            "copying preexisting module `{}` from {:?} to {}",
940            module.name,
941            source_file,
942            output_path.display()
943        );
944        match link_or_copy(&source_file, &output_path) {
945            Ok(_) => Some(output_path),
946            Err(error) => {
947                cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
948                    source_file,
949                    output_path,
950                    error,
951                });
952                None
953            }
954        }
955    };
956
957    let dwarf_object =
958        module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
959            let dwarf_obj_out = cgcx
960                .output_filenames
961                .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name))
962                .expect(
963                    "saved dwarf object in work product but `split_dwarf_path` returned `None`",
964                );
965            load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
966        });
967
968    let load_from_incr_cache = |perform, output_type: OutputType| {
969        if perform {
970            let saved_file = module.source.saved_files.get(output_type.extension())?;
971            let output_path = cgcx.output_filenames.temp_path(output_type, Some(&module.name));
972            load_from_incr_comp_dir(output_path, &saved_file)
973        } else {
974            None
975        }
976    };
977
978    let should_emit_obj = module_config.emit_obj != EmitObj::None;
979    let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
980    let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
981    let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
982    let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
983    if should_emit_obj && object.is_none() {
984        cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
985    }
986
987    WorkItemResult::Finished(CompiledModule {
988        name: module.name,
989        kind: ModuleKind::Regular,
990        object,
991        dwarf_object,
992        bytecode,
993        assembly,
994        llvm_ir,
995    })
996}
997
998fn execute_lto_work_item<B: ExtraBackendMethods>(
999    cgcx: &CodegenContext<B>,
1000    module: lto::LtoModuleCodegen<B>,
1001    module_config: &ModuleConfig,
1002) -> Result<WorkItemResult<B>, FatalError> {
1003    let module = unsafe { module.optimize(cgcx)? };
1004    finish_intra_module_work(cgcx, module, module_config)
1005}
1006
1007fn finish_intra_module_work<B: ExtraBackendMethods>(
1008    cgcx: &CodegenContext<B>,
1009    module: ModuleCodegen<B::Module>,
1010    module_config: &ModuleConfig,
1011) -> Result<WorkItemResult<B>, FatalError> {
1012    let dcx = cgcx.create_dcx();
1013    let dcx = dcx.handle();
1014
1015    if !cgcx.opts.unstable_opts.combine_cgu
1016        || module.kind == ModuleKind::Metadata
1017        || module.kind == ModuleKind::Allocator
1018    {
1019        let module = unsafe { B::codegen(cgcx, dcx, module, module_config)? };
1020        Ok(WorkItemResult::Finished(module))
1021    } else {
1022        Ok(WorkItemResult::NeedsLink(module))
1023    }
1024}
1025
1026/// Messages sent to the coordinator.
1027pub(crate) enum Message<B: WriteBackendMethods> {
1028    /// A jobserver token has become available. Sent from the jobserver helper
1029    /// thread.
1030    Token(io::Result<Acquired>),
1031
1032    /// The backend has finished processing a work item for a codegen unit.
1033    /// Sent from a backend worker thread.
1034    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1035
1036    /// A vector containing all the AutoDiff tasks that we have to pass to Enzyme.
1037    AddAutoDiffItems(Vec<AutoDiffItem>),
1038
1039    /// The frontend has finished generating something (backend IR or a
1040    /// post-LTO artifact) for a codegen unit, and it should be passed to the
1041    /// backend. Sent from the main thread.
1042    CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1043
1044    /// Similar to `CodegenDone`, but for reusing a pre-LTO artifact
1045    /// Sent from the main thread.
1046    AddImportOnlyModule {
1047        module_data: SerializedModule<B::ModuleBuffer>,
1048        work_product: WorkProduct,
1049    },
1050
1051    /// The frontend has finished generating everything for all codegen units.
1052    /// Sent from the main thread.
1053    CodegenComplete,
1054
1055    /// Some normal-ish compiler error occurred, and codegen should be wound
1056    /// down. Sent from the main thread.
1057    CodegenAborted,
1058}
1059
1060/// A message sent from the coordinator thread to the main thread telling it to
1061/// process another codegen unit.
1062pub struct CguMessage;
1063
1064// A cut-down version of `rustc_errors::DiagInner` that impls `Send`, which
1065// can be used to send diagnostics from codegen threads to the main thread.
1066// It's missing the following fields from `rustc_errors::DiagInner`.
1067// - `span`: it doesn't impl `Send`.
1068// - `suggestions`: it doesn't impl `Send`, and isn't used for codegen
1069//   diagnostics.
1070// - `sort_span`: it doesn't impl `Send`.
1071// - `is_lint`: lints aren't relevant during codegen.
1072// - `emitted_at`: not used for codegen diagnostics.
1073struct Diagnostic {
1074    level: Level,
1075    messages: Vec<(DiagMessage, Style)>,
1076    code: Option<ErrCode>,
1077    children: Vec<Subdiagnostic>,
1078    args: DiagArgMap,
1079}
1080
1081// A cut-down version of `rustc_errors::Subdiag` that impls `Send`. It's
1082// missing the following fields from `rustc_errors::Subdiag`.
1083// - `span`: it doesn't impl `Send`.
1084pub(crate) struct Subdiagnostic {
1085    level: Level,
1086    messages: Vec<(DiagMessage, Style)>,
1087}
1088
1089#[derive(PartialEq, Clone, Copy, Debug)]
1090enum MainThreadState {
1091    /// Doing nothing.
1092    Idle,
1093
1094    /// Doing codegen, i.e. MIR-to-LLVM-IR conversion.
1095    Codegenning,
1096
1097    /// Idle, but lending the compiler process's Token to an LLVM thread so it can do useful work.
1098    Lending,
1099}
1100
1101fn start_executing_work<B: ExtraBackendMethods>(
1102    backend: B,
1103    tcx: TyCtxt<'_>,
1104    crate_info: &CrateInfo,
1105    shared_emitter: SharedEmitter,
1106    codegen_worker_send: Sender<CguMessage>,
1107    coordinator_receive: Receiver<Box<dyn Any + Send>>,
1108    regular_config: Arc<ModuleConfig>,
1109    metadata_config: Arc<ModuleConfig>,
1110    allocator_config: Arc<ModuleConfig>,
1111    tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1112) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1113    let coordinator_send = tx_to_llvm_workers;
1114    let sess = tcx.sess;
1115
1116    let mut each_linked_rlib_for_lto = Vec::new();
1117    drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1118        if link::ignored_for_lto(sess, crate_info, cnum) {
1119            return;
1120        }
1121        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1122    }));
1123
1124    // Compute the set of symbols we need to retain when doing LTO (if we need to)
1125    let exported_symbols = {
1126        let mut exported_symbols = FxHashMap::default();
1127
1128        let copy_symbols = |cnum| {
1129            let symbols = tcx
1130                .exported_symbols(cnum)
1131                .iter()
1132                .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1133                .collect();
1134            Arc::new(symbols)
1135        };
1136
1137        match sess.lto() {
1138            Lto::No => None,
1139            Lto::ThinLocal => {
1140                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1141                Some(Arc::new(exported_symbols))
1142            }
1143            Lto::Fat | Lto::Thin => {
1144                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1145                for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1146                    exported_symbols.insert(cnum, copy_symbols(cnum));
1147                }
1148                Some(Arc::new(exported_symbols))
1149            }
1150        }
1151    };
1152
1153    // First up, convert our jobserver into a helper thread so we can use normal
1154    // mpsc channels to manage our messages and such.
1155    // After we've requested tokens then we'll, when we can,
1156    // get tokens on `coordinator_receive` which will
1157    // get managed in the main loop below.
1158    let coordinator_send2 = coordinator_send.clone();
1159    let helper = jobserver::client()
1160        .into_helper_thread(move |token| {
1161            drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1162        })
1163        .expect("failed to spawn helper thread");
1164
1165    let ol =
1166        if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1167            // If we know that we won’t be doing codegen, create target machines without optimisation.
1168            config::OptLevel::No
1169        } else {
1170            tcx.backend_optimization_level(())
1171        };
1172    let backend_features = tcx.global_backend_features(());
1173
1174    let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1175        let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1176        match result {
1177            Ok(dir) => Some(dir),
1178            Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1179        }
1180    } else {
1181        None
1182    };
1183
1184    let cgcx = CodegenContext::<B> {
1185        crate_types: tcx.crate_types().to_vec(),
1186        each_linked_rlib_for_lto,
1187        lto: sess.lto(),
1188        fewer_names: sess.fewer_names(),
1189        save_temps: sess.opts.cg.save_temps,
1190        time_trace: sess.opts.unstable_opts.llvm_time_trace,
1191        opts: Arc::new(sess.opts.clone()),
1192        prof: sess.prof.clone(),
1193        exported_symbols,
1194        remark: sess.opts.cg.remark.clone(),
1195        remark_dir,
1196        incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1197        coordinator_send,
1198        expanded_args: tcx.sess.expanded_args.clone(),
1199        diag_emitter: shared_emitter.clone(),
1200        output_filenames: Arc::clone(tcx.output_filenames(())),
1201        regular_module_config: regular_config,
1202        metadata_module_config: metadata_config,
1203        allocator_module_config: allocator_config,
1204        tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1205        msvc_imps_needed: msvc_imps_needed(tcx),
1206        is_pe_coff: tcx.sess.target.is_like_windows,
1207        target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1208        target_arch: tcx.sess.target.arch.to_string(),
1209        target_is_like_osx: tcx.sess.target.is_like_osx,
1210        target_is_like_aix: tcx.sess.target.is_like_aix,
1211        split_debuginfo: tcx.sess.split_debuginfo(),
1212        split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1213        parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1214    };
1215
1216    // This is the "main loop" of parallel work happening for parallel codegen.
1217    // It's here that we manage parallelism, schedule work, and work with
1218    // messages coming from clients.
1219    //
1220    // There are a few environmental pre-conditions that shape how the system
1221    // is set up:
1222    //
1223    // - Error reporting can only happen on the main thread because that's the
1224    //   only place where we have access to the compiler `Session`.
1225    // - LLVM work can be done on any thread.
1226    // - Codegen can only happen on the main thread.
1227    // - Each thread doing substantial work must be in possession of a `Token`
1228    //   from the `Jobserver`.
1229    // - The compiler process always holds one `Token`. Any additional `Tokens`
1230    //   have to be requested from the `Jobserver`.
1231    //
1232    // Error Reporting
1233    // ===============
1234    // The error reporting restriction is handled separately from the rest: We
1235    // set up a `SharedEmitter` that holds an open channel to the main thread.
1236    // When an error occurs on any thread, the shared emitter will send the
1237    // error message to the receiver main thread (`SharedEmitterMain`). The
1238    // main thread will periodically query this error message queue and emit
1239    // any error messages it has received. It might even abort compilation if
1240    // it has received a fatal error. In this case we rely on all other threads
1241    // being torn down automatically with the main thread.
1242    // Since the main thread will often be busy doing codegen work, error
1243    // reporting will be somewhat delayed, since the message queue can only be
1244    // checked in between two work packages.
1245    //
1246    // Work Processing Infrastructure
1247    // ==============================
1248    // The work processing infrastructure knows three major actors:
1249    //
1250    // - the coordinator thread,
1251    // - the main thread, and
1252    // - LLVM worker threads
1253    //
1254    // The coordinator thread is running a message loop. It instructs the main
1255    // thread about what work to do when, and it will spawn off LLVM worker
1256    // threads as open LLVM WorkItems become available.
1257    //
1258    // The job of the main thread is to codegen CGUs into LLVM work packages
1259    // (since the main thread is the only thread that can do this). The main
1260    // thread will block until it receives a message from the coordinator, upon
1261    // which it will codegen one CGU, send it to the coordinator and block
1262    // again. This way the coordinator can control what the main thread is
1263    // doing.
1264    //
1265    // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is
1266    // available, it will spawn off a new LLVM worker thread and let it process
1267    // a WorkItem. When a LLVM worker thread is done with its WorkItem,
1268    // it will just shut down, which also frees all resources associated with
1269    // the given LLVM module, and sends a message to the coordinator that the
1270    // WorkItem has been completed.
1271    //
1272    // Work Scheduling
1273    // ===============
1274    // The scheduler's goal is to minimize the time it takes to complete all
1275    // work there is, however, we also want to keep memory consumption low
1276    // if possible. These two goals are at odds with each other: If memory
1277    // consumption were not an issue, we could just let the main thread produce
1278    // LLVM WorkItems at full speed, assuring maximal utilization of
1279    // Tokens/LLVM worker threads. However, since codegen is usually faster
1280    // than LLVM processing, the queue of LLVM WorkItems would fill up and each
1281    // WorkItem potentially holds on to a substantial amount of memory.
1282    //
1283    // So the actual goal is to always produce just enough LLVM WorkItems as
1284    // not to starve our LLVM worker threads. That means, once we have enough
1285    // WorkItems in our queue, we can block the main thread, so it does not
1286    // produce more until we need them.
1287    //
1288    // Doing LLVM Work on the Main Thread
1289    // ----------------------------------
1290    // Since the main thread owns the compiler process's implicit `Token`, it is
1291    // wasteful to keep it blocked without doing any work. Therefore, what we do
1292    // in this case is: We spawn off an additional LLVM worker thread that helps
1293    // reduce the queue. The work it is doing corresponds to the implicit
1294    // `Token`. The coordinator will mark the main thread as being busy with
1295    // LLVM work. (The actual work happens on another OS thread but we just care
1296    // about `Tokens`, not actual threads).
1297    //
1298    // When any LLVM worker thread finishes while the main thread is marked as
1299    // "busy with LLVM work", we can do a little switcheroo: We give the Token
1300    // of the just finished thread to the LLVM worker thread that is working on
1301    // behalf of the main thread's implicit Token, thus freeing up the main
1302    // thread again. The coordinator can then again decide what the main thread
1303    // should do. This allows the coordinator to make decisions at more points
1304    // in time.
1305    //
1306    // Striking a Balance between Throughput and Memory Consumption
1307    // ------------------------------------------------------------
1308    // Since our two goals, (1) use as many Tokens as possible and (2) keep
1309    // memory consumption as low as possible, are in conflict with each other,
1310    // we have to find a trade off between them. Right now, the goal is to keep
1311    // all workers busy, which means that no worker should find the queue empty
1312    // when it is ready to start.
1313    // How do we do achieve this? Good question :) We actually never know how
1314    // many `Tokens` are potentially available so it's hard to say how much to
1315    // fill up the queue before switching the main thread to LLVM work. Also we
1316    // currently don't have a means to estimate how long a running LLVM worker
1317    // will still be busy with it's current WorkItem. However, we know the
1318    // maximal count of available Tokens that makes sense (=the number of CPU
1319    // cores), so we can take a conservative guess. The heuristic we use here
1320    // is implemented in the `queue_full_enough()` function.
1321    //
1322    // Some Background on Jobservers
1323    // -----------------------------
1324    // It's worth also touching on the management of parallelism here. We don't
1325    // want to just spawn a thread per work item because while that's optimal
1326    // parallelism it may overload a system with too many threads or violate our
1327    // configuration for the maximum amount of cpu to use for this process. To
1328    // manage this we use the `jobserver` crate.
1329    //
1330    // Job servers are an artifact of GNU make and are used to manage
1331    // parallelism between processes. A jobserver is a glorified IPC semaphore
1332    // basically. Whenever we want to run some work we acquire the semaphore,
1333    // and whenever we're done with that work we release the semaphore. In this
1334    // manner we can ensure that the maximum number of parallel workers is
1335    // capped at any one point in time.
1336    //
1337    // LTO and the coordinator thread
1338    // ------------------------------
1339    //
1340    // The final job the coordinator thread is responsible for is managing LTO
1341    // and how that works. When LTO is requested what we'll do is collect all
1342    // optimized LLVM modules into a local vector on the coordinator. Once all
1343    // modules have been codegened and optimized we hand this to the `lto`
1344    // module for further optimization. The `lto` module will return back a list
1345    // of more modules to work on, which the coordinator will continue to spawn
1346    // work for.
1347    //
1348    // Each LLVM module is automatically sent back to the coordinator for LTO if
1349    // necessary. There's already optimizations in place to avoid sending work
1350    // back to the coordinator if LTO isn't requested.
1351    return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1352        let mut worker_id_counter = 0;
1353        let mut free_worker_ids = Vec::new();
1354        let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1355            if let Some(id) = free_worker_ids.pop() {
1356                id
1357            } else {
1358                let id = worker_id_counter;
1359                worker_id_counter += 1;
1360                id
1361            }
1362        };
1363
1364        // This is where we collect codegen units that have gone all the way
1365        // through codegen and LLVM.
1366        let mut autodiff_items = Vec::new();
1367        let mut compiled_modules = vec![];
1368        let mut compiled_allocator_module = None;
1369        let mut needs_link = Vec::new();
1370        let mut needs_fat_lto = Vec::new();
1371        let mut needs_thin_lto = Vec::new();
1372        let mut lto_import_only_modules = Vec::new();
1373        let mut started_lto = false;
1374
1375        /// Possible state transitions:
1376        /// - Ongoing -> Completed
1377        /// - Ongoing -> Aborted
1378        /// - Completed -> Aborted
1379        #[derive(Debug, PartialEq)]
1380        enum CodegenState {
1381            Ongoing,
1382            Completed,
1383            Aborted,
1384        }
1385        use CodegenState::*;
1386        let mut codegen_state = Ongoing;
1387
1388        // This is the queue of LLVM work items that still need processing.
1389        let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1390
1391        // This are the Jobserver Tokens we currently hold. Does not include
1392        // the implicit Token the compiler process owns no matter what.
1393        let mut tokens = Vec::new();
1394
1395        let mut main_thread_state = MainThreadState::Idle;
1396
1397        // How many LLVM worker threads are running while holding a Token. This
1398        // *excludes* any that the main thread is lending a Token to.
1399        let mut running_with_own_token = 0;
1400
1401        // How many LLVM worker threads are running in total. This *includes*
1402        // any that the main thread is lending a Token to.
1403        let running_with_any_token = |main_thread_state, running_with_own_token| {
1404            running_with_own_token
1405                + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1406        };
1407
1408        let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1409
1410        // Run the message loop while there's still anything that needs message
1411        // processing. Note that as soon as codegen is aborted we simply want to
1412        // wait for all existing work to finish, so many of the conditions here
1413        // only apply if codegen hasn't been aborted as they represent pending
1414        // work to be done.
1415        loop {
1416            // While there are still CGUs to be codegened, the coordinator has
1417            // to decide how to utilize the compiler processes implicit Token:
1418            // For codegenning more CGU or for running them through LLVM.
1419            if codegen_state == Ongoing {
1420                if main_thread_state == MainThreadState::Idle {
1421                    // Compute the number of workers that will be running once we've taken as many
1422                    // items from the work queue as we can, plus one for the main thread. It's not
1423                    // critically important that we use this instead of just
1424                    // `running_with_own_token`, but it prevents the `queue_full_enough` heuristic
1425                    // from fluctuating just because a worker finished up and we decreased the
1426                    // `running_with_own_token` count, even though we're just going to increase it
1427                    // right after this when we put a new worker to work.
1428                    let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1429                    let additional_running = std::cmp::min(extra_tokens, work_items.len());
1430                    let anticipated_running = running_with_own_token + additional_running + 1;
1431
1432                    if !queue_full_enough(work_items.len(), anticipated_running) {
1433                        // The queue is not full enough, process more codegen units:
1434                        if codegen_worker_send.send(CguMessage).is_err() {
1435                            panic!("Could not send CguMessage to main thread")
1436                        }
1437                        main_thread_state = MainThreadState::Codegenning;
1438                    } else {
1439                        // The queue is full enough to not let the worker
1440                        // threads starve. Use the implicit Token to do some
1441                        // LLVM work too.
1442                        let (item, _) =
1443                            work_items.pop().expect("queue empty - queue_full_enough() broken?");
1444                        main_thread_state = MainThreadState::Lending;
1445                        spawn_work(
1446                            &cgcx,
1447                            &mut llvm_start_time,
1448                            get_worker_id(&mut free_worker_ids),
1449                            item,
1450                        );
1451                    }
1452                }
1453            } else if codegen_state == Completed {
1454                if running_with_any_token(main_thread_state, running_with_own_token) == 0
1455                    && work_items.is_empty()
1456                {
1457                    // All codegen work is done. Do we have LTO work to do?
1458                    if needs_fat_lto.is_empty()
1459                        && needs_thin_lto.is_empty()
1460                        && lto_import_only_modules.is_empty()
1461                    {
1462                        // Nothing more to do!
1463                        break;
1464                    }
1465
1466                    // We have LTO work to do. Perform the serial work here of
1467                    // figuring out what we're going to LTO and then push a
1468                    // bunch of work items onto our queue to do LTO. This all
1469                    // happens on the coordinator thread but it's very quick so
1470                    // we don't worry about tokens.
1471                    assert!(!started_lto);
1472                    started_lto = true;
1473
1474                    let needs_fat_lto = mem::take(&mut needs_fat_lto);
1475                    let needs_thin_lto = mem::take(&mut needs_thin_lto);
1476                    let import_only_modules = mem::take(&mut lto_import_only_modules);
1477
1478                    for (work, cost) in generate_lto_work(
1479                        &cgcx,
1480                        autodiff_items.clone(),
1481                        needs_fat_lto,
1482                        needs_thin_lto,
1483                        import_only_modules,
1484                    ) {
1485                        let insertion_index = work_items
1486                            .binary_search_by_key(&cost, |&(_, cost)| cost)
1487                            .unwrap_or_else(|e| e);
1488                        work_items.insert(insertion_index, (work, cost));
1489                        if cgcx.parallel {
1490                            helper.request_token();
1491                        }
1492                    }
1493                }
1494
1495                // In this branch, we know that everything has been codegened,
1496                // so it's just a matter of determining whether the implicit
1497                // Token is free to use for LLVM work.
1498                match main_thread_state {
1499                    MainThreadState::Idle => {
1500                        if let Some((item, _)) = work_items.pop() {
1501                            main_thread_state = MainThreadState::Lending;
1502                            spawn_work(
1503                                &cgcx,
1504                                &mut llvm_start_time,
1505                                get_worker_id(&mut free_worker_ids),
1506                                item,
1507                            );
1508                        } else {
1509                            // There is no unstarted work, so let the main thread
1510                            // take over for a running worker. Otherwise the
1511                            // implicit token would just go to waste.
1512                            // We reduce the `running` counter by one. The
1513                            // `tokens.truncate()` below will take care of
1514                            // giving the Token back.
1515                            assert!(running_with_own_token > 0);
1516                            running_with_own_token -= 1;
1517                            main_thread_state = MainThreadState::Lending;
1518                        }
1519                    }
1520                    MainThreadState::Codegenning => bug!(
1521                        "codegen worker should not be codegenning after \
1522                              codegen was already completed"
1523                    ),
1524                    MainThreadState::Lending => {
1525                        // Already making good use of that token
1526                    }
1527                }
1528            } else {
1529                // Don't queue up any more work if codegen was aborted, we're
1530                // just waiting for our existing children to finish.
1531                assert!(codegen_state == Aborted);
1532                if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1533                    break;
1534                }
1535            }
1536
1537            // Spin up what work we can, only doing this while we've got available
1538            // parallelism slots and work left to spawn.
1539            if codegen_state != Aborted {
1540                while !work_items.is_empty() && running_with_own_token < tokens.len() {
1541                    let (item, _) = work_items.pop().unwrap();
1542                    spawn_work(
1543                        &cgcx,
1544                        &mut llvm_start_time,
1545                        get_worker_id(&mut free_worker_ids),
1546                        item,
1547                    );
1548                    running_with_own_token += 1;
1549                }
1550            }
1551
1552            // Relinquish accidentally acquired extra tokens.
1553            tokens.truncate(running_with_own_token);
1554
1555            // If a thread exits successfully then we drop a token associated
1556            // with that worker and update our `running_with_own_token` count.
1557            // We may later re-acquire a token to continue running more work.
1558            // We may also not actually drop a token here if the worker was
1559            // running with an "ephemeral token".
1560            let mut free_worker = |worker_id| {
1561                if main_thread_state == MainThreadState::Lending {
1562                    main_thread_state = MainThreadState::Idle;
1563                } else {
1564                    running_with_own_token -= 1;
1565                }
1566
1567                free_worker_ids.push(worker_id);
1568            };
1569
1570            let msg = coordinator_receive.recv().unwrap();
1571            match *msg.downcast::<Message<B>>().ok().unwrap() {
1572                // Save the token locally and the next turn of the loop will use
1573                // this to spawn a new unit of work, or it may get dropped
1574                // immediately if we have no more work to spawn.
1575                Message::Token(token) => {
1576                    match token {
1577                        Ok(token) => {
1578                            tokens.push(token);
1579
1580                            if main_thread_state == MainThreadState::Lending {
1581                                // If the main thread token is used for LLVM work
1582                                // at the moment, we turn that thread into a regular
1583                                // LLVM worker thread, so the main thread is free
1584                                // to react to codegen demand.
1585                                main_thread_state = MainThreadState::Idle;
1586                                running_with_own_token += 1;
1587                            }
1588                        }
1589                        Err(e) => {
1590                            let msg = &format!("failed to acquire jobserver token: {e}");
1591                            shared_emitter.fatal(msg);
1592                            codegen_state = Aborted;
1593                        }
1594                    }
1595                }
1596
1597                Message::CodegenDone { llvm_work_item, cost } => {
1598                    // We keep the queue sorted by estimated processing cost,
1599                    // so that more expensive items are processed earlier. This
1600                    // is good for throughput as it gives the main thread more
1601                    // time to fill up the queue and it avoids scheduling
1602                    // expensive items to the end.
1603                    // Note, however, that this is not ideal for memory
1604                    // consumption, as LLVM module sizes are not evenly
1605                    // distributed.
1606                    let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1607                    let insertion_index = match insertion_index {
1608                        Ok(idx) | Err(idx) => idx,
1609                    };
1610                    work_items.insert(insertion_index, (llvm_work_item, cost));
1611
1612                    if cgcx.parallel {
1613                        helper.request_token();
1614                    }
1615                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1616                    main_thread_state = MainThreadState::Idle;
1617                }
1618
1619                Message::AddAutoDiffItems(mut items) => {
1620                    autodiff_items.append(&mut items);
1621                }
1622
1623                Message::CodegenComplete => {
1624                    if codegen_state != Aborted {
1625                        codegen_state = Completed;
1626                    }
1627                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1628                    main_thread_state = MainThreadState::Idle;
1629                }
1630
1631                // If codegen is aborted that means translation was aborted due
1632                // to some normal-ish compiler error. In this situation we want
1633                // to exit as soon as possible, but we want to make sure all
1634                // existing work has finished. Flag codegen as being done, and
1635                // then conditions above will ensure no more work is spawned but
1636                // we'll keep executing this loop until `running_with_own_token`
1637                // hits 0.
1638                Message::CodegenAborted => {
1639                    codegen_state = Aborted;
1640                }
1641
1642                Message::WorkItem { result, worker_id } => {
1643                    free_worker(worker_id);
1644
1645                    match result {
1646                        Ok(WorkItemResult::Finished(compiled_module)) => {
1647                            match compiled_module.kind {
1648                                ModuleKind::Regular => {
1649                                    assert!(needs_link.is_empty());
1650                                    compiled_modules.push(compiled_module);
1651                                }
1652                                ModuleKind::Allocator => {
1653                                    assert!(compiled_allocator_module.is_none());
1654                                    compiled_allocator_module = Some(compiled_module);
1655                                }
1656                                ModuleKind::Metadata => bug!("Should be handled separately"),
1657                            }
1658                        }
1659                        Ok(WorkItemResult::NeedsLink(module)) => {
1660                            assert!(compiled_modules.is_empty());
1661                            needs_link.push(module);
1662                        }
1663                        Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1664                            assert!(!started_lto);
1665                            assert!(needs_thin_lto.is_empty());
1666                            needs_fat_lto.push(fat_lto_input);
1667                        }
1668                        Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1669                            assert!(!started_lto);
1670                            assert!(needs_fat_lto.is_empty());
1671                            needs_thin_lto.push((name, thin_buffer));
1672                        }
1673                        Err(Some(WorkerFatalError)) => {
1674                            // Like `CodegenAborted`, wait for remaining work to finish.
1675                            codegen_state = Aborted;
1676                        }
1677                        Err(None) => {
1678                            // If the thread failed that means it panicked, so
1679                            // we abort immediately.
1680                            bug!("worker thread panicked");
1681                        }
1682                    }
1683                }
1684
1685                Message::AddImportOnlyModule { module_data, work_product } => {
1686                    assert!(!started_lto);
1687                    assert_eq!(codegen_state, Ongoing);
1688                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1689                    lto_import_only_modules.push((module_data, work_product));
1690                    main_thread_state = MainThreadState::Idle;
1691                }
1692            }
1693        }
1694
1695        if codegen_state == Aborted {
1696            return Err(());
1697        }
1698
1699        let needs_link = mem::take(&mut needs_link);
1700        if !needs_link.is_empty() {
1701            assert!(compiled_modules.is_empty());
1702            let dcx = cgcx.create_dcx();
1703            let dcx = dcx.handle();
1704            let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1705            let module = unsafe {
1706                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?
1707            };
1708            compiled_modules.push(module);
1709        }
1710
1711        // Drop to print timings
1712        drop(llvm_start_time);
1713
1714        // Regardless of what order these modules completed in, report them to
1715        // the backend in the same order every time to ensure that we're handing
1716        // out deterministic results.
1717        compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1718
1719        Ok(CompiledModules {
1720            modules: compiled_modules,
1721            allocator_module: compiled_allocator_module,
1722        })
1723    })
1724    .expect("failed to spawn coordinator thread");
1725
1726    // A heuristic that determines if we have enough LLVM WorkItems in the
1727    // queue so that the main thread can do LLVM work instead of codegen
1728    fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1729        // This heuristic scales ahead-of-time codegen according to available
1730        // concurrency, as measured by `workers_running`. The idea is that the
1731        // more concurrency we have available, the more demand there will be for
1732        // work items, and the fuller the queue should be kept to meet demand.
1733        // An important property of this approach is that we codegen ahead of
1734        // time only as much as necessary, so as to keep fewer LLVM modules in
1735        // memory at once, thereby reducing memory consumption.
1736        //
1737        // When the number of workers running is less than the max concurrency
1738        // available to us, this heuristic can cause us to instruct the main
1739        // thread to work on an LLVM item (that is, tell it to "LLVM") instead
1740        // of codegen, even though it seems like it *should* be codegenning so
1741        // that we can create more work items and spawn more LLVM workers.
1742        //
1743        // But this is not a problem. When the main thread is told to LLVM,
1744        // according to this heuristic and how work is scheduled, there is
1745        // always at least one item in the queue, and therefore at least one
1746        // pending jobserver token request. If there *is* more concurrency
1747        // available, we will immediately receive a token, which will upgrade
1748        // the main thread's LLVM worker to a real one (conceptually), and free
1749        // up the main thread to codegen if necessary. On the other hand, if
1750        // there isn't more concurrency, then the main thread working on an LLVM
1751        // item is appropriate, as long as the queue is full enough for demand.
1752        //
1753        // Speaking of which, how full should we keep the queue? Probably less
1754        // full than you'd think. A lot has to go wrong for the queue not to be
1755        // full enough and for that to have a negative effect on compile times.
1756        //
1757        // Workers are unlikely to finish at exactly the same time, so when one
1758        // finishes and takes another work item off the queue, we often have
1759        // ample time to codegen at that point before the next worker finishes.
1760        // But suppose that codegen takes so long that the workers exhaust the
1761        // queue, and we have one or more workers that have nothing to work on.
1762        // Well, it might not be so bad. Of all the LLVM modules we create and
1763        // optimize, one has to finish last. It's not necessarily the case that
1764        // by losing some concurrency for a moment, we delay the point at which
1765        // that last LLVM module is finished and the rest of compilation can
1766        // proceed. Also, when we can't take advantage of some concurrency, we
1767        // give tokens back to the job server. That enables some other rustc to
1768        // potentially make use of the available concurrency. That could even
1769        // *decrease* overall compile time if we're lucky. But yes, if no other
1770        // rustc can make use of the concurrency, then we've squandered it.
1771        //
1772        // However, keeping the queue full is also beneficial when we have a
1773        // surge in available concurrency. Then items can be taken from the
1774        // queue immediately, without having to wait for codegen.
1775        //
1776        // So, the heuristic below tries to keep one item in the queue for every
1777        // four running workers. Based on limited benchmarking, this appears to
1778        // be more than sufficient to avoid increasing compilation times.
1779        let quarter_of_workers = workers_running - 3 * workers_running / 4;
1780        items_in_queue > 0 && items_in_queue >= quarter_of_workers
1781    }
1782}
1783
1784/// `FatalError` is explicitly not `Send`.
1785#[must_use]
1786pub(crate) struct WorkerFatalError;
1787
1788fn spawn_work<'a, B: ExtraBackendMethods>(
1789    cgcx: &'a CodegenContext<B>,
1790    llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1791    worker_id: usize,
1792    work: WorkItem<B>,
1793) {
1794    if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1795        *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1796    }
1797
1798    let cgcx = cgcx.clone();
1799
1800    B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1801        // Set up a destructor which will fire off a message that we're done as
1802        // we exit.
1803        struct Bomb<B: ExtraBackendMethods> {
1804            coordinator_send: Sender<Box<dyn Any + Send>>,
1805            result: Option<Result<WorkItemResult<B>, FatalError>>,
1806            worker_id: usize,
1807        }
1808        impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1809            fn drop(&mut self) {
1810                let worker_id = self.worker_id;
1811                let msg = match self.result.take() {
1812                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1813                    Some(Err(FatalError)) => {
1814                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1815                    }
1816                    None => Message::WorkItem::<B> { result: Err(None), worker_id },
1817                };
1818                drop(self.coordinator_send.send(Box::new(msg)));
1819            }
1820        }
1821
1822        let mut bomb =
1823            Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1824
1825        // Execute the work itself, and if it finishes successfully then flag
1826        // ourselves as a success as well.
1827        //
1828        // Note that we ignore any `FatalError` coming out of `execute_work_item`,
1829        // as a diagnostic was already sent off to the main thread - just
1830        // surface that there was an error in this worker.
1831        bomb.result = {
1832            let module_config = cgcx.config(work.module_kind());
1833
1834            Some(match work {
1835                WorkItem::Optimize(m) => {
1836                    let _timer =
1837                        cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1838                    execute_optimize_work_item(&cgcx, m, module_config)
1839                }
1840                WorkItem::CopyPostLtoArtifacts(m) => {
1841                    let _timer = cgcx.prof.generic_activity_with_arg(
1842                        "codegen_copy_artifacts_from_incr_cache",
1843                        &*m.name,
1844                    );
1845                    Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1846                }
1847                WorkItem::LTO(m) => {
1848                    let _timer =
1849                        cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1850                    execute_lto_work_item(&cgcx, m, module_config)
1851                }
1852            })
1853        };
1854    })
1855    .expect("failed to spawn work thread");
1856}
1857
1858enum SharedEmitterMessage {
1859    Diagnostic(Diagnostic),
1860    InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1861    Fatal(String),
1862}
1863
1864#[derive(Clone)]
1865pub struct SharedEmitter {
1866    sender: Sender<SharedEmitterMessage>,
1867}
1868
1869pub struct SharedEmitterMain {
1870    receiver: Receiver<SharedEmitterMessage>,
1871}
1872
1873impl SharedEmitter {
1874    fn new() -> (SharedEmitter, SharedEmitterMain) {
1875        let (sender, receiver) = channel();
1876
1877        (SharedEmitter { sender }, SharedEmitterMain { receiver })
1878    }
1879
1880    pub fn inline_asm_error(
1881        &self,
1882        span: SpanData,
1883        msg: String,
1884        level: Level,
1885        source: Option<(String, Vec<InnerSpan>)>,
1886    ) {
1887        drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1888    }
1889
1890    fn fatal(&self, msg: &str) {
1891        drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1892    }
1893}
1894
1895impl Translate for SharedEmitter {
1896    fn fluent_bundle(&self) -> Option<&FluentBundle> {
1897        None
1898    }
1899
1900    fn fallback_fluent_bundle(&self) -> &FluentBundle {
1901        panic!("shared emitter attempted to translate a diagnostic");
1902    }
1903}
1904
1905impl Emitter for SharedEmitter {
1906    fn emit_diagnostic(
1907        &mut self,
1908        mut diag: rustc_errors::DiagInner,
1909        _registry: &rustc_errors::registry::Registry,
1910    ) {
1911        // Check that we aren't missing anything interesting when converting to
1912        // the cut-down local `DiagInner`.
1913        assert_eq!(diag.span, MultiSpan::new());
1914        assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1915        assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1916        assert_eq!(diag.is_lint, None);
1917        // No sensible check for `diag.emitted_at`.
1918
1919        let args = mem::replace(&mut diag.args, DiagArgMap::default());
1920        drop(
1921            self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1922                level: diag.level(),
1923                messages: diag.messages,
1924                code: diag.code,
1925                children: diag
1926                    .children
1927                    .into_iter()
1928                    .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1929                    .collect(),
1930                args,
1931            })),
1932        );
1933    }
1934
1935    fn source_map(&self) -> Option<&SourceMap> {
1936        None
1937    }
1938}
1939
1940impl SharedEmitterMain {
1941    fn check(&self, sess: &Session, blocking: bool) {
1942        loop {
1943            let message = if blocking {
1944                match self.receiver.recv() {
1945                    Ok(message) => Ok(message),
1946                    Err(_) => Err(()),
1947                }
1948            } else {
1949                match self.receiver.try_recv() {
1950                    Ok(message) => Ok(message),
1951                    Err(_) => Err(()),
1952                }
1953            };
1954
1955            match message {
1956                Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1957                    // The diagnostic has been received on the main thread.
1958                    // Convert it back to a full `Diagnostic` and emit.
1959                    let dcx = sess.dcx();
1960                    let mut d =
1961                        rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1962                    d.code = diag.code; // may be `None`, that's ok
1963                    d.children = diag
1964                        .children
1965                        .into_iter()
1966                        .map(|sub| rustc_errors::Subdiag {
1967                            level: sub.level,
1968                            messages: sub.messages,
1969                            span: MultiSpan::new(),
1970                        })
1971                        .collect();
1972                    d.args = diag.args;
1973                    dcx.emit_diagnostic(d);
1974                    sess.dcx().abort_if_errors();
1975                }
1976                Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1977                    assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1978                    let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1979                    if !span.is_dummy() {
1980                        err.span(span.span());
1981                    }
1982
1983                    // Point to the generated assembly if it is available.
1984                    if let Some((buffer, spans)) = source {
1985                        let source = sess
1986                            .source_map()
1987                            .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
1988                        let spans: Vec<_> = spans
1989                            .iter()
1990                            .map(|sp| {
1991                                Span::with_root_ctxt(
1992                                    source.normalized_byte_pos(sp.start as u32),
1993                                    source.normalized_byte_pos(sp.end as u32),
1994                                )
1995                            })
1996                            .collect();
1997                        err.span_note(spans, "instantiated into assembly here");
1998                    }
1999
2000                    err.emit();
2001                }
2002                Ok(SharedEmitterMessage::Fatal(msg)) => {
2003                    sess.dcx().fatal(msg);
2004                }
2005                Err(_) => {
2006                    break;
2007                }
2008            }
2009        }
2010    }
2011}
2012
2013pub struct Coordinator<B: ExtraBackendMethods> {
2014    pub sender: Sender<Box<dyn Any + Send>>,
2015    future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2016    // Only used for the Message type.
2017    phantom: PhantomData<B>,
2018}
2019
2020impl<B: ExtraBackendMethods> Coordinator<B> {
2021    fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2022        self.future.take().unwrap().join()
2023    }
2024}
2025
2026impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2027    fn drop(&mut self) {
2028        if let Some(future) = self.future.take() {
2029            // If we haven't joined yet, signal to the coordinator that it should spawn no more
2030            // work, and wait for worker threads to finish.
2031            drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2032            drop(future.join());
2033        }
2034    }
2035}
2036
2037pub struct OngoingCodegen<B: ExtraBackendMethods> {
2038    pub backend: B,
2039    pub metadata: EncodedMetadata,
2040    pub metadata_module: Option<CompiledModule>,
2041    pub crate_info: CrateInfo,
2042    pub codegen_worker_receive: Receiver<CguMessage>,
2043    pub shared_emitter_main: SharedEmitterMain,
2044    pub output_filenames: Arc<OutputFilenames>,
2045    pub coordinator: Coordinator<B>,
2046}
2047
2048impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2049    pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2050        self.shared_emitter_main.check(sess, true);
2051        let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2052            Ok(Ok(compiled_modules)) => compiled_modules,
2053            Ok(Err(())) => {
2054                sess.dcx().abort_if_errors();
2055                panic!("expected abort due to worker thread errors")
2056            }
2057            Err(_) => {
2058                bug!("panic during codegen/LLVM phase");
2059            }
2060        });
2061
2062        sess.dcx().abort_if_errors();
2063
2064        let work_products =
2065            copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2066        produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2067
2068        // FIXME: time_llvm_passes support - does this use a global context or
2069        // something?
2070        if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2071            self.backend.print_pass_timings()
2072        }
2073
2074        if sess.print_llvm_stats() {
2075            self.backend.print_statistics()
2076        }
2077
2078        (
2079            CodegenResults {
2080                metadata: self.metadata,
2081                crate_info: self.crate_info,
2082
2083                modules: compiled_modules.modules,
2084                allocator_module: compiled_modules.allocator_module,
2085                metadata_module: self.metadata_module,
2086            },
2087            work_products,
2088        )
2089    }
2090
2091    pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2092        self.wait_for_signal_to_codegen_item();
2093        self.check_for_errors(tcx.sess);
2094        drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2095    }
2096
2097    pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2098        drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2099    }
2100
2101    pub(crate) fn check_for_errors(&self, sess: &Session) {
2102        self.shared_emitter_main.check(sess, false);
2103    }
2104
2105    pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2106        match self.codegen_worker_receive.recv() {
2107            Ok(CguMessage) => {
2108                // Ok to proceed.
2109            }
2110            Err(_) => {
2111                // One of the LLVM threads must have panicked, fall through so
2112                // error handling can be reached.
2113            }
2114        }
2115    }
2116}
2117
2118pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2119    _backend: &B,
2120    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2121    module: ModuleCodegen<B::Module>,
2122    cost: u64,
2123) {
2124    let llvm_work_item = WorkItem::Optimize(module);
2125    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2126}
2127
2128pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2129    _backend: &B,
2130    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2131    module: CachedModuleCodegen,
2132) {
2133    let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2134    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2135}
2136
2137pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2138    _backend: &B,
2139    tcx: TyCtxt<'_>,
2140    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2141    module: CachedModuleCodegen,
2142) {
2143    let filename = pre_lto_bitcode_filename(&module.name);
2144    let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2145    let file = fs::File::open(&bc_path)
2146        .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2147
2148    let mmap = unsafe {
2149        Mmap::map(file).unwrap_or_else(|e| {
2150            panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2151        })
2152    };
2153    // Schedule the module to be loaded
2154    drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2155        module_data: SerializedModule::FromUncompressedFile(mmap),
2156        work_product: module.source,
2157    })));
2158}
2159
2160fn pre_lto_bitcode_filename(module_name: &str) -> String {
2161    format!("{module_name}.{PRE_LTO_BC_EXT}")
2162}
2163
2164fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2165    // This should never be true (because it's not supported). If it is true,
2166    // something is wrong with commandline arg validation.
2167    assert!(
2168        !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2169            && tcx.sess.target.is_like_windows
2170            && tcx.sess.opts.cg.prefer_dynamic)
2171    );
2172
2173    // We need to generate _imp__ symbol if we are generating an rlib or we include one
2174    // indirectly from ThinLTO. In theory these are not needed as ThinLTO could resolve
2175    // these, but it currently does not do so.
2176    let can_have_static_objects =
2177        tcx.sess.lto() == Lto::Thin || tcx.crate_types().iter().any(|ct| *ct == CrateType::Rlib);
2178
2179    tcx.sess.target.is_like_windows &&
2180    can_have_static_objects   &&
2181    // ThinLTO can't handle this workaround in all cases, so we don't
2182    // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing
2183    // dynamic linking when linker plugin LTO is enabled.
2184    !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2185}