1mod autodiff;
96
97use std::cmp;
98use std::collections::hash_map::Entry;
99use std::fs::{self, File};
100use std::io::Write;
101use std::path::{Path, PathBuf};
102
103use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
104use rustc_data_structures::sync;
105use rustc_data_structures::unord::{UnordMap, UnordSet};
106use rustc_hir::LangItem;
107use rustc_hir::attrs::InlineAttr;
108use rustc_hir::def::DefKind;
109use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE};
110use rustc_hir::definitions::DefPathDataName;
111use rustc_middle::bug;
112use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
113use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
114use rustc_middle::mir::mono::{
115 CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, Linkage, MonoItem, MonoItemData,
116 MonoItemPartitions, Visibility,
117};
118use rustc_middle::ty::print::{characteristic_def_id_of_type, with_no_trimmed_paths};
119use rustc_middle::ty::{self, InstanceKind, TyCtxt};
120use rustc_middle::util::Providers;
121use rustc_session::CodegenUnits;
122use rustc_session::config::{DumpMonoStatsFormat, SwitchWithOptPath};
123use rustc_span::Symbol;
124use rustc_target::spec::SymbolVisibility;
125use tracing::debug;
126
127use crate::collector::{self, MonoItemCollectionStrategy, UsageMap};
128use crate::errors::{CouldntDumpMonoStats, SymbolAlreadyDefined};
129
130struct PartitioningCx<'a, 'tcx> {
131 tcx: TyCtxt<'tcx>,
132 usage_map: &'a UsageMap<'tcx>,
133}
134
135struct PlacedMonoItems<'tcx> {
136 codegen_units: Vec<CodegenUnit<'tcx>>,
138
139 internalization_candidates: UnordSet<MonoItem<'tcx>>,
140}
141
142fn partition<'tcx, I>(
144 tcx: TyCtxt<'tcx>,
145 mono_items: I,
146 usage_map: &UsageMap<'tcx>,
147) -> Vec<CodegenUnit<'tcx>>
148where
149 I: Iterator<Item = MonoItem<'tcx>>,
150{
151 let _prof_timer = tcx.prof.generic_activity("cgu_partitioning");
152
153 let cx = &PartitioningCx { tcx, usage_map };
154
155 let PlacedMonoItems { mut codegen_units, internalization_candidates } = {
158 let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_items");
159 let placed = place_mono_items(cx, mono_items);
160
161 debug_dump(tcx, "PLACE", &placed.codegen_units);
162
163 placed
164 };
165
166 {
170 let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_merge_cgus");
171 merge_codegen_units(cx, &mut codegen_units);
172 debug_dump(tcx, "MERGE", &codegen_units);
173 }
174
175 if !tcx.sess.link_dead_code() {
178 let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_internalize_symbols");
179 internalize_symbols(cx, &mut codegen_units, internalization_candidates);
180
181 debug_dump(tcx, "INTERNALIZE", &codegen_units);
182 }
183
184 if tcx.sess.instrument_coverage() {
186 mark_code_coverage_dead_code_cgu(&mut codegen_units);
187 }
188
189 if !codegen_units.is_sorted_by(|a, b| a.name().as_str() <= b.name().as_str()) {
191 let mut names = String::new();
192 for cgu in codegen_units.iter() {
193 names += &format!("- {}\n", cgu.name());
194 }
195 bug!("unsorted CGUs:\n{names}");
196 }
197
198 codegen_units
199}
200
201fn place_mono_items<'tcx, I>(cx: &PartitioningCx<'_, 'tcx>, mono_items: I) -> PlacedMonoItems<'tcx>
202where
203 I: Iterator<Item = MonoItem<'tcx>>,
204{
205 let mut codegen_units = UnordMap::default();
206 let is_incremental_build = cx.tcx.sess.opts.incremental.is_some();
207 let mut internalization_candidates = UnordSet::default();
208
209 let can_export_generics = cx.tcx.local_crate_exports_generics();
214 let always_export_generics = can_export_generics && cx.tcx.sess.opts.share_generics();
215
216 let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx);
217 let cgu_name_cache = &mut UnordMap::default();
218
219 for mono_item in mono_items {
220 match mono_item.instantiation_mode(cx.tcx) {
225 InstantiationMode::GloballyShared { .. } => {}
226 InstantiationMode::LocalCopy => continue,
227 }
228
229 let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
230 let is_volatile = is_incremental_build && mono_item.is_generic_fn();
231
232 let cgu_name = match characteristic_def_id {
233 Some(def_id) => compute_codegen_unit_name(
234 cx.tcx,
235 cgu_name_builder,
236 def_id,
237 is_volatile,
238 cgu_name_cache,
239 ),
240 None => fallback_cgu_name(cgu_name_builder),
241 };
242
243 let cgu = codegen_units.entry(cgu_name).or_insert_with(|| CodegenUnit::new(cgu_name));
244
245 let mut can_be_internalized = true;
246 let (linkage, visibility) = mono_item_linkage_and_visibility(
247 cx.tcx,
248 &mono_item,
249 &mut can_be_internalized,
250 can_export_generics,
251 always_export_generics,
252 );
253
254 let autodiff_active = cfg!(llvm_enzyme)
256 && matches!(mono_item, MonoItem::Fn(_))
257 && cx
258 .tcx
259 .codegen_fn_attrs(mono_item.def_id())
260 .autodiff_item
261 .as_ref()
262 .is_some_and(|ad| ad.is_active());
263
264 if !autodiff_active && visibility == Visibility::Hidden && can_be_internalized {
265 internalization_candidates.insert(mono_item);
266 }
267 let size_estimate = mono_item.size_estimate(cx.tcx);
268
269 cgu.items_mut()
270 .insert(mono_item, MonoItemData { inlined: false, linkage, visibility, size_estimate });
271
272 let mut reachable_inlined_items = FxIndexSet::default();
277 get_reachable_inlined_items(cx.tcx, mono_item, cx.usage_map, &mut reachable_inlined_items);
278
279 for inlined_item in reachable_inlined_items {
283 cgu.items_mut().entry(inlined_item).or_insert_with(|| MonoItemData {
285 inlined: true,
286 linkage: Linkage::Internal,
287 visibility: Visibility::Default,
288 size_estimate: inlined_item.size_estimate(cx.tcx),
289 });
290 }
291 }
292
293 if codegen_units.is_empty() {
296 let cgu_name = fallback_cgu_name(cgu_name_builder);
297 codegen_units.insert(cgu_name, CodegenUnit::new(cgu_name));
298 }
299
300 let mut codegen_units: Vec<_> = cx.tcx.with_stable_hashing_context(|ref hcx| {
301 codegen_units.into_items().map(|(_, cgu)| cgu).collect_sorted(hcx, true)
302 });
303
304 for cgu in codegen_units.iter_mut() {
305 cgu.compute_size_estimate();
306 }
307
308 return PlacedMonoItems { codegen_units, internalization_candidates };
309
310 fn get_reachable_inlined_items<'tcx>(
311 tcx: TyCtxt<'tcx>,
312 item: MonoItem<'tcx>,
313 usage_map: &UsageMap<'tcx>,
314 visited: &mut FxIndexSet<MonoItem<'tcx>>,
315 ) {
316 usage_map.for_each_inlined_used_item(tcx, item, |inlined_item| {
317 let is_new = visited.insert(inlined_item);
318 if is_new {
319 get_reachable_inlined_items(tcx, inlined_item, usage_map, visited);
320 }
321 });
322 }
323}
324
325fn merge_codegen_units<'tcx>(
328 cx: &PartitioningCx<'_, 'tcx>,
329 codegen_units: &mut Vec<CodegenUnit<'tcx>>,
330) {
331 assert!(cx.tcx.sess.codegen_units().as_usize() >= 1);
332
333 assert!(codegen_units.is_sorted_by(|a, b| a.name().as_str() <= b.name().as_str()));
335
336 let mut cgu_contents: UnordMap<Symbol, Vec<Symbol>> =
338 codegen_units.iter().map(|cgu| (cgu.name(), vec![cgu.name()])).collect();
339
340 let max_codegen_units = cx.tcx.sess.codegen_units().as_usize();
355 while codegen_units.len() > max_codegen_units {
356 codegen_units.sort_by_key(|cgu| cmp::Reverse(cgu.size_estimate()));
358
359 let cgu_dst = &codegen_units[max_codegen_units - 1];
360
361 let mut max_overlap = 0;
364 let mut max_overlap_i = max_codegen_units;
365 for (i, cgu_src) in codegen_units.iter().enumerate().skip(max_codegen_units) {
366 if cgu_src.size_estimate() <= max_overlap {
367 break;
370 }
371
372 let overlap = compute_inlined_overlap(cgu_dst, cgu_src);
373 if overlap > max_overlap {
374 max_overlap = overlap;
375 max_overlap_i = i;
376 }
377 }
378
379 let mut cgu_src = codegen_units.swap_remove(max_overlap_i);
380 let cgu_dst = &mut codegen_units[max_codegen_units - 1];
381
382 cgu_dst.items_mut().append(cgu_src.items_mut());
386 cgu_dst.compute_size_estimate();
387
388 let mut consumed_cgu_names = cgu_contents.remove(&cgu_src.name()).unwrap();
391 cgu_contents.get_mut(&cgu_dst.name()).unwrap().append(&mut consumed_cgu_names);
392 }
393
394 const NON_INCR_MIN_CGU_SIZE: usize = 1800;
400
401 while cx.tcx.sess.opts.incremental.is_none()
411 && matches!(cx.tcx.sess.codegen_units(), CodegenUnits::Default(_))
412 && codegen_units.len() > 1
413 && codegen_units.iter().any(|cgu| cgu.size_estimate() < NON_INCR_MIN_CGU_SIZE)
414 {
415 codegen_units.sort_by_key(|cgu| cmp::Reverse(cgu.size_estimate()));
417
418 let mut smallest = codegen_units.pop().unwrap();
419 let second_smallest = codegen_units.last_mut().unwrap();
420
421 second_smallest.items_mut().append(smallest.items_mut());
425 second_smallest.compute_size_estimate();
426
427 }
429
430 let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx);
431
432 if cx.tcx.sess.opts.incremental.is_some() {
434 let new_cgu_names = UnordMap::from(
440 cgu_contents
441 .items()
442 .filter(|(_, cgu_contents)| cgu_contents.len() > 1)
445 .map(|(current_cgu_name, cgu_contents)| {
446 let mut cgu_contents: Vec<&str> =
447 cgu_contents.iter().map(|s| s.as_str()).collect();
448
449 cgu_contents.sort_unstable();
453
454 (*current_cgu_name, cgu_contents.join("--"))
455 }),
456 );
457
458 for cgu in codegen_units.iter_mut() {
459 if let Some(new_cgu_name) = new_cgu_names.get(&cgu.name()) {
460 let new_cgu_name = if cx.tcx.sess.opts.unstable_opts.human_readable_cgu_names {
461 Symbol::intern(&CodegenUnit::shorten_name(new_cgu_name))
462 } else {
463 Symbol::intern(&CodegenUnit::mangle_name(new_cgu_name))
467 };
468 cgu.set_name(new_cgu_name);
469 }
470 }
471
472 codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));
474 } else {
475 codegen_units.sort_by_key(|cgu| cmp::Reverse(cgu.size_estimate()));
495 let num_digits = codegen_units.len().ilog10() as usize + 1;
496 for (index, cgu) in codegen_units.iter_mut().enumerate() {
497 let suffix = format!("{index:0num_digits$}");
501 let numbered_codegen_unit_name =
502 cgu_name_builder.build_cgu_name_no_mangle(LOCAL_CRATE, &["cgu"], Some(suffix));
503 cgu.set_name(numbered_codegen_unit_name);
504 }
505 }
506}
507
508fn compute_inlined_overlap<'tcx>(cgu1: &CodegenUnit<'tcx>, cgu2: &CodegenUnit<'tcx>) -> usize {
511 let (src_cgu, dst_cgu) =
514 if cgu1.items().len() <= cgu2.items().len() { (cgu1, cgu2) } else { (cgu2, cgu1) };
515
516 let mut overlap = 0;
517 for (item, data) in src_cgu.items().iter() {
518 if data.inlined && dst_cgu.items().contains_key(item) {
519 overlap += data.size_estimate;
520 }
521 }
522 overlap
523}
524
525fn internalize_symbols<'tcx>(
526 cx: &PartitioningCx<'_, 'tcx>,
527 codegen_units: &mut [CodegenUnit<'tcx>],
528 internalization_candidates: UnordSet<MonoItem<'tcx>>,
529) {
530 #[derive(Clone, PartialEq, Eq, Debug)]
534 enum MonoItemPlacement {
535 SingleCgu(Symbol),
536 MultipleCgus,
537 }
538
539 let mut mono_item_placements = UnordMap::default();
540 let single_codegen_unit = codegen_units.len() == 1;
541
542 if !single_codegen_unit {
543 for cgu in codegen_units.iter() {
544 for item in cgu.items().keys() {
545 match mono_item_placements.entry(*item) {
548 Entry::Occupied(e) => {
549 let placement = e.into_mut();
550 debug_assert!(match *placement {
551 MonoItemPlacement::SingleCgu(cgu_name) => cgu_name != cgu.name(),
552 MonoItemPlacement::MultipleCgus => true,
553 });
554 *placement = MonoItemPlacement::MultipleCgus;
555 }
556 Entry::Vacant(e) => {
557 e.insert(MonoItemPlacement::SingleCgu(cgu.name()));
558 }
559 }
560 }
561 }
562 }
563
564 for cgu in codegen_units {
567 let home_cgu = MonoItemPlacement::SingleCgu(cgu.name());
568
569 for (item, data) in cgu.items_mut() {
570 if !internalization_candidates.contains(item) {
571 continue;
573 }
574
575 if !single_codegen_unit {
576 debug_assert_eq!(mono_item_placements[item], home_cgu);
577
578 if cx
579 .usage_map
580 .get_user_items(*item)
581 .iter()
582 .filter_map(|user_item| {
583 mono_item_placements.get(user_item)
586 })
587 .any(|placement| *placement != home_cgu)
588 {
589 continue;
592 }
593 }
594
595 data.linkage = Linkage::Internal;
598 data.visibility = Visibility::Default;
599 }
600 }
601}
602
603fn mark_code_coverage_dead_code_cgu<'tcx>(codegen_units: &mut [CodegenUnit<'tcx>]) {
604 assert!(!codegen_units.is_empty());
605
606 let dead_code_cgu = codegen_units
614 .iter_mut()
615 .filter(|cgu| cgu.items().iter().any(|(_, data)| data.linkage == Linkage::External))
616 .min_by_key(|cgu| cgu.size_estimate());
617
618 let dead_code_cgu = if let Some(cgu) = dead_code_cgu { cgu } else { &mut codegen_units[0] };
621
622 dead_code_cgu.make_code_coverage_dead_code_cgu();
623}
624
625fn characteristic_def_id_of_mono_item<'tcx>(
626 tcx: TyCtxt<'tcx>,
627 mono_item: MonoItem<'tcx>,
628) -> Option<DefId> {
629 match mono_item {
630 MonoItem::Fn(instance) => {
631 let def_id = match instance.def {
632 ty::InstanceKind::Item(def) => def,
633 ty::InstanceKind::VTableShim(..)
634 | ty::InstanceKind::ReifyShim(..)
635 | ty::InstanceKind::FnPtrShim(..)
636 | ty::InstanceKind::ClosureOnceShim { .. }
637 | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
638 | ty::InstanceKind::Intrinsic(..)
639 | ty::InstanceKind::DropGlue(..)
640 | ty::InstanceKind::Virtual(..)
641 | ty::InstanceKind::CloneShim(..)
642 | ty::InstanceKind::ThreadLocalShim(..)
643 | ty::InstanceKind::FnPtrAddrShim(..)
644 | ty::InstanceKind::FutureDropPollShim(..)
645 | ty::InstanceKind::AsyncDropGlue(..)
646 | ty::InstanceKind::AsyncDropGlueCtorShim(..) => return None,
647 };
648
649 if tcx.trait_of_assoc(def_id).is_some() {
654 let self_ty = instance.args.type_at(0);
655 return characteristic_def_id_of_type(self_ty).or(Some(def_id));
657 }
658
659 if let Some(impl_def_id) = tcx.impl_of_assoc(def_id) {
660 if tcx.sess.opts.incremental.is_some()
661 && tcx
662 .trait_id_of_impl(impl_def_id)
663 .is_some_and(|def_id| tcx.is_lang_item(def_id, LangItem::Drop))
664 {
665 return None;
669 }
670
671 let impl_self_ty = tcx.instantiate_and_normalize_erasing_regions(
673 instance.args,
674 ty::TypingEnv::fully_monomorphized(),
675 tcx.type_of(impl_def_id),
676 );
677 if let Some(def_id) = characteristic_def_id_of_type(impl_self_ty) {
678 return Some(def_id);
679 }
680 }
681
682 Some(def_id)
683 }
684 MonoItem::Static(def_id) => Some(def_id),
685 MonoItem::GlobalAsm(item_id) => Some(item_id.owner_id.to_def_id()),
686 }
687}
688
689fn compute_codegen_unit_name(
690 tcx: TyCtxt<'_>,
691 name_builder: &mut CodegenUnitNameBuilder<'_>,
692 def_id: DefId,
693 volatile: bool,
694 cache: &mut CguNameCache,
695) -> Symbol {
696 let mut current_def_id = def_id;
698 let mut cgu_def_id = None;
699 loop {
701 if current_def_id.is_crate_root() {
702 if cgu_def_id.is_none() {
703 cgu_def_id = Some(def_id.krate.as_def_id());
705 }
706 break;
707 } else if tcx.def_kind(current_def_id) == DefKind::Mod {
708 if cgu_def_id.is_none() {
709 cgu_def_id = Some(current_def_id);
710 }
711 } else {
712 cgu_def_id = None;
716 }
717
718 current_def_id = tcx.parent(current_def_id);
719 }
720
721 let cgu_def_id = cgu_def_id.unwrap();
722
723 *cache.entry((cgu_def_id, volatile)).or_insert_with(|| {
724 let def_path = tcx.def_path(cgu_def_id);
725
726 let components = def_path.data.iter().map(|part| match part.data.name() {
727 DefPathDataName::Named(name) => name,
728 DefPathDataName::Anon { .. } => unreachable!(),
729 });
730
731 let volatile_suffix = volatile.then_some("volatile");
732
733 name_builder.build_cgu_name(def_path.krate, components, volatile_suffix)
734 })
735}
736
737fn fallback_cgu_name(name_builder: &mut CodegenUnitNameBuilder<'_>) -> Symbol {
739 name_builder.build_cgu_name(LOCAL_CRATE, &["fallback"], Some("cgu"))
740}
741
742fn mono_item_linkage_and_visibility<'tcx>(
743 tcx: TyCtxt<'tcx>,
744 mono_item: &MonoItem<'tcx>,
745 can_be_internalized: &mut bool,
746 can_export_generics: bool,
747 always_export_generics: bool,
748) -> (Linkage, Visibility) {
749 if let Some(explicit_linkage) = mono_item.explicit_linkage(tcx) {
750 return (explicit_linkage, Visibility::Default);
751 }
752 let vis = mono_item_visibility(
753 tcx,
754 mono_item,
755 can_be_internalized,
756 can_export_generics,
757 always_export_generics,
758 );
759 (Linkage::External, vis)
760}
761
762type CguNameCache = UnordMap<(DefId, bool), Symbol>;
763
764fn static_visibility<'tcx>(
765 tcx: TyCtxt<'tcx>,
766 can_be_internalized: &mut bool,
767 def_id: DefId,
768) -> Visibility {
769 if tcx.is_reachable_non_generic(def_id) {
770 *can_be_internalized = false;
771 default_visibility(tcx, def_id, false)
772 } else {
773 Visibility::Hidden
774 }
775}
776
777fn mono_item_visibility<'tcx>(
778 tcx: TyCtxt<'tcx>,
779 mono_item: &MonoItem<'tcx>,
780 can_be_internalized: &mut bool,
781 can_export_generics: bool,
782 always_export_generics: bool,
783) -> Visibility {
784 let instance = match mono_item {
785 MonoItem::Fn(instance) => instance,
787
788 MonoItem::Static(def_id) => return static_visibility(tcx, can_be_internalized, *def_id),
790 MonoItem::GlobalAsm(item_id) => {
791 return static_visibility(tcx, can_be_internalized, item_id.owner_id.to_def_id());
792 }
793 };
794
795 let def_id = match instance.def {
796 InstanceKind::Item(def_id)
797 | InstanceKind::DropGlue(def_id, Some(_))
798 | InstanceKind::FutureDropPollShim(def_id, _, _)
799 | InstanceKind::AsyncDropGlue(def_id, _)
800 | InstanceKind::AsyncDropGlueCtorShim(def_id, _) => def_id,
801
802 InstanceKind::ThreadLocalShim(def_id) => {
804 return static_visibility(tcx, can_be_internalized, def_id);
805 }
806
807 InstanceKind::VTableShim(..)
809 | InstanceKind::ReifyShim(..)
810 | InstanceKind::FnPtrShim(..)
811 | InstanceKind::Virtual(..)
812 | InstanceKind::Intrinsic(..)
813 | InstanceKind::ClosureOnceShim { .. }
814 | InstanceKind::ConstructCoroutineInClosureShim { .. }
815 | InstanceKind::DropGlue(..)
816 | InstanceKind::CloneShim(..)
817 | InstanceKind::FnPtrAddrShim(..) => return Visibility::Hidden,
818 };
819
820 if tcx.is_entrypoint(def_id) {
833 *can_be_internalized = false;
834 return Visibility::Hidden;
835 }
836
837 let is_generic = instance.args.non_erasable_generics().next().is_some();
838
839 let Some(def_id) = def_id.as_local() else {
841 return if is_generic
842 && (always_export_generics
843 || (can_export_generics
844 && tcx.codegen_fn_attrs(def_id).inline == InlineAttr::Never))
845 {
846 *can_be_internalized = false;
849 default_visibility(tcx, def_id, true)
850 } else {
851 Visibility::Hidden
852 };
853 };
854
855 if is_generic {
856 if always_export_generics
857 || (can_export_generics && tcx.codegen_fn_attrs(def_id).inline == InlineAttr::Never)
858 {
859 if tcx.is_unreachable_local_definition(def_id) {
860 Visibility::Hidden
862 } else {
863 *can_be_internalized = false;
865 default_visibility(tcx, def_id.to_def_id(), true)
866 }
867 } else {
868 Visibility::Hidden
871 }
872 } else {
873 if tcx.is_reachable_non_generic(def_id.to_def_id()) {
877 *can_be_internalized = false;
878 debug_assert!(!is_generic);
879 return default_visibility(tcx, def_id.to_def_id(), false);
880 }
881
882 let attrs = tcx.codegen_fn_attrs(def_id);
917 if attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL) {
918 *can_be_internalized = false;
919 }
920
921 Visibility::Hidden
922 }
923}
924
925fn default_visibility(tcx: TyCtxt<'_>, id: DefId, is_generic: bool) -> Visibility {
926 if tcx.sess.default_visibility() == SymbolVisibility::Interposable {
928 return Visibility::Default;
929 }
930
931 let export_level = if is_generic {
932 SymbolExportLevel::Rust
934 } else {
935 match tcx.reachable_non_generics(id.krate).get(&id) {
936 Some(SymbolExportInfo { level: SymbolExportLevel::C, .. }) => SymbolExportLevel::C,
937 _ => SymbolExportLevel::Rust,
938 }
939 };
940
941 match export_level {
942 SymbolExportLevel::C => Visibility::Default,
945
946 SymbolExportLevel::Rust => tcx.sess.default_visibility().into(),
948 }
949}
950
951fn debug_dump<'a, 'tcx: 'a>(tcx: TyCtxt<'tcx>, label: &str, cgus: &[CodegenUnit<'tcx>]) {
952 let dump = move || {
953 use std::fmt::Write;
954
955 let mut num_cgus = 0;
956 let mut all_cgu_sizes = Vec::new();
957
958 let mut inlined_items = UnordSet::default();
964
965 let mut root_items = 0;
966 let mut unique_inlined_items = 0;
967 let mut placed_inlined_items = 0;
968
969 let mut root_size = 0;
970 let mut unique_inlined_size = 0;
971 let mut placed_inlined_size = 0;
972
973 for cgu in cgus.iter() {
974 num_cgus += 1;
975 all_cgu_sizes.push(cgu.size_estimate());
976
977 for (item, data) in cgu.items() {
978 if !data.inlined {
979 root_items += 1;
980 root_size += data.size_estimate;
981 } else {
982 if inlined_items.insert(item) {
983 unique_inlined_items += 1;
984 unique_inlined_size += data.size_estimate;
985 }
986 placed_inlined_items += 1;
987 placed_inlined_size += data.size_estimate;
988 }
989 }
990 }
991
992 all_cgu_sizes.sort_unstable_by_key(|&n| cmp::Reverse(n));
993
994 let unique_items = root_items + unique_inlined_items;
995 let placed_items = root_items + placed_inlined_items;
996 let items_ratio = placed_items as f64 / unique_items as f64;
997
998 let unique_size = root_size + unique_inlined_size;
999 let placed_size = root_size + placed_inlined_size;
1000 let size_ratio = placed_size as f64 / unique_size as f64;
1001
1002 let mean_cgu_size = placed_size as f64 / num_cgus as f64;
1003
1004 assert_eq!(placed_size, all_cgu_sizes.iter().sum::<usize>());
1005
1006 let s = &mut String::new();
1007 let _ = writeln!(s, "{label}");
1008 let _ = writeln!(
1009 s,
1010 "- unique items: {unique_items} ({root_items} root + {unique_inlined_items} inlined), \
1011 unique size: {unique_size} ({root_size} root + {unique_inlined_size} inlined)\n\
1012 - placed items: {placed_items} ({root_items} root + {placed_inlined_items} inlined), \
1013 placed size: {placed_size} ({root_size} root + {placed_inlined_size} inlined)\n\
1014 - placed/unique items ratio: {items_ratio:.2}, \
1015 placed/unique size ratio: {size_ratio:.2}\n\
1016 - CGUs: {num_cgus}, mean size: {mean_cgu_size:.1}, sizes: {}",
1017 list(&all_cgu_sizes),
1018 );
1019 let _ = writeln!(s);
1020
1021 for (i, cgu) in cgus.iter().enumerate() {
1022 let name = cgu.name();
1023 let size = cgu.size_estimate();
1024 let num_items = cgu.items().len();
1025 let mean_size = size as f64 / num_items as f64;
1026
1027 let mut placed_item_sizes: Vec<_> =
1028 cgu.items().values().map(|data| data.size_estimate).collect();
1029 placed_item_sizes.sort_unstable_by_key(|&n| cmp::Reverse(n));
1030 let sizes = list(&placed_item_sizes);
1031
1032 let _ = writeln!(s, "- CGU[{i}]");
1033 let _ = writeln!(s, " - {name}, size: {size}");
1034 let _ =
1035 writeln!(s, " - items: {num_items}, mean size: {mean_size:.1}, sizes: {sizes}",);
1036
1037 for (item, data) in cgu.items_in_deterministic_order(tcx) {
1038 let linkage = data.linkage;
1039 let symbol_name = item.symbol_name(tcx).name;
1040 let symbol_hash_start = symbol_name.rfind('h');
1041 let symbol_hash = symbol_hash_start.map_or("<no hash>", |i| &symbol_name[i..]);
1042 let kind = if !data.inlined { "root" } else { "inlined" };
1043 let size = data.size_estimate;
1044 let _ = with_no_trimmed_paths!(writeln!(
1045 s,
1046 " - {item} [{linkage:?}] [{symbol_hash}] ({kind}, size: {size})"
1047 ));
1048 }
1049
1050 let _ = writeln!(s);
1051 }
1052
1053 return std::mem::take(s);
1054
1055 fn list(ns: &[usize]) -> String {
1058 let mut v = Vec::new();
1059 if ns.is_empty() {
1060 return "[]".to_string();
1061 }
1062
1063 let mut elem = |curr, curr_count| {
1064 if curr_count == 1 {
1065 v.push(format!("{curr}"));
1066 } else {
1067 v.push(format!("{curr} (x{curr_count})"));
1068 }
1069 };
1070
1071 let mut curr = ns[0];
1072 let mut curr_count = 1;
1073
1074 for &n in &ns[1..] {
1075 if n != curr {
1076 elem(curr, curr_count);
1077 curr = n;
1078 curr_count = 1;
1079 } else {
1080 curr_count += 1;
1081 }
1082 }
1083 elem(curr, curr_count);
1084
1085 format!("[{}]", v.join(", "))
1086 }
1087 };
1088
1089 debug!("{}", dump());
1090}
1091
1092#[inline(never)] fn assert_symbols_are_distinct<'a, 'tcx, I>(tcx: TyCtxt<'tcx>, mono_items: I)
1094where
1095 I: Iterator<Item = &'a MonoItem<'tcx>>,
1096 'tcx: 'a,
1097{
1098 let _prof_timer = tcx.prof.generic_activity("assert_symbols_are_distinct");
1099
1100 let mut symbols: Vec<_> =
1101 mono_items.map(|mono_item| (mono_item, mono_item.symbol_name(tcx))).collect();
1102
1103 symbols.sort_by_key(|sym| sym.1);
1104
1105 for &[(mono_item1, ref sym1), (mono_item2, ref sym2)] in symbols.array_windows() {
1106 if sym1 == sym2 {
1107 let span1 = mono_item1.local_span(tcx);
1108 let span2 = mono_item2.local_span(tcx);
1109
1110 let span = match (span1, span2) {
1112 (Some(span1), Some(span2)) => {
1113 Some(if span1.lo().0 > span2.lo().0 { span1 } else { span2 })
1114 }
1115 (span1, span2) => span1.or(span2),
1116 };
1117
1118 tcx.dcx().emit_fatal(SymbolAlreadyDefined { span, symbol: sym1.to_string() });
1119 }
1120 }
1121}
1122
1123fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> MonoItemPartitions<'_> {
1124 let collection_strategy = if tcx.sess.link_dead_code() {
1125 MonoItemCollectionStrategy::Eager
1126 } else {
1127 MonoItemCollectionStrategy::Lazy
1128 };
1129
1130 let (items, usage_map) = collector::collect_crate_mono_items(tcx, collection_strategy);
1131
1132 tcx.dcx().abort_if_errors();
1136
1137 let (codegen_units, _) = tcx.sess.time("partition_and_assert_distinct_symbols", || {
1138 sync::join(
1139 || {
1140 let mut codegen_units = partition(tcx, items.iter().copied(), &usage_map);
1141 codegen_units[0].make_primary();
1142 &*tcx.arena.alloc_from_iter(codegen_units)
1143 },
1144 || assert_symbols_are_distinct(tcx, items.iter()),
1145 )
1146 });
1147
1148 if tcx.prof.enabled() {
1149 for cgu in codegen_units {
1151 tcx.prof.artifact_size(
1152 "codegen_unit_size_estimate",
1153 cgu.name().as_str(),
1154 cgu.size_estimate() as u64,
1155 );
1156 }
1157 }
1158
1159 #[cfg(not(llvm_enzyme))]
1160 let autodiff_mono_items: Vec<_> = vec![];
1161 #[cfg(llvm_enzyme)]
1162 let mut autodiff_mono_items: Vec<_> = vec![];
1163 let mono_items: DefIdSet = items
1164 .iter()
1165 .filter_map(|mono_item| match *mono_item {
1166 MonoItem::Fn(ref instance) => {
1167 #[cfg(llvm_enzyme)]
1168 autodiff_mono_items.push((mono_item, instance));
1169 Some(instance.def_id())
1170 }
1171 MonoItem::Static(def_id) => Some(def_id),
1172 _ => None,
1173 })
1174 .collect();
1175
1176 let autodiff_items =
1177 autodiff::find_autodiff_source_functions(tcx, &usage_map, autodiff_mono_items);
1178 let autodiff_items = tcx.arena.alloc_from_iter(autodiff_items);
1179
1180 if let SwitchWithOptPath::Enabled(ref path) = tcx.sess.opts.unstable_opts.dump_mono_stats
1182 && let Err(err) =
1183 dump_mono_items_stats(tcx, codegen_units, path, tcx.crate_name(LOCAL_CRATE))
1184 {
1185 tcx.dcx().emit_fatal(CouldntDumpMonoStats { error: err.to_string() });
1186 }
1187
1188 if tcx.sess.opts.unstable_opts.print_mono_items {
1189 let mut item_to_cgus: UnordMap<_, Vec<_>> = Default::default();
1190
1191 for cgu in codegen_units {
1192 for (&mono_item, &data) in cgu.items() {
1193 item_to_cgus.entry(mono_item).or_default().push((cgu.name(), data.linkage));
1194 }
1195 }
1196
1197 let mut item_keys: Vec<_> = items
1198 .iter()
1199 .map(|i| {
1200 let mut output = with_no_trimmed_paths!(i.to_string());
1201 output.push_str(" @@");
1202 let mut empty = Vec::new();
1203 let cgus = item_to_cgus.get_mut(i).unwrap_or(&mut empty);
1204 cgus.sort_by_key(|(name, _)| *name);
1205 cgus.dedup();
1206 for &(ref cgu_name, linkage) in cgus.iter() {
1207 output.push(' ');
1208 output.push_str(cgu_name.as_str());
1209
1210 let linkage_abbrev = match linkage {
1211 Linkage::External => "External",
1212 Linkage::AvailableExternally => "Available",
1213 Linkage::LinkOnceAny => "OnceAny",
1214 Linkage::LinkOnceODR => "OnceODR",
1215 Linkage::WeakAny => "WeakAny",
1216 Linkage::WeakODR => "WeakODR",
1217 Linkage::Internal => "Internal",
1218 Linkage::ExternalWeak => "ExternalWeak",
1219 Linkage::Common => "Common",
1220 };
1221
1222 output.push('[');
1223 output.push_str(linkage_abbrev);
1224 output.push(']');
1225 }
1226 output
1227 })
1228 .collect();
1229
1230 item_keys.sort();
1231
1232 for item in item_keys {
1233 println!("MONO_ITEM {item}");
1234 }
1235 }
1236
1237 MonoItemPartitions {
1238 all_mono_items: tcx.arena.alloc(mono_items),
1239 codegen_units,
1240 autodiff_items,
1241 }
1242}
1243
1244fn dump_mono_items_stats<'tcx>(
1247 tcx: TyCtxt<'tcx>,
1248 codegen_units: &[CodegenUnit<'tcx>],
1249 output_directory: &Option<PathBuf>,
1250 crate_name: Symbol,
1251) -> Result<(), Box<dyn std::error::Error>> {
1252 let output_directory = if let Some(directory) = output_directory {
1253 fs::create_dir_all(directory)?;
1254 directory
1255 } else {
1256 Path::new(".")
1257 };
1258
1259 let format = tcx.sess.opts.unstable_opts.dump_mono_stats_format;
1260 let ext = format.extension();
1261 let filename = format!("{crate_name}.mono_items.{ext}");
1262 let output_path = output_directory.join(&filename);
1263 let mut file = File::create_buffered(&output_path)?;
1264
1265 let mut items_per_def_id: FxIndexMap<_, Vec<_>> = Default::default();
1267 for cgu in codegen_units {
1268 cgu.items()
1269 .keys()
1270 .filter(|mono_item| mono_item.is_user_defined())
1272 .for_each(|mono_item| {
1273 items_per_def_id.entry(mono_item.def_id()).or_default().push(mono_item);
1274 });
1275 }
1276
1277 #[derive(serde::Serialize)]
1278 struct MonoItem {
1279 name: String,
1280 instantiation_count: usize,
1281 size_estimate: usize,
1282 total_estimate: usize,
1283 }
1284
1285 let mut stats: Vec<_> = items_per_def_id
1287 .into_iter()
1288 .map(|(def_id, items)| {
1289 let name = with_no_trimmed_paths!(tcx.def_path_str(def_id));
1290 let instantiation_count = items.len();
1291 let size_estimate = items[0].size_estimate(tcx);
1292 let total_estimate = instantiation_count * size_estimate;
1293 MonoItem { name, instantiation_count, size_estimate, total_estimate }
1294 })
1295 .collect();
1296 stats.sort_unstable_by_key(|item| cmp::Reverse(item.total_estimate));
1297
1298 if !stats.is_empty() {
1299 match format {
1300 DumpMonoStatsFormat::Json => serde_json::to_writer(file, &stats)?,
1301 DumpMonoStatsFormat::Markdown => {
1302 writeln!(
1303 file,
1304 "| Item | Instantiation count | Estimated Cost Per Instantiation | Total Estimated Cost |"
1305 )?;
1306 writeln!(file, "| --- | ---: | ---: | ---: |")?;
1307
1308 for MonoItem { name, instantiation_count, size_estimate, total_estimate } in stats {
1309 writeln!(
1310 file,
1311 "| `{name}` | {instantiation_count} | {size_estimate} | {total_estimate} |"
1312 )?;
1313 }
1314 }
1315 }
1316 }
1317
1318 Ok(())
1319}
1320
1321pub(crate) fn provide(providers: &mut Providers) {
1322 providers.collect_and_partition_mono_items = collect_and_partition_mono_items;
1323
1324 providers.is_codegened_item =
1325 |tcx, def_id| tcx.collect_and_partition_mono_items(()).all_mono_items.contains(&def_id);
1326
1327 providers.codegen_unit = |tcx, name| {
1328 tcx.collect_and_partition_mono_items(())
1329 .codegen_units
1330 .iter()
1331 .find(|cgu| cgu.name() == name)
1332 .unwrap_or_else(|| panic!("failed to find cgu with name {name:?}"))
1333 };
1334
1335 providers.size_estimate = |tcx, instance| {
1336 match instance.def {
1337 InstanceKind::Item(..)
1340 | InstanceKind::DropGlue(..)
1341 | InstanceKind::AsyncDropGlueCtorShim(..) => {
1342 let mir = tcx.instance_mir(instance.def);
1343 mir.basic_blocks.iter().map(|bb| bb.statements.len() + 1).sum()
1344 }
1345 _ => 1,
1347 }
1348 };
1349
1350 collector::provide(providers);
1351}