1//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7//===----------------------------------------------------------------------===// 9// This file implements Function import based on summaries. 11//===----------------------------------------------------------------------===// 50#include <system_error> 56#define DEBUG_TYPE "function-import" 59"Number of functions thin link decided to import");
61"Number of hot functions thin link decided to import");
63"Number of critical functions thin link decided to import");
65"Number of global variables thin link decided to import");
66STATISTIC(NumImportedFunctions,
"Number of functions imported in backend");
68"Number of global variables imported in backend");
69STATISTIC(NumImportedModules,
"Number of modules imported from");
70STATISTIC(NumDeadSymbols,
"Number of dead stripped symbols in index");
71STATISTIC(NumLiveSymbols,
"Number of live symbols in index");
73/// Limit on instruction count of imported functions. 76cl::desc(
"Only import functions with less than N instructions"));
80cl::desc(
"Only import first N functions if N>=0 (default -1)"));
84cl::desc(
"Import functions with noinline attribute"));
89cl::desc(
"As we import functions, multiply the " 90"`import-instr-limit` threshold by this factor " 91"before processing newly imported functions"));
96cl::desc(
"As we import functions called from hot callsite, multiply the " 97"`import-instr-limit` threshold by this factor " 98"before processing newly imported functions"));
102cl::desc(
"Multiply the `import-instr-limit` threshold for hot callsites"));
108"Multiply the `import-instr-limit` threshold for critical callsites"));
110// FIXME: This multiplier was not really tuned up. 113cl::desc(
"Multiply the `import-instr-limit` threshold for cold callsites"));
116cl::desc(
"Print imported functions"));
120cl::desc(
"Print information for functions rejected for importing"));
127cl::desc(
"Enable import metadata like 'thinlto_src_module' and " 128"'thinlto_src_file'"));
130/// Summary file to use for function importing when using -function-import from 134cl::desc(
"The summary file to use for function importing."));
136/// Used when testing importing from distributed indexes via opt 140cl::desc(
"Import all external functions in index."));
142/// This is a test-only option. 143/// If this option is enabled, the ThinLTO indexing step will import each 144/// function declaration as a fallback. In a real build this may increase ram 145/// usage of the indexing step unnecessarily. 146/// TODO: Implement selective import (based on combined summary analysis) to 147/// ensure the imported function has a use case in the postlink pipeline. 150cl::desc(
"If true, import function declaration as fallback if the function " 151"definition is not imported."));
153/// Pass a workload description file - an example of workload would be the 154/// functions executed to satisfy a RPC request. A workload is defined by a root 155/// function and the list of functions that are (frequently) needed to satisfy 156/// it. The module that defines the root will have all those functions imported. 157/// The file contains a JSON dictionary. The keys are root functions, the values 158/// are lists of functions to import in the module defining the root. It is 159/// assumed -funique-internal-linkage-names was used, thus ensuring function 160/// names are unique even for local linkage ones. 162"thinlto-workload-def",
163cl::desc(
"Pass a workload definition. This is a file containing a JSON " 164"dictionary. The keys are root functions, the values are lists of " 165"functions to import in the module defining the root. It is " 166"assumed -funique-internal-linkage-names was used, to ensure " 167"local linkage functions have unique names. For example: \n" 169" \"rootFunction_1\": [\"function_to_import_1\", " 170"\"function_to_import_2\"], \n" 171" \"rootFunction_2\": [\"function_to_import_3\", " 172"\"function_to_import_4\"] \n" 182// Load lazily a module from \p FileName in \p Context. 183static std::unique_ptr<Module>
loadFile(
const std::string &FileName,
187// Metadata isn't loaded until functions are imported, to minimize 188// the memory overhead. 189 std::unique_ptr<Module> Result =
191/* ShouldLazyLoadMetadata = */true);
193 Err.print(
"function-import",
errs());
203// We can import a local when there is one definition. 206// In other cases, make sure we import the copy in the caller's module if the 207// referenced value has local linkage. The only time a local variable can 208// share an entry in the index is if there is a local with the same name in 209// another module that had the same source file name (in a different 210// directory), where each was compiled in their own directory so there was not 211// distinguishing path. 216/// Given a list of possible callee implementation for a call site, qualify the 217/// legality of importing each. The return is a range of pairs. Each pair 218/// corresponds to a candidate. The first value is the ImportFailureReason for 219/// that candidate, the second is the candidate. 222ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
226 [&Index, CalleeSummaryList,
227 CallerModulePath](
const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
230auto *GVSummary = SummaryPtr.get();
231if (!Index.isGlobalValueLive(GVSummary))
232return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
235return {FunctionImporter::ImportFailureReason::InterposableLinkage,
238auto *Summary = dyn_cast<FunctionSummary>(GVSummary->getBaseObject());
240// Ignore any callees that aren't actually functions. This could happen 241// in the case of GUID hash collisions. It could also happen in theory 242// for SamplePGO profiles collected on old versions of the code after 243// renaming, since we synthesize edges to any inlined callees appearing 246return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
248// If this is a local function, make sure we import the copy in the 249// caller's module. The only time a local function can share an entry in 250// the index is if there is a local with the same name in another module 251// that had the same source file name (in a different directory), where 252// each was compiled in their own directory so there was not 253// distinguishing path. 254// If the local function is from another module, it must be a reference 255// due to indirect call profile data since a function pointer can point 256// to a local in another module. Do the import from another module if 257// there is only one entry in the list or when all files in the program 258// are compiled with full path - in both cases the local function has 259// unique PGO name and GUID. 263 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
266// Skip if it isn't legal to import (e.g. may reference unpromotable 268if (Summary->notEligibleToImport())
269return {FunctionImporter::ImportFailureReason::NotEligible,
272return {FunctionImporter::ImportFailureReason::None, GVSummary};
276/// Given a list of possible callee implementation for a call site, select one 277/// that fits the \p Threshold for function definition import. If none are 278/// found, the Reason will give the last reason for the failure (last, in the 279/// order of CalleeSummaryList entries). While looking for a callee definition, 280/// sets \p TooLargeOrNoInlineSummary to the last seen too-large or noinline 281/// candidate; other modules may want to know the function summary or 282/// declaration even if a definition is not needed. 284/// FIXME: select "best" instead of first that fits. But what is "best"? 285/// - The smallest: more likely to be inlined. 286/// - The one with the least outgoing edges (already well optimized). 287/// - One from a module already being imported from in order to reduce the 288/// number of source modules parsed/linked. 289/// - One that has PGO data attached. 290/// - [insert you fancy metric here] 293ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
294unsigned Threshold,
StringRef CallerModulePath,
297// Records the last summary with reason noinline or too-large. 298 TooLargeOrNoInlineSummary =
nullptr;
299auto QualifiedCandidates =
301for (
auto QualifiedValue : QualifiedCandidates) {
302 Reason = QualifiedValue.first;
303// Skip a summary if its import is not (proved to be) legal. 304if (Reason != FunctionImporter::ImportFailureReason::None)
307 cast<FunctionSummary>(QualifiedValue.second->getBaseObject());
309// Don't bother importing the definition if the chance of inlining it is 310// not high enough (except under `--force-import-all`). 311if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
313 TooLargeOrNoInlineSummary = Summary;
314 Reason = FunctionImporter::ImportFailureReason::TooLarge;
318// Don't bother importing the definition if we can't inline it anyway. 320 TooLargeOrNoInlineSummary = Summary;
321 Reason = FunctionImporter::ImportFailureReason::NoInline;
332usingEdgeInfo = std::tuple<
constFunctionSummary *,
unsigned/* Threshold */>;
334}
// anonymous namespace 340if (!Imports.insert(Def).second)
344// Remove Decl in case it's there. Note that a definition takes precedence 345// over a declaration for a given GUID. 352auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);
353// Insert Decl only if Def is not present. Note that a definition takes 354// precedence over a declaration for a given GUID. 355if (!Imports.contains(Def))
356 Imports.insert(Decl);
362for (
constauto &[SrcMod, GUID, ImportType] : *
this)
369std::optional<GlobalValueSummary::ImportKind>
372if (
auto IDPair = IDs.getImportIDs(FromModule, GUID)) {
373auto [Def, Decl] = *IDPair;
374if (Imports.contains(Def))
376if (Imports.contains(Decl))
382/// Import globals referenced by a function or other globals that are being 383/// imported, if importing such global is possible. 392bool shouldImportGlobal(
constValueInfo &VI) {
393constauto &GVS = DefinedGVSummaries.
find(VI.getGUID());
394if (GVS == DefinedGVSummaries.
end())
396// We should not skip import if the module contains a non-prevailing 397// definition with interposable linkage type. This is required for 398// correctness in the situation where there is a prevailing def available 399// for import and marked read-only. In this case, the non-prevailing def 400// will be converted to a declaration, while the prevailing one becomes 401// internal, thus no definitions will be available for linking. In order to 402// prevent undefined symbol link error, the prevailing definition must be 404// FIXME: Consider adding a check that the suitable prevailing definition 405// exists and marked read-only. 406if (VI.getSummaryList().size() > 1 &&
408 !IsPrevailing(VI.getGUID(), GVS->second))
417for (
constauto &VI : Summary.refs()) {
418if (!shouldImportGlobal(VI)) {
420dbgs() <<
"Ref ignored! Target already in destination module.\n");
426for (
constauto &RefSummary : VI.getSummaryList()) {
427constauto *GVS = dyn_cast<GlobalVarSummary>(RefSummary.get());
428// Functions could be referenced by global vars - e.g. a vtable; but we 429// don't currently imagine a reason those would be imported here, rather 430// than as part of the logic deciding which functions to import (i.e. 431// based on profile information). Should we decide to handle them here, 432// we can refactor accordingly at that time. 433bool CanImportDecl =
false;
436 Summary.modulePath()) ||
437 !
Index.canImportGlobalVar(GVS,
/* AnalyzeRefs */true,
446// If there isn't an entry for GUID, insert <GUID, Definition> pair. 447// Otherwise, definition should take precedence over declaration. 448if (ImportList.
addDefinition(RefSummary->modulePath(), VI.getGUID()) !=
449 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)
452// Only update stat and exports if we haven't already imported this 454 NumImportedGlobalVarsThinLink++;
455// Any references made by this variable will be marked exported 456// later, in ComputeCrossModuleImport, after import decisions are 457// complete, which is more efficient than adding them here. 459 (*ExportLists)[RefSummary->modulePath()].insert(VI);
461// If variable is not writeonly we attempt to recursively analyze 462// its references in order to import referenced constants. 463if (!
Index.isWriteOnly(GVS))
477 :
Index(
Index), DefinedGVSummaries(DefinedGVSummaries),
478 IsPrevailing(IsPrevailing), ImportList(ImportList),
479 ExportLists(ExportLists) {}
483 onImportingSummaryImpl(Summary, Worklist);
484while (!Worklist.
empty())
485 onImportingSummaryImpl(*Worklist.
pop_back_val(), Worklist);
491/// Determine the list of imports and exports for each module. 504 : IsPrevailing(IsPrevailing),
Index(
Index), ExportLists(ExportLists) {}
509 /// Given the list of globals defined in a module, compute the list of imports 510 /// as well as the list of "exports", i.e. the list of symbols referenced from 511 /// another module (that may require promotion). 517static std::unique_ptr<ModuleImportsManager>
525/// A ModuleImportsManager that operates based on a workload definition (see 526/// -thinlto-workload-def). For modules that do not define workload roots, it 527/// applies the base ModuleImportsManager import policy. 529// Keep a module name -> value infos to import association. We use it to 530// determine if a module's import list should be done by the base 531// ModuleImportsManager or by us. 538auto SetIter = Workloads.
find(ModName);
539if (SetIter == Workloads.
end()) {
541 <<
" does not contain the root of any context.\n");
543 ModName, ImportList);
546 <<
" contains the root(s) of context(s).\n");
550auto &ValueInfos = SetIter->second;
553auto It = DefinedGVSummaries.
find(VI.getGUID());
554if (It != DefinedGVSummaries.
end() &&
555 IsPrevailing(VI.getGUID(), It->second)) {
557dbgs() <<
"[Workload] " << VI.name()
558 <<
" has the prevailing variant already in the module " 559 << ModName <<
". No need to import\n");
569 [&](
constauto &Candidate) {
571 <<
" from " << Candidate.second->modulePath()
572 <<
" ImportFailureReason: " 574return Candidate.first ==
575 FunctionImporter::ImportFailureReason::None;
577 [](
constauto &Candidate) {
return Candidate.second; });
578if (PotentialCandidates.empty()) {
580 <<
" because can't find eligible Callee. Guid is: " 581 << Function::getGUID(VI.name()) <<
"\n");
584 /// We will prefer importing the prevailing candidate, if not, we'll 585 /// still pick the first available candidate. The reason we want to make 586 /// sure we do import the prevailing candidate is because the goal of 587 /// workload-awareness is to enable optimizations specializing the call 588 /// graph of that workload. Suppose a function is already defined in the 589 /// module, but it's not the prevailing variant. Suppose also we do not 590 /// inline it (in fact, if it were interposable, we can't inline it), 591 /// but we could specialize it to the workload in other ways. However, 592 /// the linker would drop it in the favor of the prevailing copy. 593 /// Instead, by importing the prevailing variant (assuming also the use 594 /// of `-avail-extern-to-local`), we keep the specialization. We could 595 /// alteranatively make the non-prevailing variant local, but the 596 /// prevailing one is also the one for which we would have previously 597 /// collected profiles, making it preferrable. 599 PotentialCandidates, [&](
constauto *Candidate) {
600return IsPrevailing(VI.getGUID(), Candidate);
602if (PrevailingCandidates.empty()) {
603 GVS = *PotentialCandidates.begin();
608 <<
"[Workload] Found multiple non-prevailing candidates for " 610 <<
". This is unexpected. Are module paths passed to the " 611"compiler unique for the modules passed to the linker?");
612// We could in theory have multiple (interposable) copies of a symbol 613// when there is no prevailing candidate, if say the prevailing copy was 614// in a native object being linked in. However, we should in theory be 615// marking all of these non-prevailing IR copies dead in that case, in 616// which case they won't be candidates. 620 GVS = *PrevailingCandidates.begin();
624// We checked that for the prevailing case, but if we happen to have for 625// example an internal that's defined in this module, it'd have no 626// PrevailingCandidates. 627if (ExportingModule == ModName) {
629 <<
" because its defining module is the same as the " 633LLVM_DEBUG(
dbgs() <<
"[Workload][Including]" << VI.name() <<
" from " 634 << ExportingModule <<
" : " 635 << Function::getGUID(VI.name()) <<
"\n");
639 (*ExportLists)[ExportingModule].insert(VI);
645// Since the workload def uses names, we need a quick lookup 651if (!NameToValueInfo.
insert(std::make_pair(VI.name(), VI)).second)
656 dbgs() <<
"[Workload] Function name " << Name
657 <<
" present in the workload definition is ambiguous. Consider " 658"compiling with -funique-internal-linkage-names.";
663if (std::error_code EC = BufferOrErr.getError()) {
667auto Buffer = std::move(BufferOrErr.get());
668 std::map<std::string, std::vector<std::string>> WorkloadDefs;
670// The JSON is supposed to contain a dictionary matching the type of 671// WorkloadDefs. For example: 673// "rootFunction_1": ["function_to_import_1", "function_to_import_2"], 674// "rootFunction_2": ["function_to_import_3", "function_to_import_4"] 681for (
constauto &Workload : WorkloadDefs) {
682constauto &Root = Workload.first;
683 DbgReportIfAmbiguous(Root);
685constauto &AllCallees = Workload.second;
686auto RootIt = NameToValueInfo.
find(Root);
687if (RootIt == NameToValueInfo.
end()) {
689 <<
" not found in this linkage unit.\n");
692auto RootVI = RootIt->second;
693if (RootVI.getSummaryList().size() != 1) {
695 <<
" should have exactly one summary, but has " 696 << RootVI.getSummaryList().size() <<
". Skipping.\n");
700 RootVI.getSummaryList().
front()->modulePath();
702 <<
" is : " << RootDefiningModule <<
"\n");
703auto &Set = Workloads[RootDefiningModule];
704for (
constauto &Callee : AllCallees) {
706 DbgReportIfAmbiguous(Callee);
707auto ElemIt = NameToValueInfo.
find(Callee);
708if (ElemIt == NameToValueInfo.
end()) {
712 Set.insert(ElemIt->second);
717void loadFromCtxProf() {
720if (std::error_code EC = BufferOrErr.getError()) {
724auto Buffer = std::move(BufferOrErr.get());
732constauto &CtxMap = *Ctx;
734for (
constauto &[RootGuid, Root] : CtxMap) {
735// Avoid ContainedGUIDs to get in/out of scope. Reuse its memory for 736// subsequent roots, but clear its contents. 737 ContainedGUIDs.
clear();
739auto RootVI =
Index.getValueInfo(RootGuid);
742 <<
" not found in this linkage unit.\n");
745if (RootVI.getSummaryList().size() != 1) {
747 <<
" should have exactly one summary, but has " 748 << RootVI.getSummaryList().size() <<
". Skipping.\n");
752 RootVI.getSummaryList().
front()->modulePath();
753LLVM_DEBUG(
dbgs() <<
"[Workload] Root defining module for " << RootGuid
754 <<
" is : " << RootDefiningModule <<
"\n");
755auto &Set = Workloads[RootDefiningModule];
756 Root.getContainedGuids(ContainedGUIDs);
757for (
autoGuid : ContainedGUIDs)
772"Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");
780for (
constauto &[Root, Set] : Workloads) {
781dbgs() <<
"[Workload] Root: " << Root <<
" we have " << Set.size()
782 <<
" distinct callees.\n";
783for (
constauto &VI : Set) {
784dbgs() <<
"[Workload] Root: " << Root
785 <<
" Would include: " << VI.getGUID() <<
"\n";
798LLVM_DEBUG(
dbgs() <<
"[Workload] Using the regular imports manager.\n");
799return std::unique_ptr<ModuleImportsManager>(
802LLVM_DEBUG(
dbgs() <<
"[Workload] Using the contextual imports manager.\n");
803return std::make_unique<WorkloadImportsManager>(IsPrevailing, Index,
819return"InterposableLinkage";
821return"LocalLinkageNotInModule";
830/// Compute the list of functions to import for a given caller. Mark these 831/// imported functions and the symbols they reference in their source module as 832/// exported from their source module. 843staticint ImportCount = 0;
844for (
constauto &Edge : Summary.calls()) {
855if (DefinedGVSummaries.
count(VI.getGUID())) {
856// FIXME: Consider not skipping import if the module contains 857// a non-prevailing def with interposable linkage. The prevailing copy 858// can safely be imported (see shouldImportGlobal()). 859LLVM_DEBUG(
dbgs() <<
"ignored! Target already in destination module.\n");
873constauto NewThreshold =
874 Threshold * GetBonusMultiplier(Edge.second.getHotness());
876autoIT = ImportThresholds.
insert(std::make_pair(
877 VI.getGUID(), std::make_tuple(NewThreshold,
nullptr,
nullptr)));
878bool PreviouslyVisited = !
IT.second;
879auto &ProcessedThreshold = std::get<0>(
IT.first->second);
880auto &CalleeSummary = std::get<1>(
IT.first->second);
881auto &FailureInfo = std::get<2>(
IT.first->second);
885bool IsCriticalCallsite =
891// Since the traversal of the call graph is DFS, we can revisit a function 892// a second time with a higher threshold. In this case, it is added back 893// to the worklist with the new threshold (so that its own callee chains 894// can be considered with the higher threshold). 895if (NewThreshold <= ProcessedThreshold) {
897dbgs() <<
"ignored! Target was already imported with Threshold " 898 << ProcessedThreshold <<
"\n");
901// Update with new larger threshold. 902 ProcessedThreshold = NewThreshold;
903 ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
905// If we already rejected importing a callee at the same or higher 906// threshold, don't waste time calling selectCallee. 907if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
909dbgs() <<
"ignored! Target was already rejected with Threshold " 910 << ProcessedThreshold <<
"\n");
913"Expected FailureInfo for previously rejected candidate");
914 FailureInfo->Attempts++;
921// `SummaryForDeclImport` is an summary eligible for declaration import. 925 Summary.modulePath(), SummaryForDeclImport, Reason);
927// There isn't a callee for definition import but one for declaration 932// Note `ExportLists` only keeps track of exports due to imported 936// Update with new larger threshold if this was a retry (otherwise 937// we would have already inserted with NewThreshold above). Also 938// update failure info if requested. 939if (PreviouslyVisited) {
940 ProcessedThreshold = NewThreshold;
943"Expected FailureInfo for previously rejected candidate");
944 FailureInfo->Reason = Reason;
945 FailureInfo->Attempts++;
946 FailureInfo->MaxHotness =
947 std::max(FailureInfo->MaxHotness, Edge.second.getHotness());
951"Expected no FailureInfo for newly rejected candidate");
952 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
953 VI, Edge.second.getHotness(), Reason, 1);
956 std::string Msg = std::string(
"Failed to import function ") +
957 VI.name().str() +
" due to " +
959autoError = make_error<StringError>(
962"Error importing module: ");
966 <<
"ignored! No qualifying callee with summary found.\n");
971// "Resolve" the summary 972 CalleeSummary = CalleeSummary->getBaseObject();
973 ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
976 (ResolvedCalleeSummary->
instCount() <= NewThreshold)) &&
977"selectCallee() didn't honor the threshold");
979auto ExportModulePath = ResolvedCalleeSummary->
modulePath();
981// Try emplace the definition entry, and update stats based on insertion 983if (ImportList.
addDefinition(ExportModulePath, VI.getGUID()) !=
985 NumImportedFunctionsThinLink++;
987 NumImportedHotFunctionsThinLink++;
988if (IsCriticalCallsite)
989 NumImportedCriticalFunctionsThinLink++;
992// Any calls/references made by this function will be marked exported 993// later, in ComputeCrossModuleImport, after import decisions are 994// complete, which is more efficient than adding them here. 996 (*ExportLists)[ExportModulePath].insert(VI);
999auto GetAdjustedThreshold = [](
unsigned Threshold,
bool IsHotCallsite) {
1000// Adjust the threshold for next level of imported functions. 1001// The threshold is different for hot callsites because we can then 1002// inline chains of hot calls. 1008constauto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
1012// Insert the newly imported function to the worklist. 1013 Worklist.
emplace_back(ResolvedCalleeSummary, AdjThreshold);
1020// Worklist contains the list of function imported in this module, for which 1021// we will analyse the callees and may import further down the callgraph. 1023GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
1027// Populate the worklist with the import for the functions in the current 1029for (
constauto &GVSummary : DefinedGVSummaries) {
1031// FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID 1032// so this map look up (and possibly others) can be avoided. 1033auto VI = Index.getValueInfo(GVSummary.first);
1035if (!Index.isGlobalValueLive(GVSummary.second)) {
1040 dyn_cast<FunctionSummary>(GVSummary.second->getBaseObject());
1042// Skip import for global variables 1046 DefinedGVSummaries, IsPrevailing, Worklist, GVI,
1047 ImportList, ExportLists, ImportThresholds);
1050// Process the newly imported functions and add callees to the worklist. 1051while (!Worklist.
empty()) {
1053auto *Summary = std::get<0>(GVInfo);
1054auto Threshold = std::get<1>(GVInfo);
1056if (
auto *FS = dyn_cast<FunctionSummary>(Summary))
1058 IsPrevailing, Worklist, GVI, ImportList,
1059 ExportLists, ImportThresholds);
1062// Print stats about functions considered but rejected for importing 1065dbgs() <<
"Missed imports into module " << ModName <<
"\n";
1066for (
auto &
I : ImportThresholds) {
1067auto &ProcessedThreshold = std::get<0>(
I.second);
1068auto &CalleeSummary = std::get<1>(
I.second);
1069auto &FailureInfo = std::get<2>(
I.second);
1071continue;
// We are going to import. 1074if (!FailureInfo->VI.getSummaryList().empty())
1075 FS = dyn_cast<FunctionSummary>(
1076 FailureInfo->VI.getSummaryList()[0]->getBaseObject());
1077dbgs() << FailureInfo->VI
1079 <<
", Threshold = " << ProcessedThreshold
1080 <<
", Size = " << (FS ? (int)FS->instCount() : -1)
1082 <<
", Attempts = " << FailureInfo->Attempts <<
"\n";
1089auto SL = VI.getSummaryList();
1097if (
constauto &VI = Index.getValueInfo(
G))
1102// Return the number of global variable summaries in ExportSet. 1107for (
auto &VI : ExportSet)
1115unsigned DefinedFS = 0;
1119// Compute import statistics for each source module in ImportList. 1125for (
constauto &[FromModule, GUID,
Type] : ImportList) {
1144for (
constauto &ImportPerModule : ImportLists)
1145for (
constauto &[FromModule, GUID, ImportType] : ImportPerModule.second)
1146 FlattenedImports.
insert(GUID);
1148// Checks that all GUIDs of read/writeonly vars we see in export lists 1149// are also in the import lists. Otherwise we my face linker undefs, 1150// because readonly and writeonly vars are internalized in their 1151// source modules. The exception would be if it has a linkage type indicating 1152// that there may have been a copy existing in the importing module (e.g. 1153// linkonce_odr). In that case we cannot accurately do this checking. 1154auto IsReadOrWriteOnlyVarNeedingImporting = [&](
StringRef ModulePath,
1156auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
1157 Index.findSummaryInModule(VI, ModulePath));
1158return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&
1164for (
auto &ExportPerModule : ExportLists)
1165for (
auto &VI : ExportPerModule.second)
1166if (!FlattenedImports.
count(VI.getGUID()) &&
1167 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
1174/// Compute all the import and export for every module using the Index. 1183// For each module that has function defined, compute the import/export lists. 1184for (
constauto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
1185auto &ImportList = ImportLists[DefinedGVSummaries.first];
1187 << DefinedGVSummaries.first <<
"'\n");
1188 MIS->computeImportForModule(DefinedGVSummaries.second,
1189 DefinedGVSummaries.first, ImportList);
1192// When computing imports we only added the variables and functions being 1193// imported to the export list. We also need to mark any references and calls 1194// they make as exported as well. We do this here, as it is more efficient 1195// since we may import the same values multiple times into different modules 1196// during the import computation. 1197for (
auto &ELI : ExportLists) {
1198// `NewExports` tracks the VI that gets exported because the full definition 1199// of its user/referencer gets exported. 1201constauto &DefinedGVSummaries =
1202 ModuleToDefinedGVSummaries.
lookup(ELI.first);
1203for (
auto &EI : ELI.second) {
1204// Find the copy defined in the exporting module so that we can mark the 1205// values it references in that specific definition as exported. 1206// Below we will add all references and called values, without regard to 1207// whether they are also defined in this module. We subsequently prune the 1208// list to only include those defined in the exporting module, see comment 1210auto DS = DefinedGVSummaries.
find(EI.getGUID());
1211// Anything marked exported during the import computation must have been 1212// defined in the exporting module. 1214auto *S = DS->getSecond();
1215 S = S->getBaseObject();
1216if (
auto *GVS = dyn_cast<GlobalVarSummary>(S)) {
1217// Export referenced functions and variables. We don't export/promote 1218// objects referenced by writeonly variable initializer, because 1219// we convert such variables initializers to "zeroinitializer". 1220// See processGlobalForThinLTO. 1221if (!Index.isWriteOnly(GVS))
1222for (
constauto &VI : GVS->refs())
1225auto *FS = cast<FunctionSummary>(S);
1226for (
constauto &Edge : FS->calls())
1227 NewExports.
insert(Edge.first);
1228for (
constauto &
Ref : FS->refs())
1232// Prune list computed above to only include values defined in the 1233// exporting module. We do this after the above insertion since we may hit 1234// the same ref/call target multiple times in above loop, and it is more 1235// efficient to avoid a set lookup each time. 1236for (
auto EI = NewExports.
begin(); EI != NewExports.
end();) {
1237if (!DefinedGVSummaries.
count(EI->getGUID()))
1238 NewExports.
erase(EI++);
1242 ELI.second.insert(NewExports.
begin(), NewExports.
end());
1249for (
constauto &ModuleImports : ImportLists) {
1250auto ModName = ModuleImports.first;
1251auto &Exports = ExportLists[ModName];
1256 << Exports.
size() - NumGVS <<
" functions and " << NumGVS
1257 <<
" vars. Imports from " << Histogram.
size()
1259for (
constauto &[SrcModName,
Stats] : Histogram) {
1261 <<
" function definitions and " 1263 <<
" function declarations imported from " << SrcModName
1266 <<
" global vars imported from " << SrcModName <<
"\n");
1279 << Histogram.
size() <<
" modules.\n");
1280for (
constauto &[SrcModName,
Stats] : Histogram) {
1282 <<
" function definitions and " 1284 <<
" function declarations imported from " << SrcModName
1287 << SrcModName <<
"\n");
1292/// Compute all the imports for the given module using the Index. 1294/// \p isPrevailing is a callback that will be called with a global value's GUID 1295/// and summary and should return whether the module corresponding to the 1296/// summary contains the linker-prevailing copy of that value. 1298/// \p ImportList will be populated with a map that can be passed to 1299/// FunctionImporter::importFunctions() above (see description there). 1306// Collect the list of functions this module defines. 1309 Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap);
1311// Compute the import list for this module. 1312LLVM_DEBUG(
dbgs() <<
"Computing import for Module '" << ModulePath <<
"'\n");
1314 MIS->computeImportForModule(FunctionSummaryMap, ModulePath, ImportList);
1321/// Mark all external summaries in \p Index for import into the given module. 1322/// Used for testing the case of distributed builds using a distributed index. 1324/// \p ImportList will be populated with a map that can be passed to 1325/// FunctionImporter::importFunctions() above (see description there). 1329for (
constauto &GlobalList : Index) {
1330// Ignore entries for undefined references. 1331if (GlobalList.second.SummaryList.empty())
1334auto GUID = GlobalList.first;
1335assert(GlobalList.second.SummaryList.size() == 1 &&
1336"Expected individual combined index to have one summary per GUID");
1337auto &Summary = GlobalList.second.SummaryList[0];
1338// Skip the summaries for the importing module. These are included to 1339// e.g. record required linkage changes. 1340if (Summary->modulePath() == ModulePath)
1342// Add an entry to provoke importing by thinBackend. 1343 ImportList.
addGUID(Summary->modulePath(), GUID, Summary->importType());
1350// For SamplePGO, the indirect call targets for local functions will 1351// have its original name annotated in profile. We try to find the 1352// corresponding PGOFuncName as the GUID, and fix up the edges 1356for (
auto &EI : FS->mutableCalls()) {
1357if (!EI.first.getSummaryList().empty())
1359auto GUID = Index.getGUIDFromOriginalID(EI.first.getGUID());
1362// Update the edge to point directly to the correct GUID. 1363auto VI = Index.getValueInfo(GUID);
1365 VI.getSummaryList(),
1366 [&](
const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
1367// The mapping from OriginalId to GUID may return a GUID 1368// that corresponds to a static variable. Filter it out here. 1369// This can happen when 1370// 1) There is a call to a library function which is not defined 1372// 2) There is a static variable with the OriginalGUID identical 1373// to the GUID of the library function in 1); 1374// When this happens the static variable in 2) will be found, 1375// which needs to be filtered out. 1376 return SummaryPtr->getSummaryKind() ==
1377 GlobalValueSummary::GlobalVarKind;
1385for (
constauto &Entry : Index) {
1386for (
constauto &S : Entry.second.SummaryList) {
1387if (
auto *FS = dyn_cast<FunctionSummary>(S.get()))
1397assert(!Index.withGlobalValueDeadStripping());
1399// Don't do anything when nothing is live, this is friendly with tests. 1400 GUIDPreservedSymbols.
empty()) {
1401// Still need to update indirect calls. 1405unsigned LiveSymbols = 0;
1407 Worklist.
reserve(GUIDPreservedSymbols.
size() * 2);
1408for (
auto GUID : GUIDPreservedSymbols) {
1412for (
constauto &S : VI.getSummaryList())
1416// Add values flagged in the index as live roots to the worklist. 1417for (
constauto &Entry : Index) {
1418auto VI = Index.getValueInfo(Entry);
1419for (
constauto &S : Entry.second.SummaryList) {
1420if (
auto *FS = dyn_cast<FunctionSummary>(S.get()))
1431// Make value live and add it to the worklist if it was not live before. 1433// FIXME: If we knew which edges were created for indirect call profiles, 1434// we could skip them here. Any that are live should be reached via 1435// other edges, e.g. reference edges. Otherwise, using a profile collected 1436// on a slightly different binary might provoke preserving, importing 1437// and ultimately promoting calls to functions not linked into this 1438// binary, which increases the binary size unnecessarily. Note that 1439// if this code changes, the importer needs to change so that edges 1440// to functions marked dead are skipped. 1443 [](
const std::unique_ptr<llvm::GlobalValueSummary> &S) {
1448// We only keep live symbols that are known to be non-prevailing if any are 1449// available_externally, linkonceodr, weakodr. Those symbols are discarded 1450// later in the EliminateAvailableExternally pass and setting them to 1451// not-live could break downstreams users of liveness information (PR36483) 1452// or limit optimization opportunities. 1454bool KeepAliveLinkage =
false;
1455bool Interposable =
false;
1456for (
constauto &S : VI.getSummaryList()) {
1460 KeepAliveLinkage =
true;
1466if (!KeepAliveLinkage)
1471"Interposable and available_externally/linkonce_odr/weak_odr " 1476for (
constauto &S : VI.getSummaryList())
1482while (!Worklist.
empty()) {
1484for (
constauto &Summary : VI.getSummaryList()) {
1485if (
auto *AS = dyn_cast<AliasSummary>(Summary.get())) {
1486// If this is an alias, visit the aliasee VI to ensure that all copies 1487// are marked live and it is added to the worklist for further 1488// processing of its references. 1489visit(AS->getAliaseeVI(),
true);
1492for (
autoRef : Summary->refs())
1494if (
auto *FS = dyn_cast<FunctionSummary>(Summary.get()))
1495for (
auto Call : FS->calls())
1496visit(Call.first,
false);
1499 Index.setWithGlobalValueDeadStripping();
1501unsigned DeadSymbols = Index.size() - LiveSymbols;
1502LLVM_DEBUG(
dbgs() << LiveSymbols <<
" symbols Live, and " << DeadSymbols
1503 <<
" symbols Dead \n");
1504 NumDeadSymbols += DeadSymbols;
1505 NumLiveSymbols += LiveSymbols;
1508// Compute dead symbols and propagate constants in combined index. 1513bool ImportEnabled) {
1517 Index.propagateAttributes(GUIDPreservedSymbols);
1520/// Compute the set of summaries needed for a ThinLTO backend compilation of 1528// Include all summaries from the importing module. 1529 ModuleToSummariesForIndex[std::string(ModulePath)] =
1530 ModuleToDefinedGVSummaries.
lookup(ModulePath);
1532// Forward port the heterogeneous std::map::operator[]() from C++26, which 1533// lets us look up the map without allocating an instance of std::string when 1534// the key-value pair exists in the map. 1535// TODO: Remove this in favor of the heterogenous std::map::operator[]() from 1536// C++26 when it becomes available for our codebase. 1539auto It = Map.find(Key);
1541 std::tie(It, std::ignore) =
1546// Include summaries for imports. 1547for (
constauto &[FromModule, GUID, ImportType] : ImportList) {
1548auto &SummariesForIndex =
1549 LookupOrCreate(ModuleToSummariesForIndex, FromModule);
1551constauto &DefinedGVSummaries = ModuleToDefinedGVSummaries.
at(FromModule);
1552constauto &DS = DefinedGVSummaries.
find(GUID);
1554"Expected a defined summary for imported global value");
1556 DecSummaries.insert(DS->second);
1558 SummariesForIndex[GUID] = DS->second;
1562/// Emit the files \p ModulePath will import from into \p OutputFilename. 1571for (
constauto &ILI : ModuleToSummariesForIndex)
1572// The ModuleToSummariesForIndex map includes an entry for the current 1573// Module (needed for writing out the index files). We don't want to 1574// include it in the imports file, however, so filter it out. 1575if (ILI.first != ModulePath)
1576 ImportsOS << ILI.first <<
"\n";
1583if (
Function *
F = dyn_cast<Function>(&GV)) {
1586F->setComdat(
nullptr);
1588 V->setInitializer(
nullptr);
1591 V->setComdat(
nullptr);
1619auto FinalizeInModule = [&](
GlobalValue &GV,
bool Propagate =
false) {
1620// See if the global summary analysis computed a new resolved linkage. 1622if (GS == DefinedGlobals.
end())
1627if (
Function *
F = dyn_cast<Function>(&GV)) {
1628// TODO: propagate ReadNone and ReadOnly. 1629if (FS->fflags().ReadNone && !
F->doesNotAccessMemory())
1630F->setDoesNotAccessMemory();
1632if (FS->fflags().ReadOnly && !
F->onlyReadsMemory())
1633F->setOnlyReadsMemory();
1635if (FS->fflags().NoRecurse && !
F->doesNotRecurse())
1636F->setDoesNotRecurse();
1638if (FS->fflags().NoUnwind && !
F->doesNotThrow())
1639F->setDoesNotThrow();
1643auto NewLinkage = GS->second->linkage();
1645// Don't internalize anything here, because the code below 1646// lacks necessary correctness checks. Leave this job to 1647// LLVM 'internalize' pass. 1649// In case it was dead and already converted to declaration. 1653// Set the potentially more constraining visibility computed from summaries. 1654// The DefaultVisibility condition is because older GlobalValueSummary does 1655// not record DefaultVisibility and we don't want to change protected/hidden 1663// Check for a non-prevailing def that has interposable linkage 1664// (e.g. non-odr weak or linkonce). In that case we can't simply 1665// convert to available_externally, since it would lose the 1666// interposable property and possibly get inlined. Simply drop 1667// the definition in that case. 1671// FIXME: Change this to collect replaced GVs and later erase 1672// them from the parent module once thinLTOResolvePrevailingGUID is 1673// changed to enable this for aliases. 1676// If all copies of the original symbol had global unnamed addr and 1677// linkonce_odr linkage, or if all of them had local unnamed addr linkage 1678// and are constants, then it should be an auto hide symbol. In that case 1679// the thin link would have marked it as CanAutoHide. Add hidden 1680// visibility to the symbol to preserve the property. 1682 GS->second->canAutoHide()) {
1688 <<
"` from " << GV.
getLinkage() <<
" to " << NewLinkage
1692// Remove declarations from comdats, including available_externally 1693// as this is a declaration for the linker, and will be dropped eventually. 1694// It is illegal for comdats to contain declarations. 1695auto *GO = dyn_cast_or_null<GlobalObject>(&GV);
1696if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1697if (GO->getComdat()->getName() == GO->getName())
1698 NonPrevailingComdats.
insert(GO->getComdat());
1699 GO->setComdat(
nullptr);
1703// Process functions and global now 1704for (
auto &GV : TheModule)
1706for (
auto &GV : TheModule.
globals())
1707 FinalizeInModule(GV);
1708for (
auto &GV : TheModule.
aliases())
1709 FinalizeInModule(GV);
1711// For a non-prevailing comdat, all its members must be available_externally. 1712// FinalizeInModule has handled non-local-linkage GlobalValues. Here we handle 1713// local linkage GlobalValues. 1714if (NonPrevailingComdats.
empty())
1717if (
auto *
C = GO.getComdat();
C && NonPrevailingComdats.
count(
C)) {
1718 GO.setComdat(
nullptr);
1725// If an alias references a GlobalValue in a non-prevailing comdat, change 1726// it to available_externally. For simplicity we only handle GlobalValue and 1727// ConstantExpr with a base object. ConstantExpr without a base object is 1728// unlikely used in a COMDAT. 1729for (
auto &GA : TheModule.
aliases()) {
1730if (GA.hasAvailableExternallyLinkage())
1733assert(Obj &&
"aliasee without an base object is unimplemented");
1742/// Run internalization on \p TheModule based on symmary analysis. 1745// Declare a callback for the internalize pass that will ask for every 1746// candidate GlobalValue if it can be internalized or not. 1747auto MustPreserveGV = [&](
constGlobalValue &GV) ->
bool {
1748// It may be the case that GV is on a chain of an ifunc, its alias and 1749// subsequent aliases. In this case, the summary for the value is not 1751if (isa<GlobalIFunc>(&GV) ||
1752 (isa<GlobalAlias>(&GV) &&
1753 isa<GlobalIFunc>(cast<GlobalAlias>(&GV)->getAliaseeObject())))
1756// Lookup the linkage recorded in the summaries during global analysis. 1757auto GS = DefinedGlobals.
find(GV.getGUID());
1758if (GS == DefinedGlobals.
end()) {
1759// Must have been promoted (possibly conservatively). Find original 1760// name so that we can access the correct summary and see if it can 1761// be internalized again. 1762// FIXME: Eventually we should control promotion instead of promoting 1763// and internalizing again. 1770if (GS == DefinedGlobals.
end()) {
1771// Also check the original non-promoted non-globalized name. In some 1772// cases a preempted weak value is linked in as a local copy because 1773// it is referenced by an alias (IRLinker::linkGlobalValueProto). 1774// In that case, since it was originally not a local value, it was 1775// recorded in the index using the original name. 1776// FIXME: This may not be needed once PR27866 is fixed. 1784// FIXME: See if we can just internalize directly here via linkage changes 1785// based on the index, rather than invoking internalizeModule. 1789/// Make alias a clone of its aliasee. 1795// Clone should use the original alias's linkage, visibility and name, and we 1796// ensure all uses of alias instead use the new clone (casted if necessary). 1804// Internalize values that we marked with specific attribute 1805// in processGlobalForThinLTO. 1807for (
auto &GV : M.globals())
1808// Skip GVs which have been converted to declarations 1809// by dropDeadSymbols. 1810if (!GV.isDeclaration() && GV.hasAttribute(
"thinlto-internalize")) {
1816// Automatically import functions in Module \p DestModule based on the summaries 1822unsigned ImportedCount = 0, ImportedGVCount = 0;
1826// Do the actual import of functions now, one Module at a time 1828// Get the module for the import 1832 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1836// If modules were created with lazy metadata loading, materialize it 1837// now, before linking it (otherwise this will be a noop). 1838if (
Error Err = SrcModule->materializeMetadata())
1839return std::move(Err);
1841// Find the globals to import 1846auto GUID =
F.getGUID();
1847auto MaybeImportType = ImportList.
getImportType(ModName, GUID);
1851 <<
" importing function" 1852 << (ImportDefinition
1854 : (MaybeImportType ?
" declaration " :
" "))
1855 << GUID <<
" " <<
F.getName() <<
" from " 1856 << SrcModule->getSourceFileName() <<
"\n");
1857if (ImportDefinition) {
1858if (
Error Err =
F.materialize())
1859return std::move(Err);
1860// MemProf should match function's definition and summary, 1861// 'thinlto_src_module' is needed. 1863// Add 'thinlto_src_module' and 'thinlto_src_file' metadata for 1864// statistics and debugging. 1866"thinlto_src_module",
1868 {MDString::get(DestModule.getContext(),
1869 SrcModule->getModuleIdentifier())}));
1873 {MDString::get(DestModule.getContext(),
1874 SrcModule->getSourceFileName())}));
1882auto GUID = GV.getGUID();
1883auto MaybeImportType = ImportList.
getImportType(ModName, GUID);
1887 <<
" importing global" 1888 << (ImportDefinition
1890 : (MaybeImportType ?
" declaration " :
" "))
1891 << GUID <<
" " << GV.getName() <<
" from " 1892 << SrcModule->getSourceFileName() <<
"\n");
1893if (ImportDefinition) {
1894if (
Error Err = GV.materialize())
1895return std::move(Err);
1896 ImportedGVCount += GlobalsToImport.
insert(&GV);
1900if (!GA.hasName() || isa<GlobalIFunc>(GA.getAliaseeObject()))
1902auto GUID = GA.getGUID();
1903auto MaybeImportType = ImportList.
getImportType(ModName, GUID);
1907 <<
" importing alias" 1908 << (ImportDefinition
1910 : (MaybeImportType ?
" declaration " :
" "))
1911 << GUID <<
" " << GA.getName() <<
" from " 1912 << SrcModule->getSourceFileName() <<
"\n");
1913if (ImportDefinition) {
1914if (
Error Err = GA.materialize())
1915return std::move(Err);
1916// Import alias as a copy of its aliasee. 1919return std::move(Err);
1923 << SrcModule->getSourceFileName() <<
"\n");
1925// Add 'thinlto_src_module' and 'thinlto_src_file' metadata for 1926// statistics and debugging. 1928"thinlto_src_module",
1930 {MDString::get(DestModule.getContext(),
1931 SrcModule->getModuleIdentifier())}));
1935 {MDString::get(DestModule.getContext(),
1936 SrcModule->getSourceFileName())}));
1938 GlobalsToImport.
insert(Fn);
1942// Upgrade debug info after we're done materializing all the globals and we 1943// have loaded all the required metadata! 1946// Set the partial sample profile ratio in the profile summary module flag 1947// of the imported source module, if applicable, so that the profile summary 1948// module flag will match with that of the destination module when it's 1950 SrcModule->setPartialSampleProfileRatio(Index);
1952// Link in the specified functions. 1957for (
constauto *GV : GlobalsToImport)
1959 <<
" from " << SrcModule->getSourceFileName() <<
"\n";
1962if (
Error Err = Mover.
move(std::move(SrcModule),
1964/*IsPerformingImport=*/true))
1966Twine(
"Function Import: link error: ") +
1969 ImportedCount += GlobalsToImport.
size();
1970 NumImportedModules++;
1975 NumImportedFunctions += (ImportedCount - ImportedGVCount);
1976 NumImportedGlobalVars += ImportedGVCount;
1978// TODO: Print counters for definitions and declarations in the debugging log. 1979LLVM_DEBUG(
dbgs() <<
"Imported " << ImportedCount - ImportedGVCount
1980 <<
" functions for Module " 1983 <<
" global variables for Module " 1985return ImportedCount;
1995if (!IndexPtrOrErr) {
2000 std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
2002// First step is collecting the import list. 2005// If requested, simply import all functions in the index. This is used 2006// when testing distributed backend handling via the opt tool, when 2007// we have distributed indexes containing exactly the summaries to import. 2010 *Index, ImportList);
2013 isPrevailing, *Index, ImportList);
2015// Conservatively mark all internal values as promoted. This interface is 2016// only used when doing importing via the function importing pass. The pass 2017// is only enabled when testing importing via the 'opt' tool, which does 2018// not do the ThinLink that would normally determine what values to promote. 2019for (
auto &
I : *Index) {
2020for (
auto &S :
I.second.SummaryList) {
2026// Next we need to promote to global scope and rename any local values that 2027// are potentially exported to other modules. 2029/*GlobalsToImport=*/nullptr);
2031// Perform the import now. 2032auto ModuleLoader = [&M](
StringRef Identifier) {
2033returnloadFile(std::string(Identifier), M.getContext());
2036/*ClearDSOLocalOnDeclarations=*/false);
2039// FIXME: Probably need to propagate Errors through the pass manager. 2042"Error importing module: ");
2051// This is only used for testing the function import pass via opt, where we 2052// don't have prevailing information from the LTO context available, so just 2053// conservatively assume everything is prevailing (which is fine for the very 2054// limited use of prevailing checking in this pass). static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
cl::opt< std::string > UseCtxProfile("use-ctx-profile", cl::init(""), cl::Hidden, cl::desc("Use the specified contextual profile file"))
static auto qualifyCalleeCandidates(const ModuleSummaryIndex &Index, ArrayRef< std::unique_ptr< GlobalValueSummary > > CalleeSummaryList, StringRef CallerModulePath)
Given a list of possible callee implementation for a call site, qualify the legality of importing eac...
static cl::opt< bool > EnableImportMetadata("enable-import-metadata", cl::init(false), cl::Hidden, cl::desc("Enable import metadata like 'thinlto_src_module' and " "'thinlto_src_file'"))
static cl::opt< float > ImportColdMultiplier("import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"), cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"))
static DenseMap< StringRef, ImportStatistics > collectImportStatistics(const ModuleSummaryIndex &Index, const FunctionImporter::ImportMapTy &ImportList)
static cl::opt< float > ImportHotMultiplier("import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"))
static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index, FunctionImporter::ExportSetTy &ExportSet)
static bool checkVariableImport(const ModuleSummaryIndex &Index, FunctionImporter::ImportListsTy &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)
static bool doImportingForModuleForTest(Module &M, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
static cl::opt< float > ImportHotInstrFactor("import-hot-evolution-factor", cl::init(1.0), cl::Hidden, cl::value_desc("x"), cl::desc("As we import functions called from hot callsite, multiply the " "`import-instr-limit` threshold by this factor " "before processing newly imported functions"))
static void computeImportForFunction(const FunctionSummary &Summary, const ModuleSummaryIndex &Index, const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, SmallVectorImpl< EdgeInfo > &Worklist, GlobalsImporter &GVImporter, FunctionImporter::ImportMapTy &ImportList, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists, FunctionImporter::ImportThresholdsTy &ImportThresholds)
Compute the list of functions to import for a given caller.
static cl::opt< float > ImportCriticalMultiplier("import-critical-multiplier", cl::init(100.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for critical callsites"))
static cl::opt< int > ImportCutoff("import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"), cl::desc("Only import first N functions if N>=0 (default -1)"))
cl::opt< std::string > UseCtxProfile
static const char * getFailureName(FunctionImporter::ImportFailureReason Reason)
static cl::opt< float > ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7), cl::Hidden, cl::value_desc("x"), cl::desc("As we import functions, multiply the " "`import-instr-limit` threshold by this factor " "before processing newly imported functions"))
static void internalizeGVsAfterImport(Module &M)
static cl::opt< bool > PrintImports("print-imports", cl::init(false), cl::Hidden, cl::desc("Print imported functions"))
void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index, FunctionSummary *FS)
static std::unique_ptr< Module > loadFile(const std::string &FileName, LLVMContext &Context)
static cl::opt< bool > ForceImportAll("force-import-all", cl::init(false), cl::Hidden, cl::desc("Import functions with noinline attribute"))
static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary, size_t NumDefs, StringRef ImporterModule)
static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI)
static cl::opt< bool > ImportDeclaration("import-declaration", cl::init(false), cl::Hidden, cl::desc("If true, import function declaration as fallback if the function " "definition is not imported."))
This is a test-only option.
static cl::opt< unsigned > ImportInstrLimit("import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"), cl::desc("Only import functions with less than N instructions"))
Limit on instruction count of imported functions.
static cl::opt< std::string > WorkloadDefinitions("thinlto-workload-def", cl::desc("Pass a workload definition. This is a file containing a JSON " "dictionary. The keys are root functions, the values are lists of " "functions to import in the module defining the root. It is " "assumed -funique-internal-linkage-names was used, to ensure " "local linkage functions have unique names. For example: \n" "{\n" " \"rootFunction_1\": [\"function_to_import_1\", " "\"function_to_import_2\"], \n" " \"rootFunction_2\": [\"function_to_import_3\", " "\"function_to_import_4\"] \n" "}"), cl::Hidden)
Pass a workload description file - an example of workload would be the functions executed to satisfy ...
static cl::opt< bool > ComputeDead("compute-dead", cl::init(true), cl::Hidden, cl::desc("Compute dead symbols"))
static cl::opt< std::string > SummaryFile("summary-file", cl::desc("The summary file to use for function importing."))
Summary file to use for function importing when using -function-import from the command line.
static cl::opt< bool > ImportAllIndex("import-all-index", cl::desc("Import all external functions in index."))
Used when testing importing from distributed indexes via opt.
static cl::opt< bool > PrintImportFailures("print-import-failures", cl::init(false), cl::Hidden, cl::desc("Print information for functions rejected for importing"))
static Function * replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA)
Make alias a clone of its aliasee.
static void dumpImportListForModule(const ModuleSummaryIndex &Index, StringRef ModulePath, FunctionImporter::ImportMapTy &ImportList)
static void ComputeCrossModuleImportForModuleFromIndexForTest(StringRef ModulePath, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList)
Mark all external summaries in Index for import into the given module.
static const GlobalValueSummary * selectCallee(const ModuleSummaryIndex &Index, ArrayRef< std::unique_ptr< GlobalValueSummary > > CalleeSummaryList, unsigned Threshold, StringRef CallerModulePath, const GlobalValueSummary *&TooLargeOrNoInlineSummary, FunctionImporter::ImportFailureReason &Reason)
Given a list of possible callee implementation for a call site, select one that fits the Threshold fo...
static void ComputeCrossModuleImportForModuleForTest(StringRef ModulePath, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList)
Compute all the imports for the given module using the Index.
Module.h This file contains the declarations for the Module class.
This file supports working with JSON data.
block placement Basic Block Placement Stats
static cl::opt< std::string > OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"), cl::init("-"))
This file contains the declarations for metadata subclasses.
static cl::opt< bool > PropagateAttrs("propagate-attrs", cl::init(true), cl::Hidden, cl::desc("Propagate attributes in index"))
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
Reader for contextual iFDO profile, which comes in bitstream format.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void visit(MachineFunction &MF, MachineBasicBlock &Start, std::function< void(MachineBasicBlock *)> op)
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Import globals referenced by a function or other globals that are being imported, if importing such g...
void onImportingSummary(const GlobalValueSummary &Summary)
GlobalsImporter(const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, FunctionImporter::ImportMapTy &ImportList, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists)
Determine the list of imports and exports for each module.
DenseMap< StringRef, FunctionImporter::ExportSetTy > *const ExportLists
virtual ~ModuleImportsManager()=default
static std::unique_ptr< ModuleImportsManager > create(function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists=nullptr)
function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing
ModuleImportsManager(function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists=nullptr)
const ModuleSummaryIndex & Index
virtual void computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName, FunctionImporter::ImportMapTy &ImportList)
Given the list of globals defined in a module, compute the list of imports as well as the list of "ex...
A ModuleImportsManager that operates based on a workload definition (see -thinlto-workload-def).
WorkloadImportsManager(function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists)
A container for analyses that lazily runs them and caches their results.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
const ValueT & at(const_arg_type_t< KeyT > Val) const
at - Return the entry for the specified key, or abort if no such entry exists.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
std::pair< ImportIDTy, ImportIDTy > createImportIDs(StringRef FromModule, GlobalValue::GUID GUID)
The map maintains the list of imports.
AddDefinitionStatus addDefinition(StringRef FromModule, GlobalValue::GUID GUID)
void addGUID(StringRef FromModule, GlobalValue::GUID GUID, GlobalValueSummary::ImportKind ImportKind)
SmallVector< StringRef, 0 > getSourceModules() const
std::optional< GlobalValueSummary::ImportKind > getImportType(StringRef FromModule, GlobalValue::GUID GUID) const
void maybeAddDeclaration(StringRef FromModule, GlobalValue::GUID GUID)
The function importer is automatically importing function from other modules based on the provided su...
Expected< bool > importFunctions(Module &M, const ImportMapTy &ImportList)
Import functions in Module M based on the supplied import list.
ImportFailureReason
The different reasons selectCallee will chose not to import a candidate.
@ LocalLinkageNotInModule
Function summary information to aid decisions and implementation of importing.
unsigned instCount() const
Get the instruction count recorded for this function.
FFlags fflags() const
Get function summary flags.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const GlobalObject * getAliaseeObject() const
Function and variable summary information to aid decisions and implementation of importing.
StringRef modulePath() const
Get the path to the module containing this function.
GlobalValue::LinkageTypes linkage() const
Return linkage type recorded for this global value.
VisibilityTypes getVisibility() const
bool isImplicitDSOLocal() const
static bool isLocalLinkage(LinkageTypes Linkage)
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LinkageTypes getLinkage() const
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
ThreadLocalMode getThreadLocalMode() const
static bool isAvailableExternallyLinkage(LinkageTypes Linkage)
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
void setLinkage(LinkageTypes LT)
unsigned getAddressSpace() const
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Module * getParent()
Get the module that this global value is contained inside of...
const GlobalObject * getAliaseeObject() const
void setDSOLocal(bool Local)
PointerType * getType() const
Global values are always pointers.
@ DefaultVisibility
The GV is visible.
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
static bool isInterposableLinkage(LinkageTypes Linkage)
Whether the definition of this global may be replaced by something non-equivalent at link time.
Error materialize()
Make sure this GlobalValue is fully read.
bool canBeOmittedFromSymbolTable() const
True if GV can be left out of the object symbol table.
bool hasAvailableExternallyLinkage() const
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
@ InternalLinkage
Rename collisions when linking (static functions).
@ WeakODRLinkage
Same, but only replaced by something equivalent.
@ ExternalLinkage
Externally visible function.
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Type * getValueType() const
Error move(std::unique_ptr< Module > Src, ArrayRef< GlobalValue * > ValuesToLink, LazyCallback AddLazyFor, bool IsPerformingImport)
Move in the provide values in ValuesToLink from Src.
This is an important class for using LLVM in a threaded context.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static StringRef getOriginalNameBeforePromote(StringRef Name)
Helper to obtain the unpromoted name for a global value (or the original name if not promoted).
A Module instance is used to store all the information related to an LLVM module.
LLVMContext & getContext() const
Get the global data context.
const std::string & getSourceFileName() const
Get the module's original source file name.
iterator_range< alias_iterator > aliases()
iterator_range< global_iterator > globals()
const std::string & getModuleIdentifier() const
Get the module identifier which is, essentially, the name of the module.
iterator_range< global_object_iterator > global_objects()
Expected< std::map< GlobalValue::GUID, PGOCtxProfContext > > loadContexts()
unsigned getAddressSpace() const
Return the address space of the Pointer type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type size() const
Determine the number of elements in the SetVector.
Vector takeVector()
Clear the SetVector and return the underlying vector.
void clear()
Completely clear the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
iterator find(StringRef Key)
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
bool insert(MapEntryTy *KeyValue)
insert - Insert the specified key/value pair into the map.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
char front() const
front - Get the first character in the string.
StringSet - A wrapper for StringMap that provides set-like functionality.
std::pair< typename Base::iterator, bool > insert(StringRef key)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isFunctionTy() const
True if this is an instance of FunctionType.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
std::pair< iterator, bool > insert(const ValueT &V)
bool erase(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
An efficient, type-erasing, non-owning reference to a callable.
The root is the trivial Path to the root value.
A raw_ostream that writes to a file descriptor.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
llvm::Expected< Value > parse(llvm::StringRef JSON)
Parses the provided JSON source, or returns a ParseError.
bool fromJSON(const Value &E, std::string &Out, Path P)
@ OF_Text
The file should be opened in text mode on platforms like z/OS that make this distinction.
This is an optimization pass for GlobalISel generic memory operations.
void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner={})
Log all errors (if any) in E to OS.
Error createFileError(const Twine &F, Error E)
Concatenate a source file path and/or name with an Error.
bool internalizeModule(Module &TheModule, std::function< bool(const GlobalValue &)> MustPreserveGV)
Helper function to internalize functions and variables in a Module.
std::error_code make_error_code(BitcodeError E)
const char * getHotnessName(CalleeInfo::HotnessType HT)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool convertToDeclaration(GlobalValue &GV)
Converts value GV to declaration, or replaces with a declaration if it is an alias.
void renameModuleForThinLTO(Module &M, const ModuleSummaryIndex &Index, bool ClearDSOLocalOnDeclarations, SetVector< GlobalValue * > *GlobalsToImport=nullptr)
Perform in-place global value handling on the given Module for exported local functions renamed and p...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
auto map_range(ContainerTy &&C, FuncTy F)
void ComputeCrossModuleImport(const ModuleSummaryIndex &Index, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, FunctionImporter::ImportListsTy &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)
Compute all the imports and exports for every module in the Index.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void sort(IteratorTy Start, IteratorTy End)
void computeDeadSymbolsAndUpdateIndirectCalls(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing)
Compute all the symbols that are "dead": i.e these that can't be reached in the graph from any of the...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
void updateIndirectCalls(ModuleSummaryIndex &Index)
Update call edges for indirect calls to local functions added from SamplePGO when needed.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Ref
The access may reference the value stored in memory.
cl::opt< bool > EnableMemProfContextDisambiguation
Enable MemProf context disambiguation for thin link.
std::unique_ptr< Module > getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, bool ShouldLazyLoadMetadata=false)
If the given file holds a bitcode image, return a Module for it which does lazy deserialization of fu...
void thinLTOInternalizeModule(Module &TheModule, const GVSummaryMapTy &DefinedGlobals)
Internalize TheModule based on the information recorded in the summaries during global summary-based ...
std::unordered_set< GlobalValueSummary * > GVSummaryPtrSet
A set of global value summary pointers.
void gatherImportedSummariesForModule(StringRef ModulePath, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, const FunctionImporter::ImportMapTy &ImportList, ModuleToSummariesForIndexTy &ModuleToSummariesForIndex, GVSummaryPtrSet &DecSummaries)
Compute the set of summaries needed for a ThinLTO backend compilation of ModulePath.
std::map< std::string, GVSummaryMapTy, std::less<> > ModuleToSummariesForIndexTy
Map of a module name to the GUIDs and summaries we will import from that module.
PrevailingType
PrevailingType enum used as a return type of callback passed to computeDeadSymbolsAndUpdateIndirectCa...
const char * toString(DWARFSectionKind Kind)
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Function * CloneFunction(Function *F, ValueToValueMapTy &VMap, ClonedCodeInfo *CodeInfo=nullptr)
Return a copy of the specified function and add it to that function's module.
void computeDeadSymbolsWithConstProp(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing, bool ImportEnabled)
Compute dead symbols and run constant propagation in combined index after that.
Error EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex)
Emit into OutputFilename the files module ModulePath will import from.
Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndexForFile(StringRef Path, bool IgnoreEmptyThinLTOIndexFile=false)
Parse the module summary index out of an IR file and return the module summary index object if found,...
DenseMap< GlobalValue::GUID, GlobalValueSummary * > GVSummaryMapTy
Map of global value GUID to its summary, used to identify values defined in a particular module,...
void thinLTOFinalizeInModule(Module &TheModule, const GVSummaryMapTy &DefinedGlobals, bool PropagateAttrs)
Based on the information recorded in the summaries during global summary-based analysis:
Struct that holds a reference to a particular GUID in a global value summary.
