1//===- ModuleInliner.cpp - Code related to module inliner -----------------===// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7//===----------------------------------------------------------------------===// 9// This file implements the mechanics required to implement inlining without 10// missing any calls in the module level. It doesn't need any infromation about 11// SCC or call graph, which is different from the SCC inliner. The decisions of 12// which calls are profitable to inline are implemented elsewhere. 14//===----------------------------------------------------------------------===// 47#define DEBUG_TYPE "module-inline" 50STATISTIC(NumDeleted,
"Number of functions deleted because all callers found");
54cl::desc(
"If using a contextual profile in this module, and an indirect " 55"call target is marked as alwaysinline, perform indirect call " 56"promotion for that target. If multiple targets for an indirect " 57"call site fit this description, they are all promoted."));
59/// Return true if the specified inline history ID 60/// indicates an inline history that includes the specified function. 64while (InlineHistoryID != -1) {
65assert(
unsigned(InlineHistoryID) < InlineHistory.size() &&
66"Invalid inline history ID");
67if (InlineHistory[InlineHistoryID].first ==
F)
69 InlineHistoryID = InlineHistory[InlineHistoryID].second;
82// It should still be possible to run the inliner as a stand-alone module 83// pass, for test scenarios. In that case, we default to the 84// DefaultInlineAdvisor, which doesn't need to keep state between module 85// pass runs. It also uses just the default InlineParams. In this case, we 86// need to use the provided FAM, which is valid for the duration of the 87// inliner pass, and thus the lifetime of the owned advisor. The one we 88// would get from the MAM can be invalidated as a result of the inliner's 90 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
96"Expected a present InlineAdvisorAnalysis also have an " 97"InlineAdvisor initialized");
98return *IAA->getAdvisor();
104// Either this is a normal library function or a "vectorizable" 105// function. Not using the VFDatabase here because this query 106// is related only to libraries handled via the TLI. 116if (!IAA.tryCreate(Params, Mode, {},
118 M.getContext().emitError(
119"Could not setup Inlining Advisor for the requested " 120"mode and/or options");
142// In the module inliner, a priority-based worklist is used for calls across 143// the entire Module. With this module inliner, the inline order is not 144// limited to bottom-up order. More globally scope inline order is enabled. 145// Also, the inline deferral logic become unnecessary in this module inliner. 146// It is possible to use other priority heuristics, e.g. profile-based 149// TODO: Here is a huge amount duplicate code between the module inliner and 150// the SCC inliner, which need some refactoring. 152assert(Calls !=
nullptr &&
"Expected an initialized InlineOrder");
154// Populate the initial list of calls in this module. 159if (
auto *CB = dyn_cast<CallBase>(&
I)) {
160if (
Function *Callee = CB->getCalledFunction()) {
161if (!Callee->isDeclaration())
162 Calls->push({CB, -1});
163elseif (!isa<IntrinsicInst>(
I)) {
168 << NV(
"Callee", Callee) <<
" will not be inlined into " 169 << NV(
"Caller", CB->getCaller())
170 <<
" because its definition is unavailable" 175 CB->isIndirectCall()) {
182for (
auto &[CB,
Target] : ICPCandidates) {
184 Calls->push({DirectCB, -1});
189// When inlining a callee produces new call sites, we want to keep track of 190// the fact that they were inlined from the callee. This allows us to avoid 191// infinite inlining in some obscure cases. To represent this, we use an 192// index into the InlineHistory vector. 195// Track the dead functions to delete once finished with inlining calls. We 196// defer deleting these to make it easier to handle the call graph updates. 199// Loop forward over all of the calls. 200while (!Calls->empty()) {
203constint InlineHistoryID =
P.second;
208 <<
" Function size: " <<
F.getInstructionCount()
216if (InlineHistoryID != -1 &&
222auto Advice = Advisor.getAdvice(*CB,
/*OnlyMandatory*/false);
223// Check whether we want to inline this callsite. 224if (!Advice->isInliningRecommended()) {
225 Advice->recordUnattemptedInlining();
229// Setup the data structure used to plumb customization into the 230// `InlineFunction` routine. 232 GetAssumptionCache, PSI,
239if (!
IR.isSuccess()) {
240 Advice->recordUnsuccessfulInlining(
IR);
250// Add any new callsites to defined functions to the worklist. 251if (!IFI.InlinedCallSites.empty()) {
252int NewHistoryID = InlineHistory.
size();
256Function *NewCallee = ICB->getCalledFunction();
258// Try to promote an indirect (virtual) call without waiting for 259// the post-inline cleanup and the next DevirtSCCRepeatedPass 260// iteration because the next iteration may not happen and we may 262// FIXME: enable for ctxprof. 265 NewCallee = ICB->getCalledFunction();
269 Calls->push({ICB, NewHistoryID});
273// For local functions, check whether this makes the callee trivially 274// dead. In that case, we can drop the body of the function eagerly 275// which may reduce the number of callers of other functions to one, 276// changing inline cost thresholds. 277bool CalleeWasDeleted =
false;
278if (
Callee.hasLocalLinkage()) {
279// To check this we also need to nuke any dead constant uses (perhaps 280// made dead by this operation on other functions). 281Callee.removeDeadConstantUsers();
282// if (Callee.use_empty() && !CG.isLibFunction(Callee)) { 284 Calls->erase_if([&](
const std::pair<CallBase *, int> &Call) {
285returnCall.first->getCaller() == &Callee;
287// Clear the body and queue the function itself for deletion when we 289// Note that after this point, it is an error to do anything other 290// than use the callee's address or delete it. 291Callee.dropAllReferences();
293"Cannot put cause a function to become dead twice!");
295 CalleeWasDeleted =
true;
299 Advice->recordInliningWithCalleeDeleted();
301 Advice->recordInlining();
304// Now that we've finished inlining all of the calls across this module, 305// delete all of the trivially dead functions. 307// Note that this walks a pointer set which has non-deterministic order but 308// that is OK as all we do is delete things and add pointers to unordered 310for (
Function *DeadF : DeadFunctions) {
311// Clear out any cached analyses. 314// And delete the actual function from the module. 315M.getFunctionList().erase(DeadF);
Expand Atomic instructions
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
static bool isKnownLibFunction(Function &F, TargetLibraryInfo &TLI)
Legalize the Machine IR a function s Machine IR
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
cl::opt< bool > CtxProfPromoteAlwaysInline("ctx-prof-promote-alwaysinline", cl::init(false), cl::Hidden, cl::desc("If using a contextual profile in this module, and an indirect " "call target is marked as alwaysinline, perform indirect call " "promotion for that target. If multiple targets for an indirect " "call site fit this description, they are all promoted."))
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A manager for alias analyses.
A container for analyses that lazily runs them and caches their results.
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
static void collectIndirectCallPromotionList(CallBase &IC, Result &Profile, SetVector< std::pair< CallBase *, Function * > > &Candidates)
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
virtual void onPassExit(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
InlineResult is basically true or false.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
A Module instance is used to store all the information related to an LLVM module.
Diagnostic information for missed-optimization remarks.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
A vector that has set insertion semantics.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
bool isKnownVectorFunctionInLibrary(StringRef F) const
Check if the function "F" is listed in a library known to LLVM.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Target - Wrapper for Target specific information.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)
Promote the given indirect call site to conditionally call Callee.
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
auto reverse(ContainerTy &&C)
std::unique_ptr< InlineOrder< std::pair< CallBase *, int > > > getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params, ModuleAnalysisManager &MAM, Module &M)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...