1//===---- GlobalMergeFunctions.cpp - Global merge functions -------*- C++ -===// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7//===----------------------------------------------------------------------===// 9// This pass implements the global merge function pass. 11//===----------------------------------------------------------------------===// 23#define DEBUG_TYPE "global-merge-func" 30cl::desc(
"Disable codegen data for function merging. Local " 31"merging is still enabled within a module."),
35"Number of functions that are actually merged using function hash");
36STATISTIC(NumAnalyzedModues,
"Number of modules that are analyzed");
37STATISTIC(NumAnalyzedFunctions,
"Number of functions that are analyzed");
38STATISTIC(NumEligibleFunctions,
"Number of functions that are eligible");
40/// Returns true if the \OpIdx operand of \p CI is the callee operand. 49 ? dyn_cast_or_null<Function>(
53if (Callee->isIntrinsic())
55autoName = Callee->getName();
56// objc_msgSend stubs must be called, and can't have their address taken. 57if (
Name.starts_with(
"objc_msgSend$"))
59// Calls to dtrace probes must generate unique patchpoints. 60if (
Name.starts_with(
"__dtrace"))
64// The operand is the callee and it has already been signed. Ignore this 65// because we cannot add another ptrauth bundle to the call instruction. 69// The target of the arc-attached call must be a constant and cannot be 78/// Returns true if function \p F is eligible for merging. 80if (
F->isDeclaration())
83if (
F->hasFnAttribute(llvm::Attribute::NoMerge) ||
84F->hasFnAttribute(llvm::Attribute::AlwaysInline))
87if (
F->hasAvailableExternallyLinkage())
90if (
F->getFunctionType()->isVarArg())
96// If function contains callsites with musttail, if we merge 97// it, the merged function will have the musttail callsite, but 98// the number of parameters can change, thus the parameter count 99// of the callsite will mismatch with the function itself. 102constauto *CB = dyn_cast<CallBase>(&
I);
103if (CB && CB->isMustTailCall())
112switch (
I->getOpcode()) {
113case Instruction::Load:
114case Instruction::Store:
115case Instruction::Call:
116case Instruction::Invoke:
123// This function takes an instruction, \p I, and an operand index, \p OpIdx. 124// It returns true if the operand should be ignored in the hash computation. 125// If \p OpIdx is out of range based on the other instruction context, it cannot 128if (OpIdx >=
I->getNumOperands())
134if (!isa<Constant>(
I->getOperand(OpIdx)))
137if (
constauto *CI = dyn_cast<CallBase>(
I))
144Type *SrcTy = V->getType();
159if (
auto *SrcAT = dyn_cast<ArrayType>(SrcTy)) {
160auto *DestAT = dyn_cast<ArrayType>(DestTy);
162assert(SrcAT->getNumElements() == DestAT->getNumElements());
164for (
unsignedintI = 0, E = SrcAT->getNumElements();
I < E; ++
I) {
167 DestAT->getElementType());
184 ++NumAnalyzedFunctions;
186 ++NumEligibleFunctions;
190// Convert the operand map to a vector for a serialization-friendly 193for (
auto &Pair : *FI.IndexOperandHashMap)
197 M.getModuleIdentifier(), FI.IndexInstruction->size(),
198 std::move(IndexOperandHashes));
200 LocalFunctionMap->insert(SF);
205/// Tuple to hold function info to process merging. 212 : SF(SF),
F(
F), IndexInstruction(
std::
move(IndexInstruction)) {}
215// Given the func info, and the parameterized locations, create and return 216// a new merged function by replacing the original constants with the new 221// Synthesize a new merged function name by appending ".Tgm" to the root 223auto *MergedFunc = FI.
F;
224 std::string NewFunctionName =
226auto *M = MergedFunc->getParent();
227assert(!M->getFunction(NewFunctionName));
230// Get the original params' types. 232// Append const parameter types that are passed in. 233 ParamTypes.
append(ConstParamTypes.
begin(), ConstParamTypes.
end());
235 ParamTypes,
/*isVarArg=*/false);
237// Declare a new function 240if (
auto *SP = MergedFunc->getSubprogram())
246 NewFunction->
addFnAttr(Attribute::NoInline);
248// Add the new function before the root function. 249 M->getFunctionList().insert(MergedFunc->getIterator(), NewFunction);
251// Move the body of MergedFunc into the NewFunction. 252 NewFunction->
splice(NewFunction->
begin(), MergedFunc);
254// Update the original args by the new args. 255auto NewArgIter = NewFunction->
arg_begin();
256for (
Argument &OrigArg : MergedFunc->args()) {
261// Replace the original Constants by the new args. 262unsigned NumOrigArgs = MergedFunc->arg_size();
263for (
unsigned ParamIdx = 0; ParamIdx < ParamLocsVec.
size(); ++ParamIdx) {
265for (
auto [InstIndex, OpndIndex] : ParamLocsVec[ParamIdx]) {
267auto *OrigC = Inst->getOperand(OpndIndex);
268if (OrigC->getType() != NewArg->
getType()) {
269IRBuilder<> Builder(Inst->getParent(), Inst->getIterator());
270 Inst->setOperand(OpndIndex,
273 Inst->setOperand(OpndIndex, NewArg);
281// Given the original function (Thunk) and the merged function (ToFunc), create 282// a thunk to the merged function. 289 Thunk->dropAllReferences();
295unsigned ParamIdx = 0;
298// Add arguments which are passed through Thunk. 300 Args.push_back(
createCast(Builder, &AI, ToFuncTy->getParamType(ParamIdx)));
304// Add new arguments defined by Params. 305for (
auto *Param : Params) {
306assert(ParamIdx < ToFuncTy->getNumParams());
308createCast(Builder, Param, ToFuncTy->getParamType(ParamIdx)));
319if (Thunk->getReturnType()->isVoidTy())
325// Check if the old merged/optimized IndexOperandHashMap is compatible with 326// the current IndexOperandHashMap. An operand hash may not be stable across 327// different builds due to varying modules combined. To address this, we relax 328// the hash check condition by comparing Const hash patterns instead of absolute 329// hash values. For example, let's assume we have three Consts located at idx1, 330// idx3, and idx6, where their corresponding hashes are hash1, hash2, and hash1 331// in the old merged map below: 332// Old (Merged): [(idx1, hash1), (idx3, hash2), (idx6, hash1)] 333// Current: [(idx1, hash1'), (idx3, hash2'), (idx6, hash1')] 334// If the current function also has three Consts in the same locations, 335// with hash sequences hash1', hash2', and hash1' where the first and third 336// are the same as the old hash sequences, we consider them matched. 342for (
constauto &[Index, OldHash] : OldInstOpndIndexToConstHash) {
343auto It = CurrInstOpndIndexToConstHash.
find(Index);
344if (It == CurrInstOpndIndexToConstHash.
end())
347auto CurrHash = It->second;
348auto J = OldHashToCurrHash.
find(OldHash);
349if (J == OldHashToCurrHash.
end())
350 OldHashToCurrHash.
insert({OldHash, CurrHash});
351elseif (J->second != CurrHash)
358// Validate the locations pointed by a param has the same hash and Constant. 364 std::optional<stable_hash> OldHash;
365 std::optional<Constant *> OldConst;
369auto [InstIndex, OpndIndex] = Loc;
371constauto *Inst = IndexInstruction.
lookup(InstIndex);
372auto *CurrConst = cast<Constant>(Inst->getOperand(OpndIndex));
375 OldConst = CurrConst;
376 }
elseif (CurrConst != *OldConst || CurrHash != *OldHash) {
385constSmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>>
387 std::map<std::vector<stable_hash>,
ParamLocs> HashSeqToLocs;
389unsigned StableFunctionCount = SFS.
size();
391for (
auto &[
IndexPair, Hash] : *RSF.IndexOperandHashMap) {
392// Const hash sequence across stable functions. 393// We will allocate a parameter per unique hash squence. 394// can't use SmallVector as key 395 std::vector<stable_hash> ConstHashSeq;
396 ConstHashSeq.push_back(Hash);
398for (
unsigned J = 1; J < StableFunctionCount; ++J) {
400auto SHash = SF->IndexOperandHashMap->at(
IndexPair);
403 ConstHashSeq.push_back(SHash);
409// For each unique Const hash sequence (parameter), add the locations. 410 HashSeqToLocs[ConstHashSeq].push_back(
IndexPair);
414for (
auto &[HashSeq, Locs] : HashSeqToLocs)
427// Collect stable functions related to the current module. 435if (Maps.contains(FI.FunctionHash))
436 HashToFuncs[FI.FunctionHash].emplace_back(&
F, std::move(FI));
439for (
auto &[Hash, Funcs] : HashToFuncs) {
440 std::optional<ParamLocsVecTy> ParamLocsVec;
442auto &SFS = Maps.at(Hash);
446// Iterate functions with the same hash. 447for (
auto &[
F, FI] : Funcs) {
448// Check if the function is compatible with any stable function 449// in terms of the number of instructions and ignored operands. 450if (RFS->InstCount != FI.IndexInstruction->size())
456auto [InstIndex, OpndIndex] = Index;
457assert(InstIndex < FHI.IndexInstruction->
size());
458auto *Inst = FHI.IndexInstruction->lookup(InstIndex);
464if (!hasValidSharedConst(RFS.get(), FI))
467for (
auto &SF : SFS) {
469assert(hasValidSharedConst(SF.get(), FI));
470// Check if there is any stable function that is compatiable with the 473 *FI.IndexOperandHashMap))
475if (!ParamLocsVec.has_value()) {
478 <<
" with Params " << ParamLocsVec->size() <<
"\n");
484// If a stable function matching the current one is found, 485// create a candidate for merging and proceed to the next function. 486 FuncMergeInfos.
emplace_back(SF.get(),
F, FI.IndexInstruction.get());
490unsigned FuncMergeInfoSize = FuncMergeInfos.
size();
491if (FuncMergeInfoSize == 0)
495 << FuncMergeInfoSize <<
" for hash: " << Hash <<
"\n");
497for (
auto &FMI : FuncMergeInfos) {
500// We've already validated all locations of constant operands pointed by 501// the parameters. Populate parameters pointing to the original constants. 506auto &[InstIndex, OpndIndex] =
ParamLocs[0];
507auto *Inst = FMI.IndexInstruction->lookup(InstIndex);
508auto *Opnd = cast<Constant>(Inst->getOperand(OpndIndex));
513// Create a merged function derived from the current function. 518dbgs() <<
"[GlobalMergeFunc] Merged function (hash:" << FMI.SF->Hash
519 <<
") " << MergedFunc->
getName() <<
" generated from " 520 << FMI.F->getName() <<
":\n";
524// Transform the current function into a thunk that calls the merged 528dbgs() <<
"[GlobalMergeFunc] Thunk generated: \n";
531 ++NumMergedFunctions;
539// Initialize the local function map regardless of the merger mode. 540 LocalFunctionMap = std::make_unique<StableFunctionMap>();
542// Disable codegen data for merging. The local merge is still enabled. 546// (Full)LTO module does not have functions added to the index. 547// In this case, we run a local merger without using codegen data. 552 MergerMode = HashFunctionMode::BuildingHashFuncion;
554 MergerMode = HashFunctionMode::UsingHashFunction;
558LLVM_DEBUG(
dbgs() <<
"Emit function map. Size: " << LocalFunctionMap->size()
560// No need to emit the function map if it is empty. 561if (LocalFunctionMap->empty())
569OS.str(),
"in-memory stable function map",
false);
571Triple TT(M.getTargetTriple());
581if (MergerMode == HashFunctionMode::UsingHashFunction) {
582// Use the prior CG data to optimistically create global merge candidates. 586// Emit the local function map to the custom section, __llvm_merge before 588if (MergerMode == HashFunctionMode::BuildingHashFuncion)
590 LocalFunctionMap->finalize();
591 FuncMap = LocalFunctionMap.get();
594returnmerge(M, FuncMap);
599classGlobalMergeFuncPassWrapper :
publicModulePass {
604 GlobalMergeFuncPassWrapper();
609 ModulePass::getAnalysisUsage(AU);
612StringRef getPassName()
const override{
return"Global Merge Functions"; }
614bool runOnModule(
Module &M)
override;
619char GlobalMergeFuncPassWrapper::ID = 0;
621"Global merge function pass",
false,
false)
627returnnew GlobalMergeFuncPassWrapper();
631GlobalMergeFuncPassWrapper::GlobalMergeFuncPassWrapper() :
ModulePass(
ID) {
636bool GlobalMergeFuncPassWrapper::runOnModule(
Module &M) {
638if (
auto *IndexWrapperPass =
639 getAnalysisIfAvailable<ImmutableModuleSummaryIndexWrapperPass>())
640Index = IndexWrapperPass->getIndex();
Performs the initial survey of the specified function
static ParamLocsVecTy computeParamInfo(const SmallVector< std::unique_ptr< StableFunctionMap::StableFunctionEntry > > &SFS)
static bool checkConstHashCompatible(const DenseMap< IndexPair, stable_hash > &OldInstOpndIndexToConstHash, const DenseMap< IndexPair, stable_hash > &CurrInstOpndIndexToConstHash)
static cl::opt< bool > DisableCGDataForMerging("disable-cgdata-for-merging", cl::Hidden, cl::desc("Disable codegen data for function merging. Local " "merging is still enabled within a module."), cl::init(false))
static bool canParameterizeCallOperand(const CallBase *CI, unsigned OpIdx)
static Function * createMergedFunction(FuncMergeInfo &FI, ArrayRef< Type * > ConstParamTypes, const ParamLocsVecTy &ParamLocsVec)
static void createThunk(FuncMergeInfo &FI, ArrayRef< Constant * > Params, Function *ToFunc)
global merge Global merge function pass
static Value * createCast(IRBuilder<> &Builder, Value *V, Type *DestTy)
static bool isEligibleInstructionForConstantSharing(const Instruction *I)
static bool ignoreOp(const Instruction *I, unsigned OpIdx)
bool isEligibleFunction(Function *F)
Returns true if function F is eligible for merging.
static bool checkConstLocationCompatible(const StableFunctionMap::StableFunctionEntry &SF, const IndexInstrMap &IndexInstruction, const ParamLocsVecTy &ParamLocsVec)
static bool isCalleeOperand(const CallBase *CI, unsigned OpIdx)
Returns true if the \OpIdx operand of CI is the callee operand.
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
This is the interface to build a ModuleSummaryIndex for a module.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A container for analyses that lazily runs them and caches their results.
Represent the analysis usage information of a pass.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
void setCallingConv(CallingConv::ID CC)
bool isOperandBundleOfType(uint32_t ID, unsigned Idx) const
Return true if the operand at index Idx is a bundle operand that has tag ID ID.
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Value * getCalledOperand() const
const Use & getCalledOperandUse() const
void setAttributes(AttributeList A)
Set the attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
void setSubprogram(DISubprogram *SP)
Set the attached subprogram.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
void splice(Function::iterator ToIt, Function *FromF)
Transfer all blocks from FromF to this function at ToIt.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
Argument * getArg(unsigned i) const
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
GlobalMergeFunc is a ModulePass that implements a function merging mechanism using stable function ha...
void analyze(Module &M)
Analyze module to create stable function into LocalFunctionMap.
void initializeMergerMode(const Module &M)
bool merge(Module &M, const StableFunctionMap *FunctionMap)
Merge functions in the module using the given function map.
void emitFunctionMap(Module &M)
Emit LocalFunctionMap into __llvm_merge section.
static constexpr const char MergingInstanceSuffix[]
The suffix used to identify the merged function that parameterizes the constant values.
void setDLLStorageClass(DLLStorageClassTypes C)
void setLinkage(LinkageTypes LT)
@ InternalLinkage
Rename collisions when linking (static functions).
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
ReturnInst * CreateRet(Value *V)
Create a 'ret <val>' instruction.
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Legacy wrapper pass to provide the ModuleSummaryIndex object.
@ OB_clang_arc_attachedcall
This class implements a map that also provides access to all stored values in a deterministic order.
ValueT lookup(const KeyT &Key) const
static std::unique_ptr< MemoryBuffer > getMemBuffer(StringRef InputData, StringRef BufferName="", bool RequiresNullTerminator=true)
Open the specified memory range as a MemoryBuffer.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
bool hasExportedFunctions(const Module &M) const
Check if the given Module has any functions available for exporting in the index.
A Module instance is used to store all the information related to an LLVM module.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
Triple - Helper class for working with autoconf configuration names.
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getStructElementType(unsigned N) const
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getStructNumElements() const
bool isStructTy() const
True if this is an instance of StructType.
bool isIntegerTy() const
True if this is an instance of IntegerType.
const Use & getOperandUse(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
StringRef getName() const
Return a constant reference to the value's name.
void dump() const
Support for debugging, callable in GDB: V->dump()
A raw_ostream that writes to an SmallVector or SmallString.
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
bool hasStableFunctionMap()
const StableFunctionMap * getStableFunctionMap()
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
ModulePass * createGlobalMergeFuncPass()
This pass performs merging similar functions globally.
FunctionHashInfo StructuralHashWithDifferences(const Function &F, IgnoreOperandFunc IgnoreOp)
Computes a structural hash of a given function, considering the structure and content of the function...
std::pair< unsigned, unsigned > IndexPair
The pair of an instruction index and a operand index.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
StringRef get_stable_name(StringRef Name)
@ Global
Append to llvm.global_dtors.
void initializeGlobalMergeFuncPassWrapperPass(PassRegistry &)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
void embedBufferInModule(Module &M, MemoryBufferRef Buf, StringRef SectionName, Align Alignment=Align(1))
Embed the memory buffer Buf into the module M as a global using the specified section name.
std::string getCodeGenDataSectionName(CGDataSectKind CGSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Implement std::hash so that hash_code can be used in STL containers.
Tuple to hold function info to process merging.
FuncMergeInfo(StableFunctionMap::StableFunctionEntry *SF, Function *F, IndexInstrMap *IndexInstruction)
StableFunctionMap::StableFunctionEntry * SF
IndexInstrMap * IndexInstruction
This struct is a compact representation of a valid (non-zero power of two) alignment.
PreservedAnalyses run(Module &M, AnalysisManager< Module > &)
const ModuleSummaryIndex * ImportSummary
static void serialize(raw_ostream &OS, const StableFunctionMap *FunctionMap)
A static helper function to serialize the stable function map without owning the stable function map.
An efficient form of StableFunction for fast look-up.
std::unique_ptr< IndexOperandHashMapType > IndexOperandHashMap
A map from an IndexPair to a stable_hash which was skipped.
unsigned InstCount
The number of instructions.
const HashFuncsMapType & getFunctionMap() const
Get the HashToFuncs map for serialization.
A stable function is a function with a stable hash while tracking the locations of ignored operands a...