Movatterモバイル変換

#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes
class	llvm::slpvectorizer::BoUpSLP
	Bottom Up SLP Vectorizer.More...

struct	llvm::slpvectorizer::BoUpSLP::EdgeInfo
	This structure holds any data we need about the edges being traversed during buildTree_rec().More...

class	llvm::slpvectorizer::BoUpSLP::LookAheadHeuristics
	A helper class used for scoring candidates for two consecutive lanes.More...

class	llvm::slpvectorizer::BoUpSLP::VLOperands
	A helper data structure to hold the operands of a vector of instructions.More...

struct	llvm::GraphTraits< BoUpSLP * >

struct	llvm::GraphTraits< BoUpSLP * >::ChildIteratorType
	Add the VectorizableTree to the index iterator to be able to return TreeEntry pointers.More...

class	llvm::GraphTraits< BoUpSLP * >::nodes_iterator
	For the node iterator we just need to turn the TreeEntry iterator into a TreeEntry* iterator so that it dereferences to NodeRef.More...

struct	llvm::DOTGraphTraits< BoUpSLP * >

class	llvm::slpvectorizer::BoUpSLP::ShuffleCostEstimator
	Merges shuffle masks and emits final shuffle instruction, if required.More...

class	llvm::slpvectorizer::BoUpSLP::ShuffleInstructionBuilder
	Merges shuffle masks and emits final shuffle instruction, if required.More...

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

namespace	llvm::slpvectorizer
	A private "module" namespace for types and utilities used by this pass.

Macros
#define	SV_NAME "slp-vectorizer"

#define	DEBUG_TYPE "SLP"

Functions
	STATISTIC (NumVectorInstructions, "Number ofvector instructions generated")

	DEBUG_COUNTER (VectorizedGraphs, "slp-vectorized", "Controls which SLP graphs should be vectorized.")

staticbool	isValidElementType (Type *Ty)
	Predicate for the element types that the SLP vectorizer supports.

staticType *	getValueType (Value *V)
	Returns the type of the given value/instruction`V`.

staticunsigned	getNumElements (Type *Ty)

staticFixedVectorType *	getWidenedType (Type *ScalarTy,unsigned VF)

staticunsigned	getFullVectorNumberOfElements (const TargetTransformInfo &TTI,Type *Ty,unsigned Sz)
	Returns the number of elements of the given type`Ty`, not less than`Sz`, which forms type, which splits by`TTI` into whole vector types during legalization.

staticunsigned	getFloorFullVectorNumberOfElements (const TargetTransformInfo &TTI,Type *Ty,unsigned Sz)
	Returns the number of elements of the given type`Ty`, not greater than`Sz`, which forms type, which splits by`TTI` into whole vector types during legalization.

static void	transformScalarShuffleIndiciesToVector (unsigned VecTyNumElements,SmallVectorImpl< int > &Mask)

staticunsigned	getShufflevectorNumGroups (ArrayRef<Value * > VL)

staticSmallVector< int >	calculateShufflevectorMask (ArrayRef<Value * > VL)

staticbool	isConstant (Value *V)

staticbool	isVectorLikeInstWithConstOps (Value *V)
	Checks if`V` is one of vector-like instructions, i.e.

staticunsigned	getPartNumElems (unsigned Size,unsigned NumParts)
	Returns power-of-2 number of elements in a single register (part), given the total number of elements`Size` and number of registers (parts)`NumParts`.

staticunsigned	getNumElems (unsigned Size,unsigned PartNumElems,unsigned Part)
	Returns correct remaining number of elements, considering total amount`Size`, (power-of-2 number) of elements in a single register`PartNumElems` and current register (part)`Part`.

static std::string	shortBundleName (ArrayRef<Value * > VL, intIdx=-1)
	Print a short descriptor of the instruction bundle suitable for debug output.

staticbool	allSameBlock (ArrayRef<Value * > VL)

staticbool	allConstant (ArrayRef<Value * > VL)

staticbool	isSplat (ArrayRef<Value * > VL)

staticbool	isCommutative (Instruction *I)

template<typenameT >
static std::optional<unsigned >	getInsertExtractIndex (const Value *Inst,unsigned Offset)

static std::optional<unsigned >	getElementIndex (const Value *Inst,unsigned Offset=0)

staticSmallBitVector	buildUseMask (int VF,ArrayRef< int > Mask, UseMask MaskArg)
	Prepares a use bitset for the given mask either for the first argument or for the second.

template<bool IsPoisonOnly = false>
staticSmallBitVector	isUndefVector (const Value *V,const SmallBitVector &UseMask={})
	Checks if the given value is actually an undefined constant vector.

static std::optional<TargetTransformInfo::ShuffleKind >	isFixedVectorShuffle (ArrayRef<Value * > VL,SmallVectorImpl< int > &Mask,AssumptionCache *AC)
	Checks if the vector of instructions can be represented as a shuffle, like: x0 = extractelement <4 x i8> x, i32 0 x3 = extractelement <4 x i8> x, i32 3 y1 = extractelement <4 x i8> y, i32 1 y2 = extractelement <4 x i8> y, i32 2 x0x0 = mul i8 x0, x0 x3x3 = mul i8 x3, x3 y1y1 = mul i8 y1, y1 y2y2 = mul i8 y2, y2 ins1 = insertelement <4 x i8> poison, i8 x0x0, i32 0 ins2 = insertelement <4 x i8> ins1, i8 x3x3, i32 1 ins3 = insertelement <4 x i8> ins2, i8 y1y1, i32 2 ins4 = insertelement <4 x i8> ins3, i8 y2y2, i32 3 ret <4 x i8> ins4 can be transformed into: %1 = shufflevector <4 x i8> x, <4 x i8> y, <4 x i32> <i32 0, i32 3, i32 5, i32 6> %2 = mul <4 x i8> %1, %1 ret <4 x i8> %2 Mask will return the Shuffle Mask equivalent to the extracted elements.

static std::optional<unsigned >	getExtractIndex (Instruction *E)

staticbool	isValidForAlternation (unsigned Opcode)

static InstructionsState	getSameOpcode (ArrayRef<Value * > VL,const TargetLibraryInfo &TLI)

staticbool	areCompatibleCmpOps (Value BaseOp0,Value BaseOp1,Value Op0,Value Op1,const TargetLibraryInfo &TLI)
	Checks if the provided operands of 2 cmp instructions are compatible, i.e.

staticbool	isCmpSameOrSwapped (const CmpInst BaseCI,const CmpInst CI,const TargetLibraryInfo &TLI)

staticbool	allSameType (ArrayRef<Value * > VL)

staticbool	doesInTreeUserNeedToExtract (Value Scalar,Instruction UserInst,TargetLibraryInfo TLI,const TargetTransformInfo TTI)

staticMemoryLocation	getLocation (Instruction *I)

staticbool	isSimple (Instruction *I)

static void	addMask (SmallVectorImpl< int > &Mask,ArrayRef< int > SubMask,bool ExtendingManyInputs=false)
	Shuffles`Mask` in accordance with the given`SubMask`.

static void	fixupOrderingIndices (MutableArrayRef<unsigned > Order)
	Order may have elements assigned special value (size) which is out of bounds.

staticSmallBitVector	getAltInstrMask (ArrayRef<Value * > VL,unsigned Opcode0,unsigned Opcode1)

static void	llvm::inversePermutation (ArrayRef<unsigned > Indices,SmallVectorImpl< int > &Mask)

static void	llvm::reorderScalars (SmallVectorImpl<Value * > &Scalars,ArrayRef< int > Mask)
	Reorders the list of scalars in accordance with the given`Mask`.

staticbool	llvm::areAllOperandsNonInsts (Value *V)
	Checks if the provided value does not require scheduling.

staticbool	llvm::isUsedOutsideBlock (Value *V)
	Checks if the provided value does not require scheduling.

staticbool	llvm::doesNotNeedToBeScheduled (Value *V)
	Checks if the specified value does not require scheduling.

staticbool	llvm::doesNotNeedToSchedule (ArrayRef<Value * > VL)
	Checks if the specified array of instructions does not require scheduling.

staticbool	llvm::hasFullVectorsOrPowerOf2 (const TargetTransformInfo &TTI,Type *Ty,unsigned Sz)
	Returns true if widened type of`Ty` elements with size`Sz` represents full vector type, i.e.

staticunsigned	llvm::getNumberOfParts (const TargetTransformInfo &TTI,VectorType *VecTy,const unsigned Limit=std::numeric_limits<unsigned >::max())
	Returns number of parts, the type`VecTy` will be split at the codegen phase.

static void	reorderReuses (SmallVectorImpl< int > &Reuses,ArrayRef< int > Mask)
	Reorders the given`Reuses` mask according to the given`Mask`.

static void	reorderOrder (SmallVectorImpl<unsigned > &Order,ArrayRef< int > Mask,bool BottomOrder=false)
	Reorders the given`Order` according to the given`Mask`.

staticbool	arePointersCompatible (Value Ptr1,Value Ptr2,const TargetLibraryInfo &TLI,bool CompareOpcodes=true)

template<typenameT >
staticAlign	computeCommonAlignment (ArrayRef<Value * > VL)
	Calculates minimal alignment as a common alignment.

staticbool	isReverseOrder (ArrayRef<unsigned > Order)
	Check if`Order` represents reverse order.

static std::optional<Value * >	calculateRtStride (ArrayRef<Value * > PointerOps,Type ElemTy,const DataLayout &DL,ScalarEvolution &SE,SmallVectorImpl<unsigned > &SortedIndices,Instruction Inst=nullptr)
	Checks if the provided list of pointers`Pointers` represents the strided pointers for type ElemTy.

static std::pair<InstructionCost,InstructionCost >	getGEPCosts (const TargetTransformInfo &TTI,ArrayRef<Value * > Ptrs,Value BasePtr,unsigned Opcode,TTI::TargetCostKind CostKind,Type ScalarTy,VectorType *VecTy)
	Calculate the scalar and the vector costs from vectorizing set of GEPs.

staticInstructionCost	getShuffleCost (const TargetTransformInfo &TTI,TTI::ShuffleKind Kind,VectorType Tp,ArrayRef< int > Mask={},TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, intIndex=0,VectorType SubTp=nullptr,ArrayRef<const Value * > Args={})
	Returns the cost of the shuffle instructions with the given`Kind`, vector type`Tp` and optional`Mask`.

staticValue *	createInsertVector (IRBuilderBase &Builder,Value Vec,Value V,unsigned Index,function_ref<Value (Value ,Value *,ArrayRef< int >)> Generator={})
	Correctly creates insert_subvector, checking that the index is multiple of the subvectors length.

staticValue *	createExtractVector (IRBuilderBase &Builder,Value *Vec,unsigned SubVecVF,unsigned Index)
	Correctly creates extract_subvector, checking that the index is multiple of the subvectors length.

staticbool	clusterSortPtrAccesses (ArrayRef<Value * > VL,ArrayRef<BasicBlock * > BBs,Type *ElemTy,const DataLayout &DL,ScalarEvolution &SE,SmallVectorImpl<unsigned > &SortedIndices)

staticbool	areTwoInsertFromSameBuildVector (InsertElementInst VU,InsertElementInst V,function_ref<Value (InsertElementInst )> GetBaseOperand)
	Check if two insertelement instructions are from the same buildvector.

staticbool	isRepeatedNonIdentityClusteredMask (ArrayRef< int > Mask,unsigned Sz)
	Checks if the given mask is a "clustered" mask with the same clusters of size`Sz`, which are not identity submasks.

static void	combineOrders (MutableArrayRef<unsigned > Order,ArrayRef<unsigned > SecondaryOrder)

staticLLVM_DUMP_METHOD void	dumpOrder (const BoUpSLP::OrdersType &Order)

static void	gatherPossiblyVectorizableLoads (const BoUpSLP &R,ArrayRef<Value * > VL,const DataLayout &DL,ScalarEvolution &SE,const TargetTransformInfo &TTI,SmallVectorImpl<SmallVector< std::pair<LoadInst *, int > > > &GatheredLoads,bool AddNew=true)
	Tries to find subvector of loads and builds new vector of only loads if can be profitable.

staticbool	needToScheduleSingleInstruction (ArrayRef<Value * > VL)

static std::pair< size_t, size_t >	generateKeySubkey (Value V,const TargetLibraryInfo TLI,function_ref<hash_code(size_t,LoadInst *)> LoadsSubkeyGenerator,bool AllowAlternate)
	Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.

staticbool	isAlternateInstruction (const Instruction I,const Instruction MainOp,const Instruction *AltOp,const TargetLibraryInfo &TLI)
	Checks if the specified instruction`I` is an alternate operation for the given`MainOp` and`AltOp` instructions.

static std::pair<InstructionCost,InstructionCost >	getVectorCallCosts (CallInst CI,FixedVectorType VecTy,TargetTransformInfo TTI,TargetLibraryInfo TLI,ArrayRef<Type * > ArgTys)

staticSmallVector<Type * >	buildIntrinsicArgTypes (const CallInst CI,const Intrinsic::ID ID,const unsigned VF,unsigned MinBW,const TargetTransformInfo TTI)
	Builds the arguments types vector for the given call instruction with the given`ID` for the specified vector factor.

staticbool	isLoadCombineCandidateImpl (Value Root,unsigned NumElts,TargetTransformInfo TTI,bool MustMatchOrInst)

staticbool	isFirstInsertElement (const InsertElementInst IE1,const InsertElementInst IE2)
	Checks if the`IE1` instructions is followed by`IE2` instruction in the buildvector sequence.

template<typenameT >
staticT *	performExtractsShuffleAction (MutableArrayRef< std::pair<T ,SmallVector< int > > > ShuffleMask,Value Base,function_ref<unsigned(T )> GetVF,function_ref< std::pair<T ,bool >(T ,ArrayRef< int >,bool)> ResizeAction,function_ref<T (ArrayRef< int >,ArrayRef<T * >)> Action)
	Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.

staticInstruction *	propagateMetadata (Instruction Inst,ArrayRef<Value > VL)

staticRecurKind	getRdxKind (Value *V)
	Gets recurrence kind from the specified value.

staticbool	checkTreeSizes (ArrayRef< std::pair<unsigned,unsigned > > Sizes,bool First)
	Checks if the quadratic mean deviation is less than 90% of the mean size.

static std::optional<unsigned >	getAggregateSize (Instruction *InsertInst)

static void	findBuildAggregate_rec (Instruction LastInsertInst,TargetTransformInfo TTI,SmallVectorImpl<Value * > &BuildVectorOpds,SmallVectorImpl<Value * > &InsertElts,unsigned OperandOffset,const BoUpSLP &R)

staticbool	findBuildAggregate (Instruction LastInsertInst,TargetTransformInfo TTI,SmallVectorImpl<Value * > &BuildVectorOpds,SmallVectorImpl<Value * > &InsertElts,const BoUpSLP &R)
	Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.

staticInstruction *	getReductionInstr (const DominatorTree DT,PHINode P,BasicBlock ParentBB,LoopInfo LI)
	Try and get a reduction instruction from a phi node.

staticbool	matchRdxBop (Instruction I,Value &V0,Value *&V1)

staticInstruction *	tryGetSecondaryReductionRoot (PHINode Phi,Instruction Root)
	We could have an initial reduction that is not an add.

staticInstruction *	getNonPhiOperand (Instruction I,PHINode Phi)
	`Returns` the first operand of`I` that does not match`Phi`.

staticbool	isReductionCandidate (Instruction *I)
	\Returns true if`I` is a candidate instruction for reduction vectorization.

template<typenameT >
staticbool	tryToVectorizeSequence (SmallVectorImpl<T * > &Incoming,function_ref<bool(T ,T )> Comparator,function_ref<bool(T ,T )> AreCompatible,function_ref<bool(ArrayRef<T * >,bool)> TryToVectorizeHelper,bool MaxVFOnly,BoUpSLP &R)

template<bool IsCompatibility>
staticbool	compareCmp (Value V,Value V2,TargetLibraryInfo &TLI,const DominatorTree &DT)
	Compare two cmp instructions.

Variables
staticcl::opt<bool >	RunSLPVectorization ("vectorize-slp", cl::init(true), cl::Hidden,cl::desc("Run the SLP vectorization passes"))

staticcl::opt<bool >	SLPReVec ("slp-revec", cl::init(false), cl::Hidden,cl::desc("Enable vectorizationfor widervector utilization"))

staticcl::opt< int >	SLPCostThreshold ("slp-threshold", cl::init(0), cl::Hidden,cl::desc("Only vectorizeif you gain more than this " "number "))

staticcl::opt<bool >	SLPSkipEarlyProfitabilityCheck ("slp-skip-early-profitability-check", cl::init(false), cl::Hidden,cl::desc("Whentrue, SLP vectorizer bypasses profitabilitychecks based on " "heuristics and makes vectorization decision via cost modeling."))

staticcl::opt<bool >	ShouldVectorizeHor ("slp-vectorize-hor", cl::init(true), cl::Hidden,cl::desc("Attempt to vectorize horizontal reductions"))

staticcl::opt<bool >	ShouldStartVectorizeHorAtStore ("slp-vectorize-hor-store", cl::init(false), cl::Hidden,cl::desc("Attempt to vectorize horizontalreductions feeding into a store"))

staticcl::opt< int >	MaxVectorRegSizeOption ("slp-max-reg-size", cl::init(128), cl::Hidden,cl::desc("Attempt to vectorizefor this register size in bits"))

staticcl::opt<unsigned >	MaxVFOption ("slp-max-vf", cl::init(0), cl::Hidden,cl::desc("Maximum SLP vectorization factor (0=unlimited)"))

staticcl::opt< int >	ScheduleRegionSizeBudget ("slp-schedule-budget", cl::init(100000), cl::Hidden,cl::desc("Limit the size of the SLP schedulingregion per block"))
	Limits the size of scheduling regions in a block.

staticcl::opt< int >	MinVectorRegSizeOption ("slp-min-reg-size", cl::init(128), cl::Hidden,cl::desc("Attempt to vectorizefor this register size in bits"))

staticcl::opt<unsigned >	RecursionMaxDepth ("slp-recursion-max-depth", cl::init(12), cl::Hidden,cl::desc("Limit therecursion depth when building a vectorizable tree"))

staticcl::opt<unsigned >	MinTreeSize ("slp-min-tree-size", cl::init(3), cl::Hidden,cl::desc("Only vectorize small treesif they are fully vectorizable"))

staticcl::opt< int >	LookAheadMaxDepth ("slp-max-look-ahead-depth", cl::init(2), cl::Hidden,cl::desc("The maximum look-ahead depthfor operand reordering scores"))

staticcl::opt< int >	RootLookAheadMaxDepth ("slp-max-root-look-ahead-depth", cl::init(2), cl::Hidden,cl::desc("The maximum look-ahead depthfor searching best rooting option"))

staticcl::opt<unsigned >	MinProfitableStridedLoads ("slp-min-strided-loads", cl::init(2), cl::Hidden,cl::desc("The minimum number ofloads, which should be considered strided, " "if the stride is > 1 or is runtime value"))

staticcl::opt<unsigned >	MaxProfitableLoadStride ("slp-max-stride", cl::init(8), cl::Hidden,cl::desc("The maximum stride, considered to be profitable."))

staticcl::opt<bool >	ViewSLPTree ("view-slp-tree", cl::Hidden,cl::desc("Display the SLP trees with Graphviz"))

staticcl::opt<bool >	VectorizeNonPowerOf2 ("slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden,cl::desc("Try to vectorize with non-power-of-2 number of elements."))

staticconst unsigned	AliasedCheckLimit = 10

static constexpr int	UsesLimit = 64

staticconst unsigned	MaxMemDepDistance = 160

staticconst int	MinScheduleRegionSize = 16
	If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.

staticconst unsigned	MaxPHINumOperands = 128
	Maximum allowed number of operands in the PHI nodes.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "SLP"

Definition at line110 of fileSLPVectorizer.cpp.

◆ SV_NAME

#define SV_NAME "slp-vectorizer"

Definition at line109 of fileSLPVectorizer.cpp.

Function Documentation

◆ addMask()

static void addMask	(	SmallVectorImpl< int > &	Mask,
		ArrayRef< int >	SubMask,
		bool	ExtendingManyInputs =`false`
	)

static

ShufflesMask in accordance with the givenSubMask.

Parameters

ExtendingManyInputs

Supports reshuffling of the mask with not only one but two input vectors.

Definition at line1151 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::ArrayRef< T >::begin(),llvm::ArrayRef< T >::empty(),llvm::ArrayRef< T >::end(),I,llvm::PoisonMaskElem, andllvm::ArrayRef< T >::size().

Referenced byllvm::slpvectorizer::BoUpSLP::getReorderingData(), andllvm::slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ allConstant()

staticbool allConstant

(

ArrayRef<Value * >

)

static

Returns: True if all of the values inVL are constants (but not globals/constant expressions).

Definition at line485 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(), andisConstant().

Referenced byllvm::slpvectorizer::BoUpSLP::getReorderingData(),llvm::slpvectorizer::BoUpSLP::isTreeNotExtendable(),llvm::slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable(), andllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ allSameBlock()

staticbool allSameBlock

(

ArrayRef<Value * >

)

static

Returns: true if all of the instructions inVL are in the same block or false otherwise.

Definition at line461 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),llvm::ArrayRef< T >::end(),llvm::find_if(),llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(),II, andisVectorLikeInstWithConstOps().

Referenced byllvm::slpvectorizer::BoUpSLP::isTreeNotExtendable(),llvm::slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable(), andllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ allSameType()

staticbool allSameType

(

ArrayRef<Value * >

)

static

Returns: true if all of the values inVL have the same type or false otherwise.

Definition at line1093 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),llvm::ArrayRef< T >::drop_front(), andllvm::ArrayRef< T >::front().

Referenced byllvm::slpvectorizer::BoUpSLP::buildTree(), andllvm::slpvectorizer::BoUpSLP::getReorderingData().

◆ areCompatibleCmpOps()

staticbool areCompatibleCmpOps	(	Value *	BaseOp0,
		Value *	BaseOp1,
		Value *	Op0,
		Value *	Op1,
		const TargetLibraryInfo &	TLI
	)

static

Checks if the provided operands of 2 cmp instructions are compatible, i.e.

compatible instructions or constants, or just some other regular values.

Definition at line873 of fileSLPVectorizer.cpp.

ReferencesgetSameOpcode(), andisConstant().

Referenced byisCmpSameOrSwapped().

◆ arePointersCompatible()

staticbool arePointersCompatible	(	Value *	Ptr1,
		Value *	Ptr2,
		const TargetLibraryInfo &	TLI,
		bool	CompareOpcodes =`true`
	)

static

Definition at line4790 of fileSLPVectorizer.cpp.

ReferencesgetSameOpcode(),llvm::getUnderlyingObject(),isConstant(), andRecursionMaxDepth.

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ areTwoInsertFromSameBuildVector()

staticbool areTwoInsertFromSameBuildVector	(	InsertElementInst *	VU,
		InsertElementInst *	V,
		function_ref<Value (InsertElementInst )>	GetBaseOperand
	)

static

Check if two insertelement instructions are from the same buildvector.

Definition at line5500 of fileSLPVectorizer.cpp.

ReferencesgetElementIndex(),llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(),llvm::InsertElementInst::getType(),llvm::Value::hasOneUse(),llvm::SmallBitVector::set(), andllvm::SmallBitVector::test().

Referenced byllvm::slpvectorizer::BoUpSLP::getTreeCost(), andllvm::slpvectorizer::BoUpSLP::vectorizeTree().

◆ buildIntrinsicArgTypes()

staticSmallVector<Type * > buildIntrinsicArgTypes	(	const CallInst *	CI,
		const Intrinsic::ID	ID,
		const unsigned	VF,
		unsigned	MinBW,
		const TargetTransformInfo *	TTI
	)

static

Builds the arguments types vector for the given call instruction with the givenID for the specified vector factor.

Definition at line11077 of fileSLPVectorizer.cpp.

Referencesllvm::CallBase::args(),llvm::enumerate(),llvm::IntegerType::get(),llvm::Value::getContext(),getWidenedType(),Idx,llvm::isVectorIntrinsicWithScalarOpAtArg(),llvm::Intrinsic::not_intrinsic, andllvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ buildUseMask()

staticSmallBitVector buildUseMask	(	int	VF,
		ArrayRef< int >	Mask,
		UseMask	MaskArg
	)

static

Prepares a use bitset for the given mask either for the first argument or for the second.

Definition at line616 of fileSLPVectorizer.cpp.

Referencesllvm::enumerate(),Idx, andllvm::PoisonMaskElem.

Referenced byperformExtractsShuffleAction().

◆ calculateRtStride()

static std::optional<Value * > calculateRtStride	(	ArrayRef<Value * >	PointerOps,
		Type *	ElemTy,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		SmallVectorImpl<unsigned > &	SortedIndices,
		Instruction *	Inst =`nullptr`
	)

static

Checks if the provided list of pointersPointers represents the strided pointers for type ElemTy.

If they are not, std::nullopt is returned. Otherwise, ifInst is not specified, just initialized optional value is returned to show that the pointers represent strided pointers. IfInst specified, the runtime stride is materialized before the givenInst.

Returns: std::nullopt if the pointers are not pointers with the runtime stride, nullptr or actual stride value, otherwise.

Definition at line4834 of fileSLPVectorizer.cpp.

Referencesllvm::SmallVectorImpl< T >::clear(),DL,llvm::SCEVExpander::expandCodeFor(),llvm::ScalarEvolution::getAddExpr(),llvm::ScalarEvolution::getConstant(),llvm::ScalarEvolution::getMinusSCEV(),llvm::ScalarEvolution::getMulExpr(),llvm::ScalarEvolution::getSCEV(),llvm::SCEV::getType(),llvm::ScalarEvolution::getUDivExactExpr(),llvm::SCEV::isNonConstantNegative(),llvm::SCEV::isZero(),Ptr,llvm::SmallVectorTemplateBase< T, bool >::push_back(),llvm::SmallVectorImpl< T >::resize(),llvm::ArrayRef< T >::size(),llvm::SmallVectorBase< Size_T >::size(), andSize.

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ calculateShufflevectorMask()

staticSmallVector< int > calculateShufflevectorMask

(

ArrayRef<Value * >

)

static

Returns: a shufflevector mask which is used to vectorize shufflevectors e.g., %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> the result is <0, 1, 2, 3, 12, 13, 14, 15, 16, 17, 18, 19, 28, 29, 30, 31>

Definition at line391 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::ArrayRef< T >::front(),getShufflevectorNumGroups(), andllvm::PoisonMaskElem.

◆ checkTreeSizes()

staticbool checkTreeSizes	(	ArrayRef< std::pair<unsigned,unsigned > >	Sizes,
		bool	First
	)

static

Checks if the quadratic mean deviation is less than 90% of the mean size.

Definition at line18678 of fileSLPVectorizer.cpp.

◆ clusterSortPtrAccesses()

staticbool clusterSortPtrAccesses	(	ArrayRef<Value * >	VL,
		ArrayRef<BasicBlock * >	BBs,
		Type *	ElemTy,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		SmallVectorImpl<unsigned > &	SortedIndices
	)

static

Definition at line5369 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),llvm::any_of(),assert(),llvm::sampleprof::Base,llvm::SmallVectorImpl< T >::clear(),llvm::SmallPtrSetImpl< PtrType >::contains(),llvm::Depth,llvm::ArrayRef< T >::drop_front(),llvm::enumerate(),llvm::MapVector< KeyT, ValueT, MapType, VectorType >::find(),llvm::MapVector< KeyT, ValueT, MapType, VectorType >::front(),llvm::ArrayRef< T >::front(),llvm::getUnderlyingObject(),llvm::SmallPtrSetImpl< PtrType >::insert(),llvm::isPointerTy(),P,Ptr,llvm::SmallVectorTemplateBase< T, bool >::push_back(),RecursionMaxDepth,llvm::ArrayRef< T >::size(),llvm::MapVector< KeyT, ValueT, MapType, VectorType >::size(),llvm::SmallVectorBase< Size_T >::size(),llvm::stable_sort(),llvm::MapVector< KeyT, ValueT, MapType, VectorType >::try_emplace(),X, andY.

Referenced byllvm::slpvectorizer::BoUpSLP::findPartiallyOrderedLoads().

◆ combineOrders()

static void combineOrders	(	MutableArrayRef<unsigned >	Order,
		ArrayRef<unsigned >	SecondaryOrder
	)

static

Definition at line5916 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::ArrayRef< T >::empty(),Idx,llvm::SmallBitVector::set(),llvm::ArrayRef< T >::size(), andllvm::SmallBitVector::test().

Referenced byllvm::slpvectorizer::BoUpSLP::reorderBottomToTop(), andllvm::slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ compareCmp()

template<bool IsCompatibility>

staticbool compareCmp	(	Value *	V,
		Value *	V2,
		TargetLibraryInfo &	TLI,
		const DominatorTree &	DT
	)

static

Compare two cmp instructions.

If IsCompatibility is true, function returns true if 2 cmps have same/swapped predicates and mos compatible corresponding operands. If IsCompatibility is false, function implements strict weak ordering relation between two cmp instructions, returning true if the first instruction is "less" than the second, i.e. its predicate is less than the predicate of the second or the operands IDs are less than the operands IDs of the second cmp instruction.

Definition at line21313 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::DomTreeNodeBase< NodeT >::getDFSNumIn(),llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(),llvm::User::getOperand(),getSameOpcode(),llvm::Type::getScalarSizeInBits(),llvm::CmpInst::getSwappedPredicate(),llvm::Value::getType(),llvm::Type::getTypeID(),llvm::Value::getValueID(),I, andisValidElementType().

◆ computeCommonAlignment()

template<typenameT >

staticAlign computeCommonAlignment

(

ArrayRef<Value * >

)

static

Calculates minimal alignment as a common alignment.

Definition at line4809 of fileSLPVectorizer.cpp.

Referencesllvm::ArrayRef< T >::drop_front(),llvm::ArrayRef< T >::front(), andllvm::getAlign().

◆ createExtractVector()

staticValue * createExtractVector	(	IRBuilderBase &	Builder,
		Value *	Vec,
		unsigned	SubVecVF,
		unsigned	Index
	)

static

Correctly creates extract_subvector, checking that the index is multiple of the subvectors length.

Otherwise, generates shuffle usingGenerator or using default shuffle.

Definition at line5010 of fileSLPVectorizer.cpp.

Referencesllvm::IRBuilderBase::CreateExtractVector(),llvm::IRBuilderBase::CreateShuffleVector(),llvm::IRBuilderBase::getInt64(),llvm::Type::getScalarType(),llvm::Value::getType(),getWidenedType(),Index, andllvm::PoisonMaskElem.

Referenced byllvm::slpvectorizer::BoUpSLP::vectorizeTree().

◆ createInsertVector()

staticValue * createInsertVector	(	IRBuilderBase &	Builder,
		Value *	Vec,
		Value *	V,
		unsigned	Index,
		function_ref<Value (Value ,Value *,ArrayRef< int >)>	Generator =`{}`
	)

static

Correctly creates insert_subvector, checking that the index is multiple of the subvectors length.

Otherwise, generates shuffle usingGenerator or using default shuffle.

Definition at line4979 of fileSLPVectorizer.cpp.

◆ DEBUG_COUNTER()

DEBUG_COUNTER	(	VectorizedGraphs	,
		"slp-vectorized"	,
		"Controls which SLP graphs should be vectorized."
	)

◆ doesInTreeUserNeedToExtract()

staticbool doesInTreeUserNeedToExtract	(	Value *	Scalar,
		Instruction *	UserInst,
		TargetLibraryInfo *	TLI,
		const TargetTransformInfo *	TTI
	)

static

Returns: True if in-tree use also needs extract. This refers to possible scalar operand in vectorized instruction.

Definition at line1100 of fileSLPVectorizer.cpp.

Referencesllvm::any_of(),llvm::CallBase::args(),llvm::enumerate(),llvm::Instruction::getOpcode(),llvm::LoadInst::getPointerOperand(), andllvm::getVectorIntrinsicIDForCall().

Referenced byllvm::slpvectorizer::BoUpSLP::buildExternalUses().

◆ dumpOrder()

staticLLVM_DUMP_METHOD void dumpOrder

(

const BoUpSLP::OrdersType &

Order

)

static

Definition at line6734 of fileSLPVectorizer.cpp.

Referencesllvm::dbgs(), andIdx.

◆ findBuildAggregate()

staticbool findBuildAggregate	(	Instruction *	LastInsertInst,
		TargetTransformInfo *	TTI,
		SmallVectorImpl<Value * > &	BuildVectorOpds,
		SmallVectorImpl<Value * > &	InsertElts,
		const BoUpSLP &	R
	)

static

Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0, i32 0 rb = insertelement <4 x float> ra, float s1, i32 1 rc = insertelement <4 x float> rb, float s2, i32 2 rd = insertelement <4 x float> rc, float s3, i32 3 starting from the last insertelement or insertvalue instruction.

Also recognize homogeneous aggregates like {<2 x float>, <2 x float>}, {{float, float}, {float, float}}, [2 x {float, float}] and so on. See llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll for examples.

Assume LastInsertInst is of InsertElementInst or InsertValueInst type.

Returns: true if it matches.

Definition at line20889 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::SmallVectorBase< Size_T >::empty(),llvm::erase(),findBuildAggregate_rec(),getAggregateSize(),llvm::SmallVectorImpl< T >::resize(), andllvm::SmallVectorBase< Size_T >::size().

◆ findBuildAggregate_rec()

static void findBuildAggregate_rec	(	Instruction *	LastInsertInst,
		TargetTransformInfo *	TTI,
		SmallVectorImpl<Value * > &	BuildVectorOpds,
		SmallVectorImpl<Value * > &	InsertElts,
		unsigned	OperandOffset,
		const BoUpSLP &	R
	)

static

Definition at line20850 of fileSLPVectorizer.cpp.

ReferencesfindBuildAggregate_rec(),getElementIndex(),llvm::User::getOperand(), andllvm::Value::hasOneUse().

Referenced byfindBuildAggregate(), andfindBuildAggregate_rec().

◆ fixupOrderingIndices()

static void fixupOrderingIndices

(

MutableArrayRef<unsigned >

Order

)

static

Order may have elements assigned special value (size) which is out of bounds.

Such indices only appear on places which correspond to undef values (see canReuseExtract for details) and used in order to avoid undef values have effect on operands ordering. The first loop below simply finds all unused indices and then the next loop nest assigns these indices for undef values positions. As an example below Order has two undef positions and they have assigned values 3 and 7 respectively: before: 6 9 5 4 9 2 1 0 after: 6 3 5 4 7 2 1 0

Definition at line1186 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::SmallBitVector::count(),llvm::SmallBitVector::find_first(),llvm::SmallBitVector::find_next(),I,Idx,llvm::SmallBitVector::none(),llvm::SmallBitVector::reset(),llvm::SmallBitVector::set(), andllvm::ArrayRef< T >::size().

Referenced byllvm::slpvectorizer::BoUpSLP::getReorderingData(),llvm::slpvectorizer::BoUpSLP::reorderBottomToTop(),reorderOrder(), andllvm::slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ gatherPossiblyVectorizableLoads()

static void gatherPossiblyVectorizableLoads	(	const BoUpSLP &	R,
		ArrayRef<Value * >	VL,
		const DataLayout &	DL,
		ScalarEvolution &	SE,
		const TargetTransformInfo &	TTI,
		SmallVectorImpl<SmallVector< std::pair<LoadInst *, int > > > &	GatheredLoads,
		bool	AddNew =`true`
	)

static

Tries to find subvector of loads and builds new vector of only loads if can be profitable.

Definition at line6789 of fileSLPVectorizer.cpp.

Referencesllvm::any_of(),assert(),llvm::ArrayRef< T >::begin(),llvm::bit_ceil(),llvm::SetVector< T, Vector, Set, N >::contains(),llvm::SmallSet< T, N, C >::contains(),llvm::SmallPtrSetImpl< PtrType >::contains(),llvm::Data,DL,llvm::SmallVectorImpl< T >::emplace_back(),llvm::ArrayRef< T >::empty(),llvm::SetVector< T, Vector, Set, N >::empty(),llvm::ArrayRef< T >::end(),llvm::enumerate(),llvm::find_if(),llvm::ArrayRef< T >::front(),llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(),llvm::LoadInst::getPointerOperand(),llvm::getPointersDiff(),llvm::Value::getType(),llvm::getUnderlyingObject(),getValueType(),llvm::has_single_bit(),Idx,llvm::SmallSet< T, N, C >::insert(),llvm::SetVector< T, Vector, Set, N >::insert(),llvm::SmallPtrSetImpl< PtrType >::insert(),isValidElementType(),llvm::Offset,P,RecursionMaxDepth,llvm::ArrayRef< T >::size(), andllvm::zip().

Referenced byllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ generateKeySubkey()

static std::pair< size_t, size_t > generateKeySubkey	(	Value *	V,
		const TargetLibraryInfo *	TLI,
		function_ref<hash_code(size_t,LoadInst *)>	LoadsSubkeyGenerator,
		bool	AllowAlternate
	)

static

Generates key/subkey pair for the given value to provide effective sorting of the values and better detection of the vectorizable values sequences.

The keys/subkeys can be used for better sorting of the values themselves (keys) and in values subgroups (subkeys).

Definition at line7417 of fileSLPVectorizer.cpp.

Referencesllvm::SmallBitVector::all(),generateKeySubkey(),llvm::CmpInst::getInversePredicate(),llvm::VFDatabase::getMappings(),llvm::CmpInst::getSwappedPredicate(),llvm::getVectorIntrinsicIDForCall(),llvm::hash_combine(),llvm::hash_value(),I,llvm::Instruction::isIntDivRem(),llvm::isTriviallyVectorizable(),isUndefVector(),isValidForAlternation(), andisVectorLikeInstWithConstOps().

Referenced bygenerateKeySubkey().

◆ getAggregateSize()

static std::optional<unsigned > getAggregateSize

(

Instruction *

InsertInst

)

static

Definition at line20822 of fileSLPVectorizer.cpp.

Referencesllvm::Type::isSingleValueType(), andIV.

Referenced byfindBuildAggregate().

◆ getAltInstrMask()

staticSmallBitVector getAltInstrMask	(	ArrayRef<Value * >	VL,
		unsigned	Opcode0,
		unsigned	Opcode1
	)

static

Returns: a bitset for selecting opcodes. false for Opcode0 and true for Opcode1.

Definition at line1212 of fileSLPVectorizer.cpp.

ReferencesgetNumElements(),getOpcode(),llvm::SmallBitVector::set(), andllvm::ArrayRef< T >::size().

Referenced byllvm::slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ getElementIndex()

static std::optional<unsigned > getElementIndex	(	const Value *	Inst,
		unsigned	Offset =`0`
	)

static

Returns: inserting or extracting index of InsertElement, ExtractElement or InsertValue instruction, using Offset as base offset for index.; std::nullopt if the index is not an immediate.

Definition at line568 of fileSLPVectorizer.cpp.

ReferencesI,IV, andllvm::Offset.

◆ getExtractIndex()

static std::optional<unsigned > getExtractIndex

(

Instruction *

)

static

Returns: True if Extract{Value,Element} instruction extracts element Idx.

Definition at line794 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::Instruction::getOpcode(), andllvm::User::getOperand().

Referenced byllvm::slpvectorizer::BoUpSLP::getReorderingData().

◆ getFloorFullVectorNumberOfElements()

staticunsigned getFloorFullVectorNumberOfElements	(	const TargetTransformInfo &	TTI,
		Type *	Ty,
		unsigned	Sz
	)

static

Returns the number of elements of the given typeTy, not greater thanSz, which forms type, which splits byTTI into whole vector types during legalization.

Definition at line286 of fileSLPVectorizer.cpp.

Referencesllvm::bit_ceil(),llvm::bit_floor(),llvm::divideCeil(),llvm::TargetTransformInfo::getNumberOfParts(),getWidenedType(), andisValidElementType().

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads(), andllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ getFullVectorNumberOfElements()

staticunsigned getFullVectorNumberOfElements	(	const TargetTransformInfo &	TTI,
		Type *	Ty,
		unsigned	Sz
	)

static

Returns the number of elements of the given typeTy, not less thanSz, which forms type, which splits byTTI into whole vector types during legalization.

Definition at line271 of fileSLPVectorizer.cpp.

Referencesllvm::bit_ceil(),llvm::divideCeil(),llvm::TargetTransformInfo::getNumberOfParts(),getWidenedType(), andisValidElementType().

◆ getGEPCosts()

static std::pair<InstructionCost,InstructionCost > getGEPCosts	(	const TargetTransformInfo &	TTI,
		ArrayRef<Value * >	Ptrs,
		Value *	BasePtr,
		unsigned	Opcode,
		TTI::TargetCostKind	CostKind,
		Type *	ScalarTy,
		VectorType *	VecTy
	)

static

Calculate the scalar and the vector costs from vectorizing set of GEPs.

Definition at line9521 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),CostKind,llvm::ArrayRef< T >::end(),llvm::find_if(),llvm::TargetTransformInfo::getGEPCost(),llvm::TargetTransformInfo::getPointersChainCost(),Ptr,llvm::SmallVectorTemplateBase< T, bool >::push_back(),llvm::SmallVectorBase< Size_T >::size(), andllvm::TargetTransformInfo::TCC_Free.

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ getInsertExtractIndex()

template<typenameT >

static std::optional<unsigned > getInsertExtractIndex	(	const Value *	Inst,
		unsigned	Offset
	)

static

Definition at line543 of fileSLPVectorizer.cpp.

Referencesllvm::Offset.

◆ getLocation()

staticMemoryLocation getLocation

(

Instruction *

)

static

Returns: the AA location that is being access by the instruction.

Definition at line1129 of fileSLPVectorizer.cpp.

Referencesllvm::MemoryLocation::get(), andI.

◆ getNonPhiOperand()

staticInstruction * getNonPhiOperand	(	Instruction *	I,
		PHINode *	Phi
	)

static

Returns the first operand ofI that does not matchPhi.

If operand is not an instruction it returns nullptr.

Definition at line21013 of fileSLPVectorizer.cpp.

ReferencesI, andmatchRdxBop().

◆ getNumElements()

staticunsigned getNumElements

(

Type *

)

static

Returns: the number of elements for Ty.

Definition at line254 of fileSLPVectorizer.cpp.

Referencesassert().

Referenced byllvm::MachineIRBuilder::buildSplatBuildVector(),cacheDIVar(),llvm::slpvectorizer::BoUpSLP::canVectorizeLoads(),llvm::slpvectorizer::BoUpSLP::computeMinimumValueSizes(),containsUndefinedElement(),CreateGCRelocates(),expandAbs(),expandCrossIntrinsic(),expandExpIntrinsic(),expandLogIntrinsic(),extractVector(),getAltInstrMask(),llvm::getNumberOfParts(),llvm::X86TTIImpl::getScalarizationOverhead(),llvm::BasicTTIImplBase< T >::getTypeBasedIntrinsicInstrCost(),llvm::SystemZTTIImpl::getVectorTruncCost(),getWidenedType(),insertVector(),llvm::X86TTIImpl::isLegalMaskedExpandLoad(),llvm::X86TTIImpl::isLegalMaskedLoad(),llvm::ShuffleVectorInst::isValidOperands(),isVectorPromotionViableForSlice(),isZero(),llvm::RewriteStatepointsForGC::runOnFunction(),llvm::InstCombinerImpl::SimplifyDemandedVectorElts(),simplifyX86pack(), andupgradeX86IntrinsicCall().

◆ getNumElems()

staticunsigned getNumElems	(	unsigned	Size,
		unsigned	PartNumElems,
		unsigned	Part
	)

static

Returns correct remaining number of elements, considering total amountSize, (power-of-2 number) of elements in a single registerPartNumElems and current register (part)Part.

Definition at line442 of fileSLPVectorizer.cpp.

ReferencesSize.

Referenced byllvm::slpvectorizer::BoUpSLP::findReusedOrderedScalars().

◆ getPartNumElems()

staticunsigned getPartNumElems	(	unsigned	Size,
		unsigned	NumParts
	)

static

Returns power-of-2 number of elements in a single register (part), given the total number of elementsSize and number of registers (parts)NumParts.

Definition at line435 of fileSLPVectorizer.cpp.

Referencesllvm::bit_ceil(),llvm::divideCeil(), andSize.

Referenced byllvm::slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), andllvm::slpvectorizer::BoUpSLP::findReusedOrderedScalars().

◆ getRdxKind()

staticRecurKind getRdxKind

(

Value *

)

static

Gets recurrence kind from the specified value.

Definition at line20819 of fileSLPVectorizer.cpp.

Referenced byllvm::slpvectorizer::BoUpSLP::computeMinimumValueSizes().

◆ getReductionInstr()

staticInstruction * getReductionInstr	(	const DominatorTree *	DT,
		PHINode *	P,
		BasicBlock *	ParentBB,
		LoopInfo *	LI
	)

static

Try and get a reduction instruction from a phi node.

Given a phi nodeP in a blockParentBB, consider possible reductions if they come from eitherParentBB or a containing loop latch.

Returns: A candidate reduction value if possible, or
nullptr
if not possible.

Definition at line20925 of fileSLPVectorizer.cpp.

Referencesllvm::DominatorTree::dominates(),llvm::LoopInfoBase< BlockT, LoopT >::getLoopFor(),llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), andP.

◆ getSameOpcode()

static InstructionsState getSameOpcode	(	ArrayRef<Value * >	VL,
		const TargetLibraryInfo &	TLI
	)

static

Returns: analysis of the Instructions inVL described in InstructionsState, the Opcode that we suppose the whole list could be vectorized even if its structure is diverse.

Definition at line909 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),assert(),llvm::CmpInst::BAD_ICMP_PREDICATE,llvm::SetVector< T, Vector, Set, N >::contains(),llvm::SmallVectorBase< Size_T >::empty(),llvm::ArrayRef< T >::end(),llvm::find_if(),llvm::SmallVectorTemplateCommon< T, typename >::front(),llvm::CallBase::getBundleOperandsStartIndex(),llvm::CallBase::getCalledFunction(),llvm::VFDatabase::getMappings(),llvm::Instruction::getOpcode(),llvm::User::getOperand(),llvm::CmpInst::getSwappedPredicate(),llvm::Value::getType(),llvm::getVectorIntrinsicIDForCall(),llvm::CallBase::hasOperandBundles(),I,llvm::SetVector< T, Vector, Set, N >::insert(),isCmpSameOrSwapped(),llvm::isTriviallyVectorizable(),isValidForAlternation(),isVectorLikeInstWithConstOps(),llvm::User::op_begin(),llvm::ArrayRef< T >::size(),llvm::SetVector< T, Vector, Set, N >::size(), andllvm::SmallVectorBase< Size_T >::size().

Referenced byareCompatibleCmpOps(),arePointersCompatible(),compareCmp(),llvm::slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(),llvm::slpvectorizer::BoUpSLP::VLOperands::reorder(), andllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ getShuffleCost()

staticInstructionCost getShuffleCost	(	const TargetTransformInfo &	TTI,
		TTI::ShuffleKind	Kind,
		VectorType *	Tp,
		ArrayRef< int >	Mask =`{}`,
		TTI::TargetCostKind	CostKind =`TTI::TCK_RecipThroughput`,
		int	Index =`0`,
		VectorType *	SubTp =`nullptr`,
		ArrayRef<const Value * >	Args =`{}`
	)

static

Returns the cost of the shuffle instructions with the givenKind, vector typeTp and optionalMask.

Adds SLP-specifc cost estimation for insert subvector pattern.

Definition at line4955 of fileSLPVectorizer.cpp.

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads(),llvm::slpvectorizer::BoUpSLP::getReorderingData(), andllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ getShufflevectorNumGroups()

staticunsigned getShufflevectorNumGroups

(

ArrayRef<Value * >

)

static

Returns

the number of groups of shufflevector A group has the following features

All of value in a group are shufflevector.
The mask of all shufflevector is isExtractSubvectorMask.
The mask of all shufflevector uses all of the elements of the source. e.g., it is 1 group (%0) %1 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7> %2 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15> it is 2 groups (%3 and %4) %5 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %6 = shufflevector <8 x i16> %3, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> %7 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %8 = shufflevector <8 x i16> %4, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7> it is 0 group %12 = shufflevector <8 x i16> %10, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3> %13 = shufflevector <8 x i16> %11, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>

Definition at line339 of fileSLPVectorizer.cpp.

Referencesllvm::SmallBitVector::all(),llvm::all_of(),assert(),llvm::ArrayRef< T >::empty(),llvm::ArrayRef< T >::front(),I,llvm::SmallBitVector::set(),llvm::ArrayRef< T >::size(), andllvm::ArrayRef< T >::slice().

Referenced bycalculateShufflevectorMask().

◆ getValueType()

staticType * getValueType

(

Value *

)

static

Returns the type of the given value/instructionV.

If it is store, returns the type of its value operand, for Cmp - the types of the compare operands and for insertelement - the type os the inserted operand. Otherwise, just the type of the value is returned.

Definition at line243 of fileSLPVectorizer.cpp.

Referenced byCollectOpsToWiden(),combineArithReduction(),combineConcatVectorOps(),combineMulToPMULDQ(),combineTargetShuffle(),combineToHorizontalAddSub(),CompactSwizzlableVector(),detectPMADDUBSW(),FoldBUILD_VECTOR(),foldCONCAT_VECTORS(),gatherPossiblyVectorizableLoads(),llvm::Function::getFunctionType(),llvm::RISCVTTIImpl::getIntImmCostInst(),getTestBitOperand(),LLVMGlobalGetValueType(),matchPMADDWD(),matchPMADDWD_2(),narrowVectorSelect(),performCONCAT_VECTORSCombine(), andllvm::SelectionDAGISel::SelectInlineAsmMemoryOperands().

◆ getVectorCallCosts()

static std::pair<InstructionCost,InstructionCost > getVectorCallCosts	(	CallInst *	CI,
		FixedVectorType *	VecTy,
		TargetTransformInfo *	TTI,
		TargetLibraryInfo *	TLI,
		ArrayRef<Type * >	ArgTys
	)

static

Definition at line9039 of fileSLPVectorizer.cpp.

Referencesllvm::VFShape::get(),llvm::TargetTransformInfo::getCallInstrCost(),llvm::ElementCount::getFixed(),llvm::CallBase::getFunctionType(),llvm::TargetTransformInfo::getIntrinsicInstrCost(),llvm::FixedVectorType::getNumElements(),llvm::getVectorIntrinsicIDForCall(),llvm::VFDatabase::getVectorizedFunction(),llvm::CallBase::isNoBuiltin(), andllvm::TargetTransformInfo::TCK_RecipThroughput.

◆ getWidenedType()

staticFixedVectorType * getWidenedType	(	Type *	ScalarTy,
		unsigned	VF
	)

static

Returns: the vector type of ScalarTy based on vectorization factor.

Definition at line263 of fileSLPVectorizer.cpp.

Referencesllvm::FixedVectorType::get(),getNumElements(), andllvm::Type::getScalarType().

◆ isAlternateInstruction()

staticbool isAlternateInstruction	(	const Instruction *	I,
		const Instruction *	MainOp,
		const Instruction *	AltOp,
		const TargetLibraryInfo &	TLI
	)

static

Checks if the specified instructionI is an alternate operation for the givenMainOp andAltOp instructions.

Definition at line9101 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::Instruction::getOpcode(),llvm::CmpInst::getSwappedPredicate(),I,isCmpSameOrSwapped(), andP.

◆ isCmpSameOrSwapped()

staticbool isCmpSameOrSwapped	(	const CmpInst *	BaseCI,
		const CmpInst *	CI,
		const TargetLibraryInfo &	TLI
	)

static

Returns: true if a compare instructionCI has similar "look" and same predicate asBaseCI, "as is" or with its operands and predicate swapped, false otherwise.

Definition at line887 of fileSLPVectorizer.cpp.

ReferencesareCompatibleCmpOps(),assert(),llvm::User::getOperand(),llvm::CmpInst::getPredicate(),llvm::CmpInst::getSwappedPredicate(), andllvm::Value::getType().

Referenced bygetSameOpcode(), andisAlternateInstruction().

◆ isCommutative()

staticbool isCommutative

(

Instruction *

)

static

Returns: True ifI is commutative, handles CmpInst and BinaryOperator.

Definition at line509 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),I, andUsesLimit.

Referenced byllvm::slpvectorizer::BoUpSLP::LookAheadHeuristics::getScoreAtLevelRec(), andllvm::FastISel::selectBinaryOp().

◆ isConstant()

staticbool isConstant

(

Value *

)

static

Returns: True if the value is a constant (but not globals/constant expressions).

Definition at line410 of fileSLPVectorizer.cpp.

Referenced byallConstant(),areCompatibleCmpOps(), andisVectorLikeInstWithConstOps().

◆ isFirstInsertElement()

staticbool isFirstInsertElement	(	const InsertElementInst *	IE1,
		const InsertElementInst *	IE2
	)

static

Checks if theIE1 instructions is followed byIE2 instruction in the buildvector sequence.

Definition at line12300 of fileSLPVectorizer.cpp.

ReferencesgetElementIndex(),llvm::Value::hasOneUse(), andllvm_unreachable.

Referenced byllvm::slpvectorizer::BoUpSLP::getTreeCost(), andllvm::slpvectorizer::BoUpSLP::vectorizeTree().

◆ isFixedVectorShuffle()

static std::optional<TargetTransformInfo::ShuffleKind > isFixedVectorShuffle	(	ArrayRef<Value * >	VL,
		SmallVectorImpl< int > &	Mask,
		AssumptionCache *	AC
	)

static

Checks if the vector of instructions can be represented as a shuffle, like: x0 = extractelement <4 x i8> x, i32 0 x3 = extractelement <4 x i8> x, i32 3 y1 = extractelement <4 x i8> y, i32 1 y2 = extractelement <4 x i8> y, i32 2 x0x0 = mul i8 x0, x0 x3x3 = mul i8 x3, x3 y1y1 = mul i8 y1, y1 y2y2 = mul i8 y2, y2 ins1 = insertelement <4 x i8> poison, i8 x0x0, i32 0 ins2 = insertelement <4 x i8> ins1, i8 x3x3, i32 1 ins3 = insertelement <4 x i8> ins2, i8 y1y1, i32 2 ins4 = insertelement <4 x i8> ins3, i8 y2y2, i32 3 ret <4 x i8> ins4 can be transformed into: %1 = shufflevector <4 x i8> x, <4 x i8> y, <4 x i32> <i32 0, i32 3, i32 5, i32 6> %2 = mul <4 x i8> %1, %1 ret <4 x i8> %2 Mask will return the Shuffle Mask equivalent to the extracted elements.

TODO: Can we split off and reuse the shuffle mask detection from ShuffleVectorInst/getShuffleCost?

Definition at line706 of fileSLPVectorizer.cpp.

Referencesllvm::any_of(),llvm::ArrayRef< T >::begin(),llvm::ArrayRef< T >::end(),llvm::find_if(),I,Idx,llvm::isGuaranteedNotToBePoison(),isUndefVector(),llvm::PoisonMaskElem,Select,llvm::ArrayRef< T >::size(),Size,llvm::TargetTransformInfo::SK_PermuteSingleSrc,llvm::TargetTransformInfo::SK_PermuteTwoSrc,llvm::TargetTransformInfo::SK_Select, andUnknown.

◆ isLoadCombineCandidateImpl()

staticbool isLoadCombineCandidateImpl	(	Value *	Root,
		unsigned	NumElts,
		TargetTransformInfo *	TTI,
		bool	MustMatchOrInst
	)

static

Definition at line12012 of fileSLPVectorizer.cpp.

Referencesllvm::dbgs(),llvm::IntegerType::get(),llvm::Value::getContext(),llvm::Type::getIntegerBitWidth(),llvm::TargetTransformInfo::isTypeLegal(),LLVM_DEBUG,llvm::PatternMatch::m_APInt(),llvm::PatternMatch::m_Or(),llvm::PatternMatch::m_Shl(),llvm::PatternMatch::m_Value(),llvm::PatternMatch::m_ZExt(),llvm::PatternMatch::match(), andllvm::APInt::urem().

Referenced byllvm::slpvectorizer::BoUpSLP::isLoadCombineCandidate(), andllvm::slpvectorizer::BoUpSLP::isLoadCombineReductionCandidate().

◆ isReductionCandidate()

staticbool isReductionCandidate

(

Instruction *

)

static

\Returns true ifI is a candidate instruction for reduction vectorization.

Definition at line21022 of fileSLPVectorizer.cpp.

ReferencesI,IsSelect(),llvm::PatternMatch::m_Select(),llvm::PatternMatch::m_Value(),llvm::PatternMatch::match(), andmatchRdxBop().

◆ isRepeatedNonIdentityClusteredMask()

staticbool isRepeatedNonIdentityClusteredMask	(	ArrayRef< int >	Mask,
		unsigned	Sz
	)

static

Checks if the given mask is a "clustered" mask with the same clusters of sizeSz, which are not identity submasks.

Definition at line5876 of fileSLPVectorizer.cpp.

ReferencesE,I, andllvm::ShuffleVectorInst::isIdentityMask().

◆ isReverseOrder()

staticbool isReverseOrder

(

ArrayRef<unsigned >

Order

)

static

Check ifOrder represents reverse order.

Definition at line4817 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),assert(),llvm::ArrayRef< T >::empty(),llvm::enumerate(), andllvm::ArrayRef< T >::size().

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads(),llvm::slpvectorizer::BoUpSLP::getReorderingData(), andllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ isSimple()

staticbool isSimple

(

Instruction *

)

static

Returns: True if the instruction is not a volatile or atomic load/store.

Definition at line1138 of fileSLPVectorizer.cpp.

ReferencesI, andMI.

Referenced byllvm::DOTGraphTraits< DOTFuncInfo * >::getNodeLabel(),llvm::DOTGraphTraits< MachineBlockFrequencyInfo * >::getNodeLabel(),llvm::DOTGraphTraits< const MachineFunction * >::getNodeLabel(),llvm::DOTGraphTraits< DOTMachineFuncInfo * >::getNodeLabel(),llvm::DOTGraphTraits< DomTreeNode * >::getNodeLabel(),llvm::DOTGraphTraits< RegionNode * >::getNodeLabel(),llvm::X86TargetLowering::shouldReduceLoadWidth(),tryToFoldExtOfMaskedLoad(),llvm::MBFIWrapper::view(), andllvm::MachineBlockFrequencyInfo::view().

◆ isSplat()

staticbool isSplat

(

ArrayRef<Value * >

)

static

Returns: True if all of the values inVL are identical or some of them are UndefValue.

Definition at line493 of fileSLPVectorizer.cpp.

◆ isUndefVector()

template<bool IsPoisonOnly = false>

staticSmallBitVector isUndefVector	(	const Value *	V,
		const SmallBitVector &	UseMask =`{}`
	)

static

Checks if the given value is actually an undefined constant vector.

Also, if theUseMask is not empty, tries to check if the non-masked elements actually mask the insertelement buildvector, if any.

Definition at line637 of fileSLPVectorizer.cpp.

Referenced bygenerateKeySubkey(),llvm::slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(),isFixedVectorShuffle(), andperformExtractsShuffleAction().

◆ isValidElementType()

staticbool isValidElementType

(

Type *

)

static

Predicate for the element types that the SLP vectorizer supports.

The most important thing to filter here are types which are invalid in LLVM vectors. We also filter target specific types which have absolutely no meaningful vectorization path such as x86_fp80 and ppc_f128. This just avoids spending time checking the cost model and realizing that they will be inevitably scalarized.

Definition at line231 of fileSLPVectorizer.cpp.

Referencesllvm::Type::getScalarType(),llvm::Type::isPPC_FP128Ty(),llvm::Type::isX86_FP80Ty(), andSLPReVec.

Referenced byllvm::slpvectorizer::BoUpSLP::canMapToVector(),compareCmp(),llvm::slpvectorizer::BoUpSLP::findReusedOrderedScalars(),gatherPossiblyVectorizableLoads(),getFloorFullVectorNumberOfElements(),getFullVectorNumberOfElements(),llvm::slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(), andllvm::hasFullVectorsOrPowerOf2().

◆ isValidForAlternation()

staticbool isValidForAlternation

(

unsigned

Opcode

)

static

Returns: true ifOpcode is allowed as part of the main/alternate instruction for SLP vectorization.

Example of unsupported opcode is SDIV that can potentially cause UB if the "shuffled out" lane would result in division by zero.

Definition at line861 of fileSLPVectorizer.cpp.

Referencesllvm::Instruction::isIntDivRem().

Referenced bygenerateKeySubkey(), andgetSameOpcode().

◆ isVectorLikeInstWithConstOps()

staticbool isVectorLikeInstWithConstOps

(

Value *

)

static

Checks ifV is one of vector-like instructions, i.e.

undef, insertelement/extractelement with constant indices for fixed vector type or extractvalue instruction.

Definition at line417 of fileSLPVectorizer.cpp.

Referencesassert(),I, andisConstant().

Referenced byallSameBlock(),generateKeySubkey(), andgetSameOpcode().

◆ matchRdxBop()

staticbool matchRdxBop	(	Instruction *	I,
		Value *&	V0,
		Value *&	V1
	)

static

Definition at line20969 of fileSLPVectorizer.cpp.

ReferencesI,llvm::PatternMatch::m_BinOp(),llvm::PatternMatch::m_Value(), andllvm::PatternMatch::match().

Referenced bygetNonPhiOperand(), andisReductionCandidate().

◆ needToScheduleSingleInstruction()

staticbool needToScheduleSingleInstruction

(

ArrayRef<Value * >

)

static

Returns: true if the specified list of values has only one instruction that requires scheduling, false otherwise.

Definition at line7398 of fileSLPVectorizer.cpp.

Referencesllvm::doesNotNeedToBeScheduled().

◆ performExtractsShuffleAction()

template<typenameT >

staticT * performExtractsShuffleAction	(	MutableArrayRef< std::pair<T *,SmallVector< int > > >	ShuffleMask,
		Value *	Base,
		function_ref<unsigned(T *)>	GetVF,
		function_ref< std::pair<T ,bool >(T ,ArrayRef< int >,bool)>	ResizeAction,
		function_ref<T (ArrayRef< int >,ArrayRef<T >)>	Action
	)

static

Does the analysis of the provided shuffle masks and performs the requested actions on the vectors with the given shuffle masks.

It tries to do it in several steps.

If the Base vector is not undef vector, resizing the very first mask to have common VF and perform action for 2 input vectors (including non-undef Base). Other shuffle masks are combined with the resulting after the 1 stage and processed as a shuffle of 2 elements.
If the Base is undef vector and have only 1 shuffle mask, perform the action only for 1 vector with the given mask, if it is not the identity mask.
If > 2 masks are used, perform the remaining shuffle actions for 2 vectors, combing the masks properly between the steps.

Definition at line12355 of fileSLPVectorizer.cpp.

Referencesllvm::SmallBitVector::all(),assert(),llvm::sampleprof::Base,buildUseMask(),I,Idx,isUndefVector(),llvm::PoisonMaskElem, andllvm::SmallBitVector::test().

◆ propagateMetadata()

staticInstruction * propagateMetadata	(	Instruction *	Inst,
		ArrayRef<Value * >	VL
	)

static

Returns: I after propagating metadata fromVL only for instructions inVL.

Definition at line15260 of fileSLPVectorizer.cpp.

Referencesllvm::propagateMetadata(), andllvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ reorderOrder()

static void reorderOrder	(	SmallVectorImpl<unsigned > &	Order,
		ArrayRef< int >	Mask,
		bool	BottomOrder =`false`
	)

static

Reorders the givenOrder according to the givenMask.

Order - is the original order of the scalars. Procedure transforms the provided order in accordance with the givenMask. If the resultingOrder is just an identity order,Order is cleared.

Definition at line4582 of fileSLPVectorizer.cpp.

Referencesllvm::all_of(),assert(),llvm::SmallVectorImpl< T >::assign(),llvm::SmallVectorTemplateCommon< T, typename >::begin(),llvm::SmallVectorImpl< T >::clear(),llvm::Data,llvm::SmallVectorBase< Size_T >::empty(),llvm::SmallVectorTemplateCommon< T, typename >::end(),llvm::enumerate(),fixupOrderingIndices(),I,llvm::inversePermutation(),llvm::ShuffleVectorInst::isIdentityMask(),llvm::PoisonMaskElem,reorderReuses(),llvm::SmallVectorImpl< T >::resize(), andllvm::SmallVectorImpl< T >::swap().

Referenced byllvm::slpvectorizer::BoUpSLP::getReorderingData(),llvm::slpvectorizer::BoUpSLP::reorderBottomToTop(), andllvm::slpvectorizer::BoUpSLP::reorderTopToBottom().

◆ reorderReuses()

static void reorderReuses	(	SmallVectorImpl< int > &	Reuses,
		ArrayRef< int >	Mask
	)

static

Reorders the givenReuses mask according to the givenMask.

Reuses contains original mask for the scalars reused in the node. Procedure transform this mask in accordance with the givenMask.

Definition at line4568 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::SmallVectorTemplateCommon< T, typename >::begin(),E,llvm::SmallVectorTemplateCommon< T, typename >::end(),I,llvm::PoisonMaskElem,llvm::SmallVectorBase< Size_T >::size(), andllvm::SmallVectorImpl< T >::swap().

Referenced byllvm::slpvectorizer::BoUpSLP::reorderBottomToTop(), andreorderOrder().

◆ shortBundleName()

static std::string shortBundleName	(	ArrayRef<Value * >	VL,
		int	Idx =`-1`
	)

static

Print a short descriptor of the instruction bundle suitable for debug output.

Definition at line449 of fileSLPVectorizer.cpp.

Referencesllvm::ArrayRef< T >::front(),Idx,OS, andllvm::ArrayRef< T >::size().

Referenced byllvm::slpvectorizer::BoUpSLP::getTreeCost().

◆ STATISTIC()

STATISTIC	(	NumVectorInstructions	,
		"Number ofvector instructions generated"
	)

◆ transformScalarShuffleIndiciesToVector()

static void transformScalarShuffleIndiciesToVector	(	unsigned	VecTyNumElements,
		SmallVectorImpl< int > &	Mask
	)

static

Definition at line300 of fileSLPVectorizer.cpp.

Referencesllvm::enumerate(),I, andllvm::PoisonMaskElem.

◆ tryGetSecondaryReductionRoot()

staticInstruction * tryGetSecondaryReductionRoot	(	PHINode *	Phi,
		Instruction *	Root
	)

static

We could have an initial reduction that is not an add.

r *= v1 + v2 + v3 + v4 In such a case start looking for a tree rooted in the first '+'. \Returns the new root if found, which may be nullptr if not an instruction.

Definition at line20995 of fileSLPVectorizer.cpp.

Referencesassert(),llvm::User::getOperand(),LHS, andRHS.

◆ tryToVectorizeSequence()

template<typenameT >

staticbool tryToVectorizeSequence	(	SmallVectorImpl<T * > &	Incoming,
		function_ref<bool(T ,T )>	Comparator,
		function_ref<bool(T ,T )>	AreCompatible,
		function_ref<bool(ArrayRef<T * >,bool)>	TryToVectorizeHelper,
		bool	MaxVFOnly,
		BoUpSLP &	R
	)

static

\Returns the minimum number of elements that we will attempt to vectorize.

Definition at line21194 of fileSLPVectorizer.cpp.

Referencesllvm::SmallVectorTemplateCommon< T, typename >::begin(),llvm::SmallVectorImpl< T >::clear(),llvm::dbgs(),llvm::SmallVectorBase< Size_T >::empty(),llvm::SmallVectorTemplateCommon< T, typename >::end(),End,llvm::SmallVectorTemplateCommon< T, typename >::front(),I,LLVM_DEBUG,llvm::SmallVectorTemplateBase< T, bool >::push_back(),llvm::SmallVectorBase< Size_T >::size(),llvm::stable_sort(), andllvm::SmallVectorImpl< T >::swap().

Variable Documentation

◆ AliasedCheckLimit

const unsigned AliasedCheckLimit = 10

static

Definition at line206 of fileSLPVectorizer.cpp.

◆ LookAheadMaxDepth

cl::opt< int > LookAheadMaxDepth("slp-max-look-ahead-depth", cl::init(2), cl::Hidden,cl::desc("The maximum look-ahead depthfor operand reordering scores"))	(	"slp-max-look-ahead-depth"	,
		cl::init(2)	,
		cl::Hidden	,
		cl::desc("The maximum look-ahead depthfor operand reordering scores")
	)

static

◆ MaxMemDepDistance

const unsigned MaxMemDepDistance = 160

static

Definition at line215 of fileSLPVectorizer.cpp.

◆ MaxPHINumOperands

const unsigned MaxPHINumOperands = 128

static

Maximum allowed number of operands in the PHI nodes.

Definition at line222 of fileSLPVectorizer.cpp.

◆ MaxProfitableLoadStride

cl::opt<unsigned > MaxProfitableLoadStride("slp-max-stride", cl::init(8), cl::Hidden,cl::desc("The maximum stride, considered to be profitable."))	(	"slp-max-stride"	,
		cl::init(8)	,
		cl::Hidden	,
		cl::desc("The maximum stride, considered to be profitable.")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ MaxVectorRegSizeOption

cl::opt< int > MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,cl::desc("Attempt to vectorizefor this register size in bits"))	(	"slp-max-reg-size"	,
		cl::init(128)	,
		cl::Hidden	,
		cl::desc("Attempt to vectorizefor this register size in bits")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::BoUpSLP().

◆ MaxVFOption

cl::opt<unsigned > MaxVFOption("slp-max-vf", cl::init(0), cl::Hidden,cl::desc("Maximum SLP vectorization factor (0=unlimited)"))	(	"slp-max-vf"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Maximum SLP vectorization factor (0=unlimited)")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::getMaximumVF().

◆ MinProfitableStridedLoads

cl::opt<unsigned > MinProfitableStridedLoads("slp-min-strided-loads", cl::init(2), cl::Hidden,cl::desc("The minimum number ofloads, which should be considered strided, " "if the stride is > 1 or is runtime value"))	(	"slp-min-strided-loads"	,
		cl::init(2)	,
		cl::Hidden	,
		cl::desc("The minimum number ofloads, which should be considered strided, " "if the stride is > 1 or is runtime value")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads().

◆ MinScheduleRegionSize

const int MinScheduleRegionSize = 16

static

If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.

Definition at line219 of fileSLPVectorizer.cpp.

◆ MinTreeSize

cl::opt<unsigned > MinTreeSize("slp-min-tree-size", cl::init(3), cl::Hidden,cl::desc("Only vectorize small treesif they are fully vectorizable"))	(	"slp-min-tree-size"	,
		cl::init(3)	,
		cl::Hidden	,
		cl::desc("Only vectorize small treesif they are fully vectorizable")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ MinVectorRegSizeOption

cl::opt< int > MinVectorRegSizeOption("slp-min-reg-size", cl::init(128), cl::Hidden,cl::desc("Attempt to vectorizefor this register size in bits"))	(	"slp-min-reg-size"	,
		cl::init(128)	,
		cl::Hidden	,
		cl::desc("Attempt to vectorizefor this register size in bits")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::BoUpSLP().

◆ RecursionMaxDepth

cl::opt<unsigned > RecursionMaxDepth("slp-recursion-max-depth", cl::init(12), cl::Hidden,cl::desc("Limit therecursion depth when building a vectorizable tree"))	(	"slp-recursion-max-depth"	,
		cl::init(12)	,
		cl::Hidden	,
		cl::desc("Limit therecursion depth when building a vectorizable tree")
	)

static

Referenced byarePointersCompatible(),clusterSortPtrAccesses(),gatherPossiblyVectorizableLoads(),llvm::slpvectorizer::BoUpSLP::getVectorElementSize(),llvm::slpvectorizer::BoUpSLP::reorderTopToBottom(), andllvm::slpvectorizer::BoUpSLP::transformNodes().

◆ RootLookAheadMaxDepth

cl::opt< int > RootLookAheadMaxDepth("slp-max-root-look-ahead-depth", cl::init(2), cl::Hidden,cl::desc("The maximum look-ahead depthfor searching best rooting option"))	(	"slp-max-root-look-ahead-depth"	,
		cl::init(2)	,
		cl::Hidden	,
		cl::desc("The maximum look-ahead depthfor searching best rooting option")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::findBestRootPair().

◆ RunSLPVectorization

cl::opt<bool > RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden,cl::desc("Run the SLP vectorization passes"))	(	"vectorize-slp"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Run the SLP vectorization passes")
	)

static

Referenced byllvm::SLPVectorizerPass::runImpl().

◆ ScheduleRegionSizeBudget

cl::opt< int > ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,cl::desc("Limit the size of the SLP schedulingregion per block"))	(	"slp-schedule-budget"	,
		cl::init(100000)	,
		cl::Hidden	,
		cl::desc("Limit the size of the SLP schedulingregion per block")
	)

static

Limits the size of scheduling regions in a block.

It avoid long compile times forvery large blocks where vector instructions are spread over a wide range. This limit is way higher than needed by real-world functions.

◆ ShouldStartVectorizeHorAtStore

cl::opt<bool > ShouldStartVectorizeHorAtStore("slp-vectorize-hor-store", cl::init(false), cl::Hidden,cl::desc( "Attempt to vectorize horizontalreductions feeding into a store"))	(	"slp-vectorize-hor-store"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc( "Attempt to vectorize horizontalreductions feeding into a store")
	)

static

◆ ShouldVectorizeHor

cl::opt<bool > ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,cl::desc("Attempt to vectorize horizontal reductions"))	(	"slp-vectorize-hor"	,
		cl::init(true)	,
		cl::Hidden	,
		cl::desc("Attempt to vectorize horizontal reductions")
	)

static

◆ SLPCostThreshold

cl::opt< int > SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,cl::desc("Only vectorizeif you gain more than this " "number "))	(	"slp-threshold"	,
		cl::init(0)	,
		cl::Hidden	,
		cl::desc("Only vectorizeif you gain more than this " "number ")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::canVectorizeLoads(), andllvm::slpvectorizer::BoUpSLP::isTreeTinyAndNotFullyVectorizable().

◆ SLPReVec

cl::opt<bool > SLPReVec("slp-revec", cl::init(false), cl::Hidden,cl::desc("Enable vectorizationfor widervector utilization"))	(	"slp-revec"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Enable vectorizationfor widervector utilization")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::ShuffleCostEstimator::gather(),isValidElementType(),llvm::slpvectorizer::BoUpSLP::reorderTopToBottom(), andllvm::slpvectorizer::BoUpSLP::vectorizeTree().

◆ SLPSkipEarlyProfitabilityCheck

cl::opt<bool > SLPSkipEarlyProfitabilityCheck("slp-skip-early-profitability-check", cl::init(false), cl::Hidden,cl::desc("Whentrue, SLP vectorizer bypasses profitabilitychecks based on " "heuristics and makes vectorization decision via cost modeling."))	(	"slp-skip-early-profitability-check"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Whentrue, SLP vectorizer bypasses profitabilitychecks based on " "heuristics and makes vectorization decision via cost modeling.")
	)

static

◆ UsesLimit

constexpr int UsesLimit = 64

staticconstexpr

Definition at line210 of fileSLPVectorizer.cpp.

Referenced byllvm::slpvectorizer::BoUpSLP::buildExternalUses(),llvm::slpvectorizer::BoUpSLP::computeMinimumValueSizes(),llvm::slpvectorizer::BoUpSLP::LookAheadHeuristics::getShallowScore(),llvm::slpvectorizer::BoUpSLP::getTreeCost(),isCommutative(),llvm::isUsedOutsideBlock(), andllvm::slpvectorizer::BoUpSLP::vectorizeTree().

◆ VectorizeNonPowerOf2

cl::opt<bool > VectorizeNonPowerOf2("slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden,cl::desc("Try to vectorize with non-power-of-2 number of elements."))	(	"slp-vectorize-non-power-of-2"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Try to vectorize with non-power-of-2 number of elements.")
	)

static

Referenced byllvm::slpvectorizer::BoUpSLP::getReorderingData().

◆ ViewSLPTree

cl::opt<bool > ViewSLPTree("view-slp-tree", cl::Hidden,cl::desc("Display the SLP trees with Graphviz"))	(	"view-slp-tree"	,
		cl::Hidden	,
		cl::desc("Display the SLP trees with Graphviz")
	)

static

Movatterモバイル変換

Classes

Namespaces

Macros

Functions

Variables

Macro Definition Documentation

◆ DEBUG_TYPE

◆ SV_NAME

Function Documentation

◆ addMask()

◆ allConstant()

◆ allSameBlock()

◆ allSameType()

◆ areCompatibleCmpOps()

◆ arePointersCompatible()

◆ areTwoInsertFromSameBuildVector()

◆ buildIntrinsicArgTypes()

◆ buildUseMask()

◆ calculateRtStride()

◆ calculateShufflevectorMask()

◆ checkTreeSizes()

◆ clusterSortPtrAccesses()

◆ combineOrders()

◆ compareCmp()

◆ computeCommonAlignment()

◆ createExtractVector()

◆ createInsertVector()

◆ DEBUG_COUNTER()

◆ doesInTreeUserNeedToExtract()

◆ dumpOrder()

◆ findBuildAggregate()

◆ findBuildAggregate_rec()

◆ fixupOrderingIndices()

◆ gatherPossiblyVectorizableLoads()

◆ generateKeySubkey()

◆ getAggregateSize()

◆ getAltInstrMask()

◆ getElementIndex()

◆ getExtractIndex()

◆ getFloorFullVectorNumberOfElements()

◆ getFullVectorNumberOfElements()

◆ getGEPCosts()

◆ getInsertExtractIndex()

◆ getLocation()

◆ getNonPhiOperand()

◆ getNumElements()

◆ getNumElems()

◆ getPartNumElems()

◆ getRdxKind()

◆ getReductionInstr()

◆ getSameOpcode()

◆ getShuffleCost()

◆ getShufflevectorNumGroups()

◆ getValueType()

◆ getVectorCallCosts()

◆ getWidenedType()

◆ isAlternateInstruction()

◆ isCmpSameOrSwapped()

◆ isCommutative()

◆ isConstant()

◆ isFirstInsertElement()

◆ isFixedVectorShuffle()

◆ isLoadCombineCandidateImpl()

◆ isReductionCandidate()

◆ isRepeatedNonIdentityClusteredMask()

◆ isReverseOrder()

◆ isSimple()

◆ isSplat()

◆ isUndefVector()

◆ isValidElementType()

◆ isValidForAlternation()

◆ isVectorLikeInstWithConstOps()

◆ matchRdxBop()

◆ needToScheduleSingleInstruction()

◆ performExtractsShuffleAction()

◆ propagateMetadata()

◆ reorderOrder()

◆ reorderReuses()

◆ shortBundleName()