Movatterモバイル変換

[0]ホーム
URL:

LLVM 20.0.0git
Classes |Namespaces |Macros |Typedefs |Enumerations |Functions |Variables
LoopVectorize.cpp File Reference
#include "llvm/Transforms/Vectorize/LoopVectorize.h"
#include "LoopVectorizationPlanner.h"
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanHCFGBuilder.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <memory>
#include <string>
#include <tuple>
#include <utility>

Go to the source code of this file.

Classes

class  llvm::InnerLoopVectorizer
 InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF).More...
 
struct  llvm::EpilogueLoopVectorizationInfo
 Encapsulate information regarding vectorization of a loop and its epilogue.More...
 
class  llvm::InnerLoopAndEpilogueVectorizer
 An extension of the inner loop vectorizer that creates a skeleton for a vectorized loop that has its epilogue (residual) also vectorized.More...
 
class  llvm::EpilogueVectorizerMainLoop
 A specialized derived class of inner loop vectorizer that performs vectorization ofmain loops in the process of vectorizing loops and their epilogues.More...
 
class  llvm::EpilogueVectorizerEpilogueLoop
 
class  llvm::LoopVectorizationCostModel
 LoopVectorizationCostModel - estimates the expected speedups due to vectorization.More...
 
struct  llvm::LoopVectorizationCostModel::RegisterUsage
 A struct that represents some properties of the register usage of a loop.More...
 
struct  llvm::LoopVectorizationCostModel::CallWideningDecision
 

Namespaces

namespace  PreferPredicateTy
 
namespace  llvm
 This is an optimization pass for GlobalISel generic memory operations.
 

Macros

#define LV_NAME   "loop-vectorize"
 
#define DEBUG_TYPE   LV_NAME
 

Typedefs

using llvm::InstructionVFPair = std::pair<Instruction *,ElementCount >
 

Enumerations

enum  PreferPredicateTy::Option {PreferPredicateTy::ScalarEpilogue = 0,PreferPredicateTy::PredicateElseScalarEpilogue,PreferPredicateTy::PredicateOrDontVectorize }
 
enum  llvm::ScalarEpilogueLowering {
  llvm::CM_ScalarEpilogueAllowed,llvm::CM_ScalarEpilogueNotAllowedOptSize,llvm::CM_ScalarEpilogueNotAllowedLowTripLoop,llvm::CM_ScalarEpilogueNotNeededUsePredicate,
  llvm::CM_ScalarEpilogueNotAllowedUsePredicate
}
 

Functions

 STATISTIC (LoopsVectorized, "Number ofloops vectorized")
 
 STATISTIC (LoopsAnalyzed, "Number ofloops analyzedfor vectorization")
 
 STATISTIC (LoopsEpilogueVectorized, "Number of epilogues vectorized")
 
staticbool hasIrregularType (Type *Ty,constDataLayout &DL)
 A helper function that returns true if the given type is irregular.
 
static std::optional<unsigned > getSmallBestKnownTC (PredicatedScalarEvolution &PSE,Loop *L,bool CanUseConstantMax=true)
 Returns "best known" trip count for the specified loopL as defined by the following procedure: 1) Returns exact trip count if it is known.
 
staticDebugLoc getDebugLocFromInstOrOperands (Instruction *I)
 Look for a meaningful debug location on the instruction or its operands.
 
static void debugVectorizationMessage (constStringRef Prefix,constStringRef DebugMsg,Instruction *I)
 Write aDebugMsg about vectorization to the debug output stream.
 
staticOptimizationRemarkAnalysis createLVAnalysis (constchar *PassName,StringRef RemarkName,Loop *TheLoop,Instruction *I,DebugLocDL={})
 Create an analysis remark that explains why vectorization failed.
 
Value * llvm::createStepForVF (IRBuilderBase &B,Type *Ty,ElementCount VF, int64_t Step)
 Return a value for Step multiplied by VF.
 
Value * llvm::getRuntimeVF (IRBuilderBase &B,Type *Ty,ElementCount VF)
 Return the runtime value for VF.
 
void llvm::reportVectorizationFailure (constStringRef DebugMsg,constStringRef OREMsg,constStringRef ORETag,OptimizationRemarkEmitter *ORE,Loop *TheLoop,Instruction *I=nullptr)
 Reports a vectorization failure: printDebugMsg for debugging purposes along with the corresponding optimization remarkRemarkName.
 
static void llvm::reportVectorizationInfo (constStringRef Msg,constStringRef ORETag,OptimizationRemarkEmitter *ORE,Loop *TheLoop,Instruction *I=nullptr,DebugLocDL={})
 Reports an informative message: printMsg for debugging purposes as well as an optimization remark.
 
static void llvm::reportVectorization (OptimizationRemarkEmitter *ORE,Loop *TheLoop,VectorizationFactor VF,unsigned IC)
 Report successful vectorization of the loop.
 
staticbool useActiveLaneMask (TailFoldingStyle Style)
 
staticbool useActiveLaneMaskForControlFlow (TailFoldingStyle Style)
 
staticbool isExplicitVecOuterLoop (Loop *OuterLp,OptimizationRemarkEmitter *ORE)
 
static void collectSupportedLoops (Loop &L,LoopInfo *LI,OptimizationRemarkEmitter *ORE,SmallVectorImpl<Loop * > &V)
 
staticValue * emitTransformedIndex (IRBuilderBase &B,Value *Index,Value *StartValue,Value *Step,InductionDescriptor::InductionKind InductionKind,constBinaryOperator *InductionBinOp)
 Compute the transformed value of Index at offset StartValue using step StepValue.
 
std::optional<unsigned > getMaxVScale (constFunction &F,constTargetTransformInfo &TTI)
 
staticbool isIndvarOverflowCheckKnownFalse (constLoopVectorizationCostModel *Cost,ElementCount VF, std::optional<unsigned > UF=std::nullopt)
 For the given VF and UF and maximum trip count computed for the loop, return whether the induction variable might overflow in the vectorized loop.
 
staticbool useMaskedInterleavedAccesses (constTargetTransformInfo &TTI)
 
static void replaceVPBBWithIRVPBB (VPBasicBlock *VPBB,BasicBlock *IRBB)
 ReplaceVPBB with a VPIRBasicBlock wrappingIRBB.
 
staticValue * getExpandedStep (constInductionDescriptor &ID,const SCEV2ValueTy &ExpandedSCEVs)
 Return the expanded step forID usingExpandedSCEVs to look up SCEV expansion results.
 
static void addFullyUnrolledInstructionsToIgnore (Loop *L,constLoopVectorizationLegality::InductionList &IL,SmallPtrSetImpl<Instruction * > &InstsToIgnore)
 Knowing that loopL executes a single vector iteration, add instructions that will get simplified and thus should not have any cost toInstsToIgnore.
 
static void cse (BasicBlock *BB)
 Perform cse of induction variable instructions.
 
staticType * maybeVectorizeType (Type *Elt,ElementCount VF)
 
static std::optional<unsigned > getVScaleForTuning (constLoop *L,constTargetTransformInfo &TTI)
 Convenience function that returns the value of vscale_range iff vscale_range.min == vscale_range.max or otherwise returns the value returned by the corresponding TTI method.
 
staticunsigned getEstimatedRuntimeVF (constLoop *L,constTargetTransformInfo &TTI,ElementCount VF)
 This function attempts to return a value that represents the vectorization factor at runtime.
 
staticbool willGenerateVectors (VPlan &Plan,ElementCount VF,constTargetTransformInfo &TTI)
 Check if any recipe ofPlan will generate a vector value, which will be assigned a vector register.
 
staticconstSCEV * getAddressAccessSCEV (Value *Ptr,LoopVectorizationLegality *Legal,PredicatedScalarEvolution &PSE,constLoop *TheLoop)
 Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.
 
staticElementCount determineVPlanVF (constTargetTransformInfo &TTI,LoopVectorizationCostModel &CM)
 
staticbool planContainsAdditionalSimplifications (VPlan &Plan,VPCostContext &CostCtx,Loop *TheLoop)
 Return true if the original loop \ TheLoop contains any instructions that do not have corresponding recipes inPlan and are not marked to be ignored inCostCtx.
 
static void addRuntimeUnrollDisableMetaData (Loop *L)
 
static void fixReductionScalarResumeWhenVectorizingEpilog (VPRecipeBase *R,VPTransformState &State,BasicBlock *LoopMiddleBlock,BasicBlock *BypassBlock)
 
staticVPWidenIntOrFpInductionRecipe * createWidenInductionRecipes (PHINode *Phi,Instruction *PhiOrTrunc,VPValue *Start,constInductionDescriptor &IndDesc,VPlan &Plan,ScalarEvolution &SE,Loop &OrigLoop)
 Creates a VPWidenIntOrFpInductionRecpipe forPhi.
 
static void addCanonicalIVRecipes (VPlan &Plan,Type *IdxTy,bool HasNUW,DebugLocDL)
 
staticVPInstruction * addResumePhiRecipeForInduction (VPWidenInductionRecipe *WideIV,VPBuilder &VectorPHBuilder,VPBuilder &ScalarPHBuilder,VPTypeAnalysis &TypeInfo,VPValue *VectorTC)
 Create and return a ResumePhi forWideIV, unless it is truncated.
 
static void addScalarResumePhis (VPRecipeBuilder &Builder,VPlan &Plan,DenseMap<VPValue *,VPValue * > &IVEndValues)
 Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions, and update the VPIRInstructions wrapping the original phis in the scalar header.
 
staticSetVector<VPIRInstruction * > collectUsersInExitBlocks (Loop *OrigLoop,VPRecipeBuilder &Builder,VPlan &Plan)
 
static void addUsersInExitBlocks (VPlan &Plan,constSetVector<VPIRInstruction * > &ExitUsersToFix)
 
static void addExitUsersForFirstOrderRecurrences (VPlan &Plan,SetVector<VPIRInstruction * > &ExitUsersToFix)
 Handle users in the exit block for first order reductions in the original exit block.
 
staticScalarEpilogueLowering getScalarEpilogueLowering (Function *F,Loop *L,LoopVectorizeHints &Hints,ProfileSummaryInfo *PSI,BlockFrequencyInfo *BFI,TargetTransformInfo *TTI,TargetLibraryInfo *TLI,LoopVectorizationLegality &LVL,InterleavedAccessInfo *IAI)
 
staticbool processLoopInVPlanNativePath (Loop *L,PredicatedScalarEvolution &PSE,LoopInfo *LI,DominatorTree *DT,LoopVectorizationLegality *LVL,TargetTransformInfo *TTI,TargetLibraryInfo *TLI,DemandedBits *DB,AssumptionCache *AC,OptimizationRemarkEmitter *ORE,BlockFrequencyInfo *BFI,ProfileSummaryInfo *PSI,LoopVectorizeHints &Hints,LoopVectorizationRequirements &Requirements)
 
static void checkMixedPrecision (Loop *L,OptimizationRemarkEmitter *ORE)
 
staticbool areRuntimeChecksProfitable (GeneratedRTChecks &Checks,VectorizationFactor &VF,Loop *L,constTargetTransformInfo &TTI,PredicatedScalarEvolution &PSE,ScalarEpilogueLowering SEL)
 
static void preparePlanForMainVectorLoop (VPlan &MainPlan,VPlan &EpiPlan)
 PrepareMainPlan for vectorizing the main vector loop during epilogue vectorization.
 
static void preparePlanForEpilogueVectorLoop (VPlan &Plan,Loop *L,const SCEV2ValueTy &ExpandedSCEVs,constEpilogueLoopVectorizationInfo &EPI)
 PreparePlan for vectorizing the epilogue loop.
 

Variables

constchar VerboseDebug [] =DEBUG_TYPE "-verbose"
 
staticcl::opt<bool > EnableEpilogueVectorization ("enable-epilogue-vectorization", cl::init(true), cl::Hidden,cl::desc("Enable vectorization of epilogue loops."))
 
staticcl::opt<unsigned > EpilogueVectorizationForceVF ("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,cl::desc("When epilogue vectorization is enabled, and avalue greater than " "1 is specified, forces the given VFfor all applicable epilogue " "loops."))
 
staticcl::opt<unsigned > EpilogueVectorizationMinVF ("epilogue-vectorization-minimum-VF", cl::Hidden,cl::desc("Onlyloops with vectorization factor equal to or larger than " "the specifiedvalue are consideredfor epilogue vectorization."))
 
staticcl::opt<unsigned > TinyTripCountVectorThreshold ("vectorizer-min-trip-count", cl::init(16), cl::Hidden,cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorizedonlyif no scalar iteration overheads " "are incurred."))
 Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.
 
staticcl::opt<unsigned > VectorizeMemoryCheckThreshold ("vectorize-memory-check-threshold", cl::init(128), cl::Hidden,cl::desc("The maximum allowed number of runtime memory checks"))
 
staticcl::opt<PreferPredicateTy::Option > PreferPredicateOverEpilogue ("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."),cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicateloops, create scalar epilogue"),clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogueif tail " "folding fails."),clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorizationif " "tail-folding fails.")))
 
staticcl::opt<TailFoldingStyle > ForceTailFoldingStyle ("force-tail-folding-style",cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None),cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),clEnumValN(TailFoldingStyle::Data, "data", "Create lane maskfordataonly, using active.lane.mask intrinsic"),clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"),clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, anduse " "itfor bothdata and control flow"),clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar todata-and-control, but remove the runtime check"),clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVLinstructionsfor tail folding. If EVL " "isunsupported, fallback todata-without-lane-mask.")))
 
staticcl::opt<bool > MaximizeBandwidth ("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
 
staticcl::opt<bool > EnableInterleavedMemAccesses ("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
 
staticcl::opt<bool > EnableMaskedInterleavedMemAccesses ("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
 An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.
 
staticcl::opt<unsigned > ForceTargetNumScalarRegs ("force-target-num-scalar-regs", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's number of scalar registers."))
 
staticcl::opt<unsigned > ForceTargetNumVectorRegs ("force-target-num-vector-regs", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's number ofvector registers."))
 
staticcl::opt<unsigned > ForceTargetMaxScalarInterleaveFactor ("force-target-max-scalar-interleave", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's max interleave factorfor " "scalar loops."))
 
staticcl::opt<unsigned > ForceTargetMaxVectorInterleaveFactor ("force-target-max-vector-interleave", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's max interleave factorfor " "vectorized loops."))
 
cl::opt<unsigned > ForceTargetInstructionCost ("force-target-instruction-cost", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's expected costfor " "an instruction to a single constant value. Mostly " "usefulfor getting consistent testing."))
 
staticcl::opt<bool > ForceTargetSupportsScalableVectors ("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,cl::desc("Pretend that scalablevectors are supported, evenif the target does " "not support them. This flag shouldonly be usedfor testing."))
 
staticcl::opt<unsigned > SmallLoopCost ("small-loop-cost", cl::init(20), cl::Hidden,cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
 
staticcl::opt<bool > LoopVectorizeWithBlockFrequency ("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,cl::desc("Enable theuse of theblock frequencyanalysis to accessPGO " "heuristics minimizing code growth in coldregions and being more " "aggressive in hot regions."))
 
staticcl::opt<bool > EnableLoadStoreRuntimeInterleave ("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,cl::desc("Enable runtime interleaving untilload/store ports are saturated"))
 
staticcl::opt<unsigned > NumberOfStoresToPredicate ("vectorize-num-stores-pred", cl::init(1), cl::Hidden,cl::desc("Max number ofstores to be predicated behind an if."))
 The number of stores in a loop that are allowed to need predication.
 
staticcl::opt<bool > EnableIndVarRegisterHeur ("enable-ind-var-reg-heur", cl::init(true), cl::Hidden,cl::desc("Count the induction variableonly once when interleaving"))
 
staticcl::opt<bool > EnableCondStoresVectorization ("enable-cond-stores-vec", cl::init(true), cl::Hidden,cl::desc("Enableif predication ofstores during vectorization."))
 
staticcl::opt<unsigned > MaxNestedScalarReductionIC ("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,cl::desc("The maximum interleave count touse when interleaving a scalar " "reduction in a nested loop."))
 
staticcl::opt<bool > PreferInLoopReductions ("prefer-inloop-reductions", cl::init(false), cl::Hidden,cl::desc("Prefer in-loopvectorreductions, " "overriding the targets preference."))
 
staticcl::opt<bool > ForceOrderedReductions ("force-ordered-reductions", cl::init(false), cl::Hidden,cl::desc("Enable the vectorisation ofloops with in-order (strict) " "FP reductions"))
 
staticcl::opt<bool > PreferPredicatedReductionSelect ("prefer-predicated-reduction-select", cl::init(false), cl::Hidden,cl::desc("Prefer predicating areduction operation over an after loop select."))
 
cl::opt<bool > llvm::EnableVPlanNativePath ("enable-vplan-native-path", cl::Hidden,cl::desc("EnableVPlan-native vectorization path with " "supportfor outer loop vectorization."))
 
staticcl::opt<bool > VPlanBuildStressTest ("vplan-build-stress-test", cl::init(false), cl::Hidden,cl::desc("BuildVPlanfor every supported loop nest in thefunction and bail " "out right after the build (stresstest theVPlanH-CFG construction " "in theVPlan-native vectorization path)."))
 
staticcl::opt<cl::boolOrDefault > ForceSafeDivisor ("force-widen-divrem-via-safe-divisor", cl::Hidden,cl::desc("Override cost based safe divisor wideningfor div/rem instructions"))
 
staticcl::opt<bool > UseWiderVFIfCallVariantsPresent ("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden,cl::desc("Try wider VFsif they enable theuse ofvector variants"))
 
staticcl::opt<bool > EnableEarlyExitVectorization ("enable-early-exit-vectorization", cl::init(false), cl::Hidden,cl::desc("Enable vectorization of early exitloops with uncountable exits."))
 
static constexpruint32_t SCEVCheckBypassWeights [] = {1, 127}
 
static constexpruint32_t MemCheckBypassWeights [] = {1, 127}
 
static constexpruint32_t MinItersBypassWeights [] = {1, 127}
 
constchar LLVMLoopVectorizeFollowupAll [] = "llvm.loop.vectorize.followup_all"
 
constchar LLVMLoopVectorizeFollowupVectorized []
 
constchar LLVMLoopVectorizeFollowupEpilogue []
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   LV_NAME

Definition at line159 of fileLoopVectorize.cpp.

◆ LV_NAME

#define LV_NAME   "loop-vectorize"

Definition at line158 of fileLoopVectorize.cpp.

Function Documentation

◆ addCanonicalIVRecipes()

static void addCanonicalIVRecipes(VPlan & Plan,
Type * IdxTy,
bool HasNUW,
DebugLoc DL 
)
static

Definition at line8928 of fileLoopVectorize.cpp.

Referencesllvm::VPUser::addOperand(),llvm::VPInstruction::BranchOnCount,llvm::VPBuilder::createNaryOp(),llvm::VPBuilder::createOverflowingOp(),DL,llvm::VPBlockBase::getEntryBasicBlock(),llvm::VPBlockBase::getExitingBasicBlock(),llvm::VPlan::getOrAddLiveIn(),llvm::VPlan::getVectorLoopRegion(),llvm::VPlan::getVectorTripCount(), andllvm::VPlan::getVFxUF().

◆ addExitUsersForFirstOrderRecurrences()

static void addExitUsersForFirstOrderRecurrences(VPlan & Plan,
SetVector<VPIRInstruction * > & ExitUsersToFix 
)
static

Handle users in the exit block for first order reductions in the original exit block.

The penultimate value of recurrences is fed to their LCSSA phi users in the original exit block using the VPIRInstruction wrapping to the LCSSA phi.

Definition at line9110 of fileLoopVectorize.cpp.

Referencesassert(),llvm::VPBuilder::createNaryOp(),llvm::VPInstruction::ExtractFromEnd,llvm::VPlan::getCanonicalIV(),llvm::VPBlockBase::getEntryBasicBlock(),llvm::VPlan::getMiddleBlock(),llvm::VPlan::getOrAddLiveIn(),llvm::VPlan::getScalarPreheader(),llvm::VPCanonicalIVPHIRecipe::getScalarType(),llvm::VPBlockBase::getSingleSuccessor(),llvm::VPlan::getVectorLoopRegion(),llvm::VPBasicBlock::phis(), andllvm::VPUser::setOperand().

◆ addFullyUnrolledInstructionsToIgnore()

static void addFullyUnrolledInstructionsToIgnore(Loop * L,
constLoopVectorizationLegality::InductionList & IL,
SmallPtrSetImpl<Instruction * > & InstsToIgnore 
)
static

Knowing that loopL executes a single vector iteration, add instructions that will get simplified and thus should not have any cost toInstsToIgnore.

Definition at line2662 of fileLoopVectorize.cpp.

Referencesllvm::all_of(),llvm::SmallPtrSetImpl< PtrType >::insert(),IV, andllvm::Value::users().

Referenced byllvm::LoopVectorizationCostModel::expectedCost().

◆ addResumePhiRecipeForInduction()

staticVPInstruction * addResumePhiRecipeForInduction(VPWidenInductionRecipe * WideIV,
VPBuilder & VectorPHBuilder,
VPBuilder & ScalarPHBuilder,
VPTypeAnalysis & TypeInfo,
VPValue * VectorTC 
)
static

Create and return a ResumePhi forWideIV, unless it is truncated.

If the induction recipe is not canonical, creates a VPDerivedIVRecipe to compute the end value of the induction.

Definition at line8954 of fileLoopVectorize.cpp.

Referencesllvm::VPBuilder::createDerivedIV(),llvm::VPBuilder::createNaryOp(),llvm::VPBuilder::createScalarCast(),llvm::VPRecipeBase::getDebugLoc(),llvm::VPWidenInductionRecipe::getInductionDescriptor(),llvm::VPHeaderPHIRecipe::getStartValue(),llvm::VPWidenInductionRecipe::getStepValue(),llvm::VPTypeAnalysis::inferScalarType(), andllvm::VPInstruction::ResumePhi.

Referenced byaddScalarResumePhis().

◆ addRuntimeUnrollDisableMetaData()

static void addRuntimeUnrollDisableMetaData(Loop * L)
static

Definition at line7545 of fileLoopVectorize.cpp.

Referencesllvm::MDNode::get(),llvm::MDString::get(),llvm::MDNode::getNumOperands(),llvm::MDNode::getOperand(),I,llvm::SmallVectorTemplateBase< T, bool >::push_back(), andllvm::MDNode::replaceOperandWith().

Referenced byllvm::LoopVectorizationPlanner::executePlan(), andllvm::LoopVectorizePass::processLoop().

◆ addScalarResumePhis()

static void addScalarResumePhis(VPRecipeBuilder & Builder,
VPlan & Plan,
DenseMap<VPValue *,VPValue * > & IVEndValues 
)
static

Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions, and update the VPIRInstructions wrapping the original phis in the scalar header.

End values for inductions are added toIVEndValues.

Definition at line8992 of fileLoopVectorize.cpp.

Referencesllvm::VPUser::addOperand(),addResumePhiRecipeForInduction(),assert(),llvm::VPBuilder::createNaryOp(),llvm::VPInstruction::ExtractFromEnd,llvm::VPlan::getCanonicalIV(),llvm::VPlan::getMiddleBlock(),llvm::VPlan::getOrAddLiveIn(),llvm::VPRecipeBuilder::getRecipe(),llvm::VPlan::getScalarHeader(),llvm::VPlan::getScalarPreheader(),llvm::VPCanonicalIVPHIRecipe::getScalarType(),llvm::VPBlockBase::getSinglePredecessor(),llvm::VPBlockBase::getSingleSuccessor(),llvm::VPlan::getVectorLoopRegion(),llvm::VPlan::getVectorTripCount(),Name, andllvm::VPInstruction::ResumePhi.

◆ addUsersInExitBlocks()

static void addUsersInExitBlocks(VPlan & Plan,
constSetVector<VPIRInstruction * > & ExitUsersToFix 
)
static

Definition at line9087 of fileLoopVectorize.cpp.

Referencesassert(),B,llvm::SetVector< T, Vector, Set, N >::empty(), andllvm::VPlan::getMiddleBlock().

◆ areRuntimeChecksProfitable()

staticbool areRuntimeChecksProfitable(GeneratedRTChecks & Checks,
VectorizationFactor & VF,
Loop * L,
constTargetTransformInfo & TTI,
PredicatedScalarEvolution & PSE,
ScalarEpilogueLowering SEL 
)
static

Definition at line10052 of fileLoopVectorize.cpp.

Referencesllvm::alignTo(),llvm::CM_ScalarEpilogueAllowed,llvm::VectorizationFactor::Cost,llvm::dbgs(),llvm::divideCeil(),getEstimatedRuntimeVF(),llvm::ElementCount::getFixed(),getSmallBestKnownTC(),llvm::InstructionCost::getValue(),llvm::details::FixedOrScalableQuantity< ElementCount, unsigned >::isKnownLT(),llvm::ElementCount::isScalar(),llvm::InstructionCost::isValid(),LLVM_DEBUG,llvm::VectorizationFactor::MinProfitableTripCount,llvm::VectorizationFactor::ScalarCost,VectorizeMemoryCheckThreshold, andllvm::VectorizationFactor::Width.

Referenced byllvm::LoopVectorizePass::processLoop().

◆ checkMixedPrecision()

static void checkMixedPrecision(Loop * L,
OptimizationRemarkEmitter * ORE 
)
static

Definition at line10011 of fileLoopVectorize.cpp.

Referencesllvm::OptimizationRemarkEmitter::emit(),llvm::SmallVectorBase< Size_T >::empty(),I,llvm::SmallPtrSetImpl< PtrType >::insert(),LV_NAME,llvm::SmallVectorImpl< T >::pop_back_val(), andllvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced byllvm::LoopVectorizePass::processLoop().

◆ collectSupportedLoops()

static void collectSupportedLoops(Loop & L,
LoopInfo * LI,
OptimizationRemarkEmitter * ORE,
SmallVectorImpl<Loop * > & V 
)
static

Definition at line2179 of fileLoopVectorize.cpp.

ReferencescollectSupportedLoops(),llvm::EnableVPlanNativePath,isExplicitVecOuterLoop(),llvm::LoopBlocksRPO::perform(), andVPlanBuildStressTest.

Referenced bycollectSupportedLoops(), andllvm::LoopVectorizePass::runImpl().

◆ collectUsersInExitBlocks()

staticSetVector<VPIRInstruction * > collectUsersInExitBlocks(Loop * OrigLoop,
VPRecipeBuilder & Builder,
VPlan & Plan 
)
static

Definition at line9053 of fileLoopVectorize.cpp.

Referencesassert(),llvm::VPlan::getExitBlocks(),llvm::LoopBase< BlockT, LoopT >::getLoopLatch(),llvm::VPlan::getMiddleBlock(),llvm::VPRecipeBuilder::getVPValueOrAddLiveIn(), andllvm::SetVector< T, Vector, Set, N >::insert().

◆ createLVAnalysis()

staticOptimizationRemarkAnalysis createLVAnalysis(constchar * PassName,
StringRef RemarkName,
Loop * TheLoop,
Instruction * I,
DebugLoc DL ={} 
)
static

Create an analysis remark that explains why vectorization failed.

PassName is the name of the pass (e.g. can be AlwaysPrint).RemarkName is the identifier for the remark. IfI is passed it is an instruction that prevents vectorization. OtherwiseTheLoop is used for the location of the remark. IfDL is passed, use it as debug location for the remark.

Returns
the remark object that can be streamed to.

Definition at line860 of fileLoopVectorize.cpp.

Referenced byllvm::reportVectorizationFailure().

◆ createWidenInductionRecipes()

staticVPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode * Phi,
Instruction * PhiOrTrunc,
VPValue * Start,
constInductionDescriptor & IndDesc,
VPlan & Plan,
ScalarEvolution & SE,
Loop & OrigLoop 
)
static

Creates a VPWidenIntOrFpInductionRecpipe forPhi.

If needed, it will also insert a recipe to expand the step for the induction recipe.

Definition at line8317 of fileLoopVectorize.cpp.

Referencesassert(),llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(),llvm::vputils::getOrCreateVPValueForSCEVExpr(),llvm::InductionDescriptor::getStartValue(),llvm::InductionDescriptor::getStep(),llvm::VPlan::getVF(), andllvm::ScalarEvolution::isLoopInvariant().

◆ cse()

static void cse(BasicBlock * BB)
static

Perform cse of induction variable instructions.

Definition at line2811 of fileLoopVectorize.cpp.

Referencesllvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), andllvm::make_early_inc_range().

Referenced byllvm::InnerLoopVectorizer::fixVectorizedLoop().

◆ debugVectorizationMessage()

static void debugVectorizationMessage(constStringRef Prefix,
constStringRef DebugMsg,
Instruction * I 
)
static

Write aDebugMsg about vectorization to the debug output stream.

IfI is passed, the message relates to that particular instruction.

Definition at line840 of fileLoopVectorize.cpp.

Referencesllvm::dbgs(), andI.

Referenced byllvm::reportVectorization(), andllvm::reportVectorizationFailure().

◆ determineVPlanVF()

staticElementCount determineVPlanVF(constTargetTransformInfo & TTI,
LoopVectorizationCostModel & CM 
)
static

Definition at line7028 of fileLoopVectorize.cpp.

Referencesllvm::TargetTransformInfo::enableScalableVectorization(),llvm::ElementCount::get(),llvm::TargetTransformInfo::getRegisterBitWidth(),llvm::LoopVectorizationCostModel::getSmallestAndWidestTypes(),N,RegSize,llvm::TargetTransformInfo::RGK_FixedWidthVector, andllvm::TargetTransformInfo::RGK_ScalableVector.

Referenced byllvm::LoopVectorizationPlanner::planInVPlanNativePath().

◆ emitTransformedIndex()

staticValue * emitTransformedIndex(IRBuilderBase & B,
Value * Index,
Value * StartValue,
Value * Step,
InductionDescriptor::InductionKind InductionKind,
constBinaryOperator * InductionBinOp 
)
static

Compute the transformed value of Index at offset StartValue using step StepValue.

For integer induction, returns StartValue + Index * StepValue. For pointer induction, returns StartValue[Index * StepValue]. FIXME: The newly created binary instructions should contain nsw/nuw flags, which can be found from the original scalar operations.

Definition at line2216 of fileLoopVectorize.cpp.

Referencesassert(),B,CreateAdd(),CreateMul(),llvm::Value::getName(),llvm::BinaryOperator::getOpcode(),llvm::Value::getType(),llvm::InductionDescriptor::IK_FpInduction,llvm::InductionDescriptor::IK_IntInduction,llvm::InductionDescriptor::IK_NoInduction,llvm::InductionDescriptor::IK_PtrInduction,llvm::Type::isFloatingPointTy(),llvm::Type::isIntegerTy(),llvm_unreachable,llvm::Offset,llvm::Value::setName(),X, andY.

Referenced byllvm::InnerLoopVectorizer::createInductionAdditionalBypassValues(), andllvm::VPDerivedIVRecipe::execute().

◆ fixReductionScalarResumeWhenVectorizingEpilog()

static void fixReductionScalarResumeWhenVectorizingEpilog(VPRecipeBase * R,
VPTransformState & State,
BasicBlock * LoopMiddleBlock,
BasicBlock * BypassBlock 
)
static

Definition at line7582 of fileLoopVectorize.cpp.

Referencesassert(),llvm::VPInstruction::ComputeReductionResult,llvm::count_if(),llvm::find_if(),llvm::VPTransformState::get(),llvm::PHINode::getIncomingValueForBlock(),llvm::RecurrenceDescriptor::getRecurrenceKind(),llvm::RecurrenceDescriptor::getRecurrenceStartValue(),llvm::RecurrenceDescriptor::getSentinelValue(),llvm::CmpInst::ICMP_EQ,llvm::CmpInst::ICMP_NE,llvm::RecurrenceDescriptor::isAnyOfRecurrenceKind(),llvm::RecurrenceDescriptor::isFindLastIVRecurrenceKind(),llvm::PatternMatch::m_OneUse(),llvm::PatternMatch::m_Select(),llvm::PatternMatch::m_Specific(),llvm::PatternMatch::m_SpecificICmp(),llvm::PatternMatch::m_Value(), andllvm::PatternMatch::match().

Referenced byllvm::LoopVectorizationPlanner::executePlan().

◆ getAddressAccessSCEV()

staticconstSCEV * getAddressAccessSCEV(Value * Ptr,
LoopVectorizationLegality * Legal,
PredicatedScalarEvolution & PSE,
constLoop * TheLoop 
)
static

Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.

This SCEV can be sent to the Target in order to estimate the address calculation cost.

Definition at line5577 of fileLoopVectorize.cpp.

Referencesllvm::ScalarEvolution::getSCEV(),llvm::PredicatedScalarEvolution::getSCEV(),llvm::PredicatedScalarEvolution::getSE(),Idx,llvm::ScalarEvolution::isLoopInvariant(),llvm::IRSimilarity::Legal, andPtr.

◆ getDebugLocFromInstOrOperands()

staticDebugLoc getDebugLocFromInstOrOperands(Instruction * I)
static

Look for a meaningful debug location on the instruction or its operands.

Definition at line820 of fileLoopVectorize.cpp.

ReferencesI.

◆ getEstimatedRuntimeVF()

staticunsigned getEstimatedRuntimeVF(constLoop * L,
constTargetTransformInfo & TTI,
ElementCount VF 
)
static

This function attempts to return a value that represents the vectorization factor at runtime.

For fixed-width VFs we know this precisely at compile time, but for scalable VFs we calculate it based on an estimate of the vscale value.

Definition at line4255 of fileLoopVectorize.cpp.

Referencesassert(),llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(),getVScaleForTuning(), andllvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable().

Referenced byareRuntimeChecksProfitable(),llvm::LoopVectorizationCostModel::isEpilogueVectorizationProfitable(),llvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor(), andllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ getExpandedStep()

staticValue * getExpandedStep(constInductionDescriptor & ID,
const SCEV2ValueTy & ExpandedSCEVs 
)
static

Return the expanded step forID usingExpandedSCEVs to look up SCEV expansion results.

Definition at line2647 of fileLoopVectorize.cpp.

Referencesassert(),llvm::CallingConv::C, andI.

Referenced byllvm::InnerLoopVectorizer::createInductionAdditionalBypassValues().

◆ getMaxVScale()

std::optional<unsigned > getMaxVScale(constFunction & F,
constTargetTransformInfo & TTI 
)

Definition at line2295 of fileLoopVectorize.cpp.

ReferencesF, andllvm::TargetTransformInfo::getMaxVScale().

Referenced byllvm::LoopVectorizationCostModel::computeMaxVF(), andisIndvarOverflowCheckKnownFalse().

◆ getScalarEpilogueLowering()

staticScalarEpilogueLowering getScalarEpilogueLowering(Function * F,
Loop * L,
LoopVectorizeHints & Hints,
ProfileSummaryInfo * PSI,
BlockFrequencyInfo * BFI,
TargetTransformInfo * TTI,
TargetLibraryInfo * TLI,
LoopVectorizationLegality & LVL,
InterleavedAccessInfo * IAI 
)
static

Definition at line9895 of fileLoopVectorize.cpp.

Referencesllvm::CM_ScalarEpilogueAllowed,llvm::CM_ScalarEpilogueNotAllowedOptSize,llvm::CM_ScalarEpilogueNotAllowedUsePredicate,llvm::CM_ScalarEpilogueNotNeededUsePredicate,F,llvm::LoopVectorizeHints::FK_Disabled,llvm::LoopVectorizeHints::FK_Enabled,llvm::LoopVectorizeHints::getForce(),llvm::LoopVectorizeHints::getPredicate(),llvm::IRPass,PreferPredicateTy::PredicateElseScalarEpilogue,PreferPredicateTy::PredicateOrDontVectorize,PreferPredicateOverEpilogue,llvm::TargetTransformInfo::preferPredicateOverEpilogue(),PreferPredicateTy::ScalarEpilogue, andllvm::shouldOptimizeForSize().

Referenced byllvm::LoopVectorizePass::processLoop(), andprocessLoopInVPlanNativePath().

◆ getSmallBestKnownTC()

static std::optional<unsigned > getSmallBestKnownTC(PredicatedScalarEvolution & PSE,
Loop * L,
bool CanUseConstantMax =true 
)
static

Returns "best known" trip count for the specified loopL as defined by the following procedure: 1) Returns exact trip count if it is known.

2) Returns expected trip count according to profile data if any. 3) Returns upper bound estimate if known, and ifCanUseConstantMax. 4) Returns std::nullopt if all of the above failed.

Definition at line420 of fileLoopVectorize.cpp.

Referencesllvm::getLoopEstimatedTripCount(),llvm::PredicatedScalarEvolution::getSE(),llvm::PredicatedScalarEvolution::getSmallConstantMaxTripCount(),llvm::ScalarEvolution::getSmallConstantTripCount(), andLoopVectorizeWithBlockFrequency.

Referenced byareRuntimeChecksProfitable(),llvm::LoopVectorizePass::processLoop(), andllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ getVScaleForTuning()

static std::optional<unsigned > getVScaleForTuning(constLoop * L,
constTargetTransformInfo & TTI 
)
static

Convenience function that returns the value of vscale_range iff vscale_range.min == vscale_range.max or otherwise returns the value returned by the corresponding TTI method.

Definition at line4238 of fileLoopVectorize.cpp.

Referencesllvm::Function::getFnAttribute(),llvm::TargetTransformInfo::getVScaleForTuning(),llvm::Attribute::getVScaleRangeMin(), andllvm::Function::hasFnAttribute().

Referenced bygetEstimatedRuntimeVF().

◆ hasIrregularType()

staticbool hasIrregularType(Type * Ty,
constDataLayout & DL 
)
static

A helper function that returns true if the given type is irregular.

The type is irregular if its allocated size doesn't equal the store size of an element of the corresponding vector type.

Definition at line406 of fileLoopVectorize.cpp.

ReferencesDL.

Referenced byllvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), andllvm::LoopVectorizationCostModel::memoryInstructionCanBeWidened().

◆ isExplicitVecOuterLoop()

staticbool isExplicitVecOuterLoop(Loop * OuterLp,
OptimizationRemarkEmitter * ORE 
)
static

Definition at line2151 of fileLoopVectorize.cpp.

Referencesllvm::LoopVectorizeHints::allowVectorization(),assert(),llvm::dbgs(),llvm::LoopVectorizeHints::emitRemarkWithHints(),llvm::LoopVectorizeHints::FK_Undefined,llvm::LoopVectorizeHints::getForce(),llvm::LoopBase< BlockT, LoopT >::getHeader(),llvm::LoopVectorizeHints::getInterleave(),llvm::BasicBlock::getParent(),llvm::LoopBase< BlockT, LoopT >::isInnermost(), andLLVM_DEBUG.

Referenced bycollectSupportedLoops().

◆ isIndvarOverflowCheckKnownFalse()

staticbool isIndvarOverflowCheckKnownFalse(constLoopVectorizationCostModel * Cost,
ElementCount VF,
std::optional<unsigned > UF =std::nullopt 
)
static

For the given VF and UF and maximum trip count computed for the loop, return whether the induction variable might overflow in the vectorized loop.

If not, then we know a runtime overflow check always evaluates to false and can be removed.

Definition at line2310 of fileLoopVectorize.cpp.

Referencesllvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(),getMaxVScale(), andllvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable().

Referenced byllvm::InnerLoopVectorizer::emitIterationCountCheck().

◆ maybeVectorizeType()

staticType * maybeVectorizeType(Type * Elt,
ElementCount VF 
)
static

Definition at line2858 of fileLoopVectorize.cpp.

Referencesllvm::VectorType::get(),llvm::Type::isFloatingPointTy(),llvm::Type::isIntOrPtrTy(), andllvm::ElementCount::isScalar().

Referenced byllvm::LoopVectorizationCostModel::getVectorIntrinsicCost().

◆ planContainsAdditionalSimplifications()

staticbool planContainsAdditionalSimplifications(VPlan & Plan,
VPCostContext & CostCtx,
Loop * TheLoop 
)
static

Return true if the original loop \ TheLoop contains any instructions that do not have corresponding recipes inPlan and are not marked to be ignored inCostCtx.

This means the VPlan contains simplification that the legacy cost-model did not account for.

Definition at line7409 of fileLoopVectorize.cpp.

Referencesllvm::any_of(),llvm::LoopBase< BlockT, LoopT >::blocks(),llvm::VPRegionBlock::getEntry(),llvm::VPlan::getVectorLoopRegion(),I,llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(),IR, andllvm::vp_depth_first_deep().

Referenced byllvm::LoopVectorizationPlanner::computeBestVF().

◆ preparePlanForEpilogueVectorLoop()

static void preparePlanForEpilogueVectorLoop(VPlan & Plan,
Loop * L,
const SCEV2ValueTy & ExpandedSCEVs,
constEpilogueLoopVectorizationInfo & EPI 
)
static

PreparePlan for vectorizing the epilogue loop.

That is, re-use expanded SCEVs fromExpandedSCEVs and set resume values for header recipes.

Definition at line10208 of fileLoopVectorize.cpp.

Referencesllvm::all_of(),assert(),llvm::IRBuilderBase::CreateICmpEQ(),llvm::IRBuilderBase::CreateICmpNE(),llvm::IRBuilderBase::CreateSelect(),llvm::VPlan::getEntry(),llvm::VPBlockBase::getEntryBasicBlock(),llvm::BasicBlock::getFirstNonPHIIt(),llvm::PHINode::getIncomingValueForBlock(),llvm::VPlan::getOrAddLiveIn(),llvm::BasicBlock::getParent(),llvm::RecurrenceDescriptor::getRecurrenceKind(),llvm::RecurrenceDescriptor::getRecurrenceStartValue(),llvm::RecurrenceDescriptor::getSentinelValue(),llvm::VPlan::getTripCount(),llvm::VPlan::getVectorLoopRegion(),llvm::RecurrenceDescriptor::isAnyOfRecurrenceKind(),llvm::RecurrenceDescriptor::isFindLastIVRecurrenceKind(),IV,llvm::make_early_inc_range(),P,llvm::predecessors(),llvm::VPValue::replaceAllUsesWith(), andllvm::VPlan::resetTripCount().

Referenced byllvm::LoopVectorizePass::processLoop().

◆ preparePlanForMainVectorLoop()

static void preparePlanForMainVectorLoop(VPlan & MainPlan,
VPlan & EpiPlan 
)
static

PrepareMainPlan for vectorizing the main vector loop during epilogue vectorization.

Remove ResumePhis fromMainPlan for inductions that don't have a corresponding wide induction inEpiPlan.

Definition at line10157 of fileLoopVectorize.cpp.

Referencesllvm::any_of(),llvm::VPBasicBlock::begin(),llvm::SmallPtrSetImpl< PtrType >::contains(),llvm::VPBuilder::createNaryOp(),llvm::VPRecipeBase::eraseFromParent(),llvm::VPlan::getCanonicalIV(),llvm::VPValue::getDefiningRecipe(),llvm::VPBlockBase::getEntryBasicBlock(),llvm::VPUser::getOperand(),llvm::VPlan::getScalarHeader(),llvm::VPlan::getScalarPreheader(),llvm::VPlan::getVectorLoopRegion(),llvm::VPlan::getVectorTripCount(),llvm::SmallPtrSetImpl< PtrType >::insert(),llvm::PatternMatch::m_Specific(),llvm::PatternMatch::m_SpecificInt(),llvm::make_early_inc_range(),llvm::PatternMatch::match(),llvm::VPBasicBlock::phis(),llvm::VPlanTransforms::removeDeadRecipes(), andllvm::VPInstruction::ResumePhi.

Referenced byllvm::LoopVectorizePass::processLoop().

◆ processLoopInVPlanNativePath()

staticbool processLoopInVPlanNativePath(Loop * L,
PredicatedScalarEvolution & PSE,
LoopInfo * LI,
DominatorTree * DT,
LoopVectorizationLegality * LVL,
TargetTransformInfo * TTI,
TargetLibraryInfo * TLI,
DemandedBits * DB,
AssumptionCache * AC,
OptimizationRemarkEmitter * ORE,
BlockFrequencyInfo * BFI,
ProfileSummaryInfo * PSI,
LoopVectorizeHints & Hints,
LoopVectorizationRequirements & Requirements 
)
static

Definition at line9944 of fileLoopVectorize.cpp.

Referencesassert(),llvm::LoopVectorizationCostModel::collectElementTypesForWidening(),llvm::LoopVectorizationCostModel::CostKind,llvm::dbgs(),llvm::VectorizationFactor::Disabled(),llvm::EnableVPlanNativePath,llvm::LoopVectorizationPlanner::executePlan(),F,llvm::PredicatedScalarEvolution::getBackedgeTakenCount(),llvm::LoopVectorizationLegality::getLAI(),llvm::LoopVectorizationPlanner::getPlanFor(),getScalarEpilogueLowering(),llvm::LoopVectorizeHints::getWidth(),llvm::hasBranchWeightMD(),LLVM_DEBUG,llvm::LoopVectorizationPlanner::planInVPlanNativePath(),llvm::reportVectorization(),llvm::LoopVectorizeHints::setAlreadyVectorized(),llvm::verifyFunction(),VPlanBuildStressTest, andllvm::VectorizationFactor::Width.

Referenced byllvm::LoopVectorizePass::processLoop().

◆ replaceVPBBWithIRVPBB()

static void replaceVPBBWithIRVPBB(VPBasicBlock * VPBB,
BasicBlock * IRBB 
)
static

ReplaceVPBB with a VPIRBasicBlock wrappingIRBB.

All recipes fromVPBB are moved to the end of the newly created VPIRBasicBlock. VPBB must have a single predecessor, which is rewired to the new VPIRBasicBlock. All successors of VPBB, if any, are rewired to the new VPIRBasicBlock.

Definition at line2617 of fileLoopVectorize.cpp.

Referencesassert(),llvm::VPlan::createVPIRBasicBlock(),llvm::VPBasicBlock::end(),llvm::VPBlockBase::getPlan(),llvm::make_early_inc_range(), andllvm::VPBlockUtils::reassociateBlocks().

Referenced byllvm::InnerLoopVectorizer::createVectorLoopSkeleton(), andllvm::LoopVectorizationPlanner::executePlan().

◆ STATISTIC()[1/3]

STATISTIC(LoopsAnalyzed ,
"Number ofloops analyzedfor vectorization"  
)

◆ STATISTIC()[2/3]

STATISTIC(LoopsEpilogueVectorized ,
"Number of epilogues vectorized"  
)

◆ STATISTIC()[3/3]

STATISTIC(LoopsVectorized ,
"Number ofloops vectorized"  
)

◆ useActiveLaneMask()

staticbool useActiveLaneMask(TailFoldingStyle Style)
static

Definition at line2126 of fileLoopVectorize.cpp.

◆ useActiveLaneMaskForControlFlow()

staticbool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static

Definition at line2132 of fileLoopVectorize.cpp.

◆ useMaskedInterleavedAccesses()

staticbool useMaskedInterleavedAccesses(constTargetTransformInfo & TTI)
static

Definition at line2341 of fileLoopVectorize.cpp.

Referencesllvm::TargetTransformInfo::enableMaskedInterleavedAccessVectorization(),EnableMaskedInterleavedMemAccesses, andllvm::cl::Option::getNumOccurrences().

Referenced byllvm::LoopVectorizationCostModel::computeMaxVF(),llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(),llvm::LoopVectorizationPlanner::plan(), andllvm::LoopVectorizePass::processLoop().

◆ willGenerateVectors()

staticbool willGenerateVectors(VPlan & Plan,
ElementCount VF,
constTargetTransformInfo & TTI 
)
static

Check if any recipe ofPlan will generate a vector value, which will be assigned a vector register.

Definition at line4439 of fileLoopVectorize.cpp.

Referencesassert(),llvm::collectEphemeralRecipesForVPlan(),llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::contains(),llvm::VPlan::getCanonicalIV(),llvm::VPRegionBlock::getEntry(),llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(),llvm::TargetTransformInfo::getNumberOfParts(),llvm::VPCanonicalIVPHIRecipe::getScalarType(),llvm::VPlan::getVectorLoopRegion(),llvm::VPTypeAnalysis::inferScalarType(),llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(),llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(),llvm::ElementCount::isVector(),llvm_unreachable,llvm::toVectorTy(), andllvm::vp_depth_first_shallow().

Referenced byllvm::LoopVectorizationPlanner::computeBestVF().

Variable Documentation

◆ EnableCondStoresVectorization

cl::opt<bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden,cl::desc("Enableif predication ofstores during vectorization."))("enable-cond-stores-vec" ,
cl::init(true) ,
cl::Hidden ,
cl::desc("Enableif predication ofstores during vectorization.")  
)
static

◆ EnableEarlyExitVectorization

cl::opt<bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(false), cl::Hidden,cl::desc( "Enable vectorization of early exitloops with uncountable exits."))("enable-early-exit-vectorization" ,
cl::init(false) ,
cl::Hidden ,
cl::desc( "Enable vectorization of early exitloops with uncountable exits.")  
)
static

Referenced byllvm::LoopVectorizePass::processLoop(), andllvm::LoopVectorizationCostModel::requiresScalarEpilogue().

◆ EnableEpilogueVectorization

cl::opt<bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden,cl::desc("Enable vectorization of epilogue loops."))("enable-epilogue-vectorization" ,
cl::init(true) ,
cl::Hidden ,
cl::desc("Enable vectorization of epilogue loops.")  
)
static

Referenced byllvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor().

◆ EnableIndVarRegisterHeur

cl::opt<bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden,cl::desc("Count the induction variableonly once when interleaving"))("enable-ind-var-reg-heur" ,
cl::init(true) ,
cl::Hidden ,
cl::desc("Count the induction variableonly once when interleaving")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ EnableInterleavedMemAccesses

cl::opt<bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,cl::desc("Enable vectorization on interleaved memory accesses in a loop"))("enable-interleaved-mem-accesses" ,
cl::init(false) ,
cl::Hidden ,
cl::desc("Enable vectorization on interleaved memory accesses in a loop")  
)
static

Referenced byllvm::LoopVectorizePass::processLoop().

◆ EnableLoadStoreRuntimeInterleave

cl::opt<bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,cl::desc( "Enable runtime interleaving untilload/store ports are saturated"))("enable-loadstore-runtime-interleave" ,
cl::init(true) ,
cl::Hidden ,
cl::desc( "Enable runtime interleaving untilload/store ports are saturated")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ EnableMaskedInterleavedMemAccesses

cl::opt<bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))("enable-masked-interleaved-mem-accesses" ,
cl::init(false) ,
cl::Hidden ,
cl::desc("Enable vectorization on masked interleaved memory accesses in a loop")  
)
static

An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.

Referenced byuseMaskedInterleavedAccesses().

◆ EpilogueVectorizationForceVF

cl::opt<unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,cl::desc("When epilogue vectorization is enabled, and avalue greater than " "1 is specified, forces the given VFfor all applicable epilogue " "loops."))("epilogue-vectorization-force-VF" ,
cl::init(1) ,
cl::Hidden ,
cl::desc("When epilogue vectorization is enabled, and avalue greater than " "1 is specified, forces the given VFfor all applicable epilogue " "loops.")  
)
static

Referenced byllvm::LoopVectorizationPlanner::selectEpilogueVectorizationFactor().

◆ EpilogueVectorizationMinVF

cl::opt<unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden,cl::desc("Onlyloops with vectorization factor equal to or larger than " "the specifiedvalue are consideredfor epilogue vectorization."))("epilogue-vectorization-minimum-VF" ,
cl::Hidden ,
cl::desc("Onlyloops with vectorization factor equal to or larger than " "the specifiedvalue are consideredfor epilogue vectorization.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::isEpilogueVectorizationProfitable().

◆ ForceOrderedReductions

cl::opt<bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden,cl::desc("Enable the vectorisation ofloops with in-order (strict) " "FP reductions"))("force-ordered-reductions" ,
cl::init(false) ,
cl::Hidden ,
cl::desc("Enable the vectorisation ofloops with in-order (strict) " "FP reductions")  
)
static

Referenced byllvm::LoopVectorizePass::processLoop().

◆ ForceSafeDivisor

cl::opt<cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden,cl::desc( "Override cost based safe divisor wideningfor div/rem instructions"))("force-widen-divrem-via-safe-divisor" ,
cl::Hidden ,
cl::desc( "Override cost based safe divisor wideningfor div/rem instructions")  
)
static

Referenced byllvm::LoopVectorizationCostModel::isDivRemScalarWithPredication().

◆ ForceTailFoldingStyle

cl::opt<TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style",cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None),cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),clEnumValN( TailFoldingStyle::Data, "data", "Create lane maskfordataonly, using active.lane.mask intrinsic"),clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"),clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, anduse " "itfor bothdata and control flow"),clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar todata-and-control, but remove the runtime check"),clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVLinstructionsfor tail folding. If EVL " "isunsupported, fallback todata-without-lane-mask.")))("force-tail-folding-style" ,
cl::desc("Force the tail folding style") ,
cl::init(TailFoldingStyle::None) ,
cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),clEnumValN( TailFoldingStyle::Data, "data", "Create lane maskfordataonly, using active.lane.mask intrinsic"),clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"),clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, anduse " "itfor bothdata and control flow"),clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar todata-and-control, but remove the runtime check"),clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVLinstructionsfor tail folding. If EVL " "isunsupported, fallback todata-without-lane-mask."))  
)
static

Referenced byllvm::LoopVectorizationCostModel::setTailFoldingStyles().

◆ ForceTargetInstructionCost

cl::opt<unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's expected costfor " "an instruction to a single constant value. Mostly " "usefulfor getting consistent testing."))("force-target-instruction-cost" ,
cl::init(0) ,
cl::Hidden ,
cl::desc("A flag that overrides the target's expected costfor " "an instruction to a single constant value. Mostly " "usefulfor getting consistent testing.")  
)

Referenced byllvm::VPReductionRecipe::computeCost(),llvm::VPRecipeBase::cost(),llvm::VPRegionBlock::cost(),llvm::LoopVectorizationCostModel::expectedCost(), andllvm::VPCostContext::getLegacyCost().

◆ ForceTargetMaxScalarInterleaveFactor

cl::opt<unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's max interleave factorfor " "scalar loops."))("force-target-max-scalar-interleave" ,
cl::init(0) ,
cl::Hidden ,
cl::desc("A flag that overrides the target's max interleave factorfor " "scalar loops.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ ForceTargetMaxVectorInterleaveFactor

cl::opt<unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's max interleave factorfor " "vectorized loops."))("force-target-max-vector-interleave" ,
cl::init(0) ,
cl::Hidden ,
cl::desc("A flag that overrides the target's max interleave factorfor " "vectorized loops.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ ForceTargetNumScalarRegs

cl::opt<unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's number of scalar registers."))("force-target-num-scalar-regs" ,
cl::init(0) ,
cl::Hidden ,
cl::desc("A flag that overrides the target's number of scalar registers.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ ForceTargetNumVectorRegs

cl::opt<unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden,cl::desc("A flag that overrides the target's number ofvector registers."))("force-target-num-vector-regs" ,
cl::init(0) ,
cl::Hidden ,
cl::desc("A flag that overrides the target's number ofvector registers.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ ForceTargetSupportsScalableVectors

cl::opt<bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,cl::desc( "Pretend that scalablevectors are supported, evenif the target does " "not support them. This flag shouldonly be usedfor testing."))("force-target-supports-scalable-vectors" ,
cl::init(false) ,
cl::Hidden ,
cl::desc( "Pretend that scalablevectors are supported, evenif the target does " "not support them. This flag shouldonly be usedfor testing.")  
)
static

Referenced byllvm::LoopVectorizationPlanner::planInVPlanNativePath().

◆ LLVMLoopVectorizeFollowupAll

constchar LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all"

Metadata attribute names

Definition at line167 of fileLoopVectorize.cpp.

Referenced byllvm::LoopVectorizationPlanner::executePlan(), andllvm::LoopVectorizePass::processLoop().

◆ LLVMLoopVectorizeFollowupEpilogue

constchar LLVMLoopVectorizeFollowupEpilogue[]
Initial value:
=
"llvm.loop.vectorize.followup_epilogue"

Definition at line170 of fileLoopVectorize.cpp.

Referenced byllvm::LoopVectorizePass::processLoop().

◆ LLVMLoopVectorizeFollowupVectorized

constchar LLVMLoopVectorizeFollowupVectorized[]
Initial value:
=
"llvm.loop.vectorize.followup_vectorized"

Definition at line168 of fileLoopVectorize.cpp.

Referenced byllvm::LoopVectorizationPlanner::executePlan().

◆ LoopVectorizeWithBlockFrequency

cl::opt<bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,cl::desc("Enable theuse of theblock frequencyanalysis to accessPGO " "heuristics minimizing code growth in coldregions and being more " "aggressive in hot regions."))("loop-vectorize-with-block-frequency" ,
cl::init(true) ,
cl::Hidden ,
cl::desc("Enable theuse of theblock frequencyanalysis to accessPGO " "heuristics minimizing code growth in coldregions and being more " "aggressive in hot regions.")  
)
static

Referenced bygetSmallBestKnownTC().

◆ MaximizeBandwidth

cl::opt<bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))("vectorizer-maximize-bandwidth" ,
cl::init(false) ,
cl::Hidden ,
cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop.")  
)
static

◆ MaxNestedScalarReductionIC

cl::opt<unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,cl::desc("The maximum interleave count touse when interleaving a scalar " "reduction in a nested loop."))("max-nested-scalar-reduction-interleave" ,
cl::init(2) ,
cl::Hidden ,
cl::desc("The maximum interleave count touse when interleaving a scalar " "reduction in a nested loop.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ MemCheckBypassWeights

constexpruint32_t MemCheckBypassWeights[] = {1, 127}
staticconstexpr

Definition at line398 of fileLoopVectorize.cpp.

◆ MinItersBypassWeights

constexpruint32_t MinItersBypassWeights[] = {1, 127}
staticconstexpr

Definition at line401 of fileLoopVectorize.cpp.

Referenced byllvm::InnerLoopVectorizer::emitIterationCountCheck(), andllvm::EpilogueVectorizerMainLoop::emitIterationCountCheck().

◆ NumberOfStoresToPredicate

cl::opt<unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden,cl::desc("Max number ofstores to be predicated behind an if."))("vectorize-num-stores-pred" ,
cl::init(1) ,
cl::Hidden ,
cl::desc("Max number ofstores to be predicated behind an if.")  
)
static

The number of stores in a loop that are allowed to need predication.

◆ PreferInLoopReductions

cl::opt<bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden,cl::desc("Prefer in-loopvectorreductions, " "overriding the targets preference."))("prefer-inloop-reductions" ,
cl::init(false) ,
cl::Hidden ,
cl::desc("Prefer in-loopvectorreductions, " "overriding the targets preference.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::collectElementTypesForWidening(), andllvm::LoopVectorizationCostModel::collectInLoopReductions().

◆ PreferPredicatedReductionSelect

cl::opt<bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden,cl::desc( "Prefer predicating areduction operation over an after loop select."))("prefer-predicated-reduction-select" ,
cl::init(false) ,
cl::Hidden ,
cl::desc( "Prefer predicating areduction operation over an after loop select.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::usePredicatedReductionSelect().

◆ PreferPredicateOverEpilogue

cl::opt<PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."),cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicateloops, create scalar epilogue"),clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogueif tail " "folding fails."),clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorizationif " "tail-folding fails.")))("prefer-predicate-over-epilogue" ,
cl::init(PreferPredicateTy::ScalarEpilogue) ,
cl::Hidden ,
cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop.") ,
cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicateloops, create scalar epilogue"),clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogueif tail " "folding fails."),clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorizationif " "tail-folding fails."))  
)
static

Referenced bygetScalarEpilogueLowering().

◆ SCEVCheckBypassWeights

constexpruint32_t SCEVCheckBypassWeights[] = {1, 127}
staticconstexpr

Definition at line395 of fileLoopVectorize.cpp.

◆ SmallLoopCost

cl::opt<unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden,cl::desc( "The cost of a loop that is considered 'small' by the interleaver."))("small-loop-cost" ,
cl::init(20) ,
cl::Hidden ,
cl::desc( "The cost of a loop that is considered 'small' by the interleaver.")  
)
static

Referenced byllvm::LoopVectorizationCostModel::selectInterleaveCount().

◆ TinyTripCountVectorThreshold

cl::opt<unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden,cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorizedonlyif no scalar iteration overheads " "are incurred."))("vectorizer-min-trip-count" ,
cl::init(16) ,
cl::Hidden ,
cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorizedonlyif no scalar iteration overheads " "are incurred.")  
)
static

Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.

Referenced byllvm::LoopVectorizePass::processLoop().

◆ UseWiderVFIfCallVariantsPresent

cl::opt<bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden,cl::desc("Try wider VFsif they enable theuse ofvector variants"))("vectorizer-maximize-bandwidth-for-vector-calls" ,
cl::init(true) ,
cl::Hidden ,
cl::desc("Try wider VFsif they enable theuse ofvector variants")  
)
static

◆ VectorizeMemoryCheckThreshold

cl::opt<unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden,cl::desc("The maximum allowed number of runtime memory checks"))("vectorize-memory-check-threshold" ,
cl::init(128) ,
cl::Hidden ,
cl::desc("The maximum allowed number of runtime memory checks")  
)
static

Referenced byareRuntimeChecksProfitable().

◆ VerboseDebug

constchar VerboseDebug[] =DEBUG_TYPE "-verbose"

Definition at line162 of fileLoopVectorize.cpp.

◆ VPlanBuildStressTest

cl::opt<bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden,cl::desc( "BuildVPlanfor every supported loop nest in thefunction and bail " "out right after the build (stresstest theVPlanH-CFG construction " "in theVPlan-native vectorization path)."))("vplan-build-stress-test" ,
cl::init(false) ,
cl::Hidden ,
cl::desc( "BuildVPlanfor every supported loop nest in thefunction and bail " "out right after the build (stresstest theVPlanH-CFG construction " "in theVPlan-native vectorization path).")  
)
static

Referenced bycollectSupportedLoops(),llvm::LoopVectorizationPlanner::planInVPlanNativePath(), andprocessLoopInVPlanNativePath().

Generated on Sun Jul 20 2025 15:03:42 for LLVM by doxygen 1.9.6
[8]ページ先頭
©2009-2025 Movatter.jp