1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7//===----------------------------------------------------------------------===// 9// This implements the SelectionDAG class. 11//===----------------------------------------------------------------------===// 89/// makeVTList - Return an instance of the SDVTList struct initialized with the 96// Default null implementations of the callbacks. 101void SelectionDAG::DAGNodeDeletedListener::anchor() {}
102void SelectionDAG::DAGNodeInsertedListener::anchor() {}
104#define DEBUG_TYPE "selectiondag" 108cl::desc(
"Gang up loads and stores generated by inlining of memcpy"));
111cl::desc(
"Number limit for gluing ld/st of memcpy."),
116cl::desc(
"DAG combiner limit number of steps when searching DAG " 117"for predecessor nodes"));
125//===----------------------------------------------------------------------===// 126// ConstantFPSDNode Class 127//===----------------------------------------------------------------------===// 129/// isExactlyValue - We don't rely on operator== working on double values, as 130/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 131/// As such, this method can be used to do an exact bit-for-bit comparison of 132/// two floating point values. 134return getValueAPF().bitwiseIsEqual(V);
141// convert modifies in place, so make a copy. 149//===----------------------------------------------------------------------===// 151//===----------------------------------------------------------------------===// 156N->getValueType(0).getVectorElementType().getSizeInBits();
157if (
auto *Op0 = dyn_cast<ConstantSDNode>(
N->getOperand(0))) {
158 SplatVal = Op0->getAPIntValue().
trunc(EltSize);
161if (
auto *Op0 = dyn_cast<ConstantFPSDNode>(
N->getOperand(0))) {
162 SplatVal = Op0->getValueAPF().bitcastToAPInt().
trunc(EltSize);
167auto *BV = dyn_cast<BuildVectorSDNode>(
N);
172unsigned SplatBitSize;
174unsigned EltSize =
N->getValueType(0).getVectorElementType().getSizeInBits();
175// Endianness does not matter here. We are checking for a splat given the 176// element size of the vector, and if we find such a splat for little endian 177// layout, then that should be valid also for big endian (as the full vector 178// size is known to be a multiple of the element size). 179constbool IsBigEndian =
false;
180return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
181 EltSize, IsBigEndian) &&
182 EltSize == SplatBitSize;
185// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be 186// specializations of the more general isConstantSplatVector()? 189// Look through a bit convert. 191N =
N->getOperand(0).getNode();
200unsigned i = 0, e =
N->getNumOperands();
202// Skip over all of the undef values. 203while (i != e &&
N->getOperand(i).isUndef())
206// Do not accept an all-undef vector. 207if (i == e)
returnfalse;
209// Do not accept build_vectors that aren't all constants or which have non-~0 210// elements. We have to be a bit careful here, as the type of the constant 211// may not be the same as the type of the vector elements due to type 212// legalization (the elements are promoted to a legal type for the target and 213// a vector of a type may be legal when the base element type is not). 214// We only want to check enough bits to cover the vector elements, because 215// we care if the resultant vector is all ones, not whether the individual 218unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
220if (CN->getAPIntValue().countr_one() < EltSize)
223if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
228// Okay, we have at least one ~0 value, check to see if the rest match or are 229// undefs. Even with the above element type twiddling, this should be OK, as 230// the same type legalization should have applied to all the elements. 231for (++i; i != e; ++i)
232if (
N->getOperand(i) != NotZero && !
N->getOperand(i).isUndef())
238// Look through a bit convert. 240N =
N->getOperand(0).getNode();
249bool IsAllUndef =
true;
254// Do not accept build_vectors that aren't all constants or which have non-0 255// elements. We have to be a bit careful here, as the type of the constant 256// may not be the same as the type of the vector elements due to type 257// legalization (the elements are promoted to a legal type for the target 258// and a vector of a type may be legal when the base element type is not). 259// We only want to check enough bits to cover the vector elements, because 260// we care if the resultant vector is all zeros, not whether the individual 262unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
264if (CN->getAPIntValue().countr_zero() < EltSize)
267if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
273// Do not accept an all-undef vector. 294if (!isa<ConstantSDNode>(
Op))
307if (!isa<ConstantFPSDNode>(
Op))
315assert(
N->getValueType(0).isVector() &&
"Expected a vector!");
317unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
318if (EltSize <= NewEltSize)
322return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
327return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
337if (!isa<ConstantSDNode>(
Op))
340APIntC =
Op->getAsAPIntVal().trunc(EltSize);
341if (
Signed &&
C.trunc(NewEltSize).sext(EltSize) !=
C)
343if (!
Signed &&
C.trunc(NewEltSize).zext(EltSize) !=
C)
351// Return false if the node has no operands. 352// This is "logically inconsistent" with the definition of "all" but 353// is probably the desired behavior. 354if (
N->getNumOperands() == 0)
360returnN->getOpcode() ==
ISD::FREEZE &&
N->getOperand(0).isUndef();
363template <
typename ConstNodeType>
365 std::function<
bool(ConstNodeType *)>
Match,
367// FIXME: Add support for scalar UNDEF cases? 368if (
auto *
C = dyn_cast<ConstNodeType>(
Op))
371// FIXME: Add support for vector UNDEF cases? 376EVT SVT =
Op.getValueType().getScalarType();
378if (AllowUndefs &&
Op.getOperand(i).isUndef()) {
384auto *Cst = dyn_cast<ConstNodeType>(
Op.getOperand(i));
385if (!Cst || Cst->getValueType(0) != SVT || !
Match(Cst))
390// Build used template types. 391templatebool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
393templatebool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
399bool AllowUndefs,
bool AllowTypeMismatch) {
400if (!AllowTypeMismatch &&
LHS.getValueType() !=
RHS.getValueType())
403// TODO: Add support for scalar UNDEF cases? 404if (
auto *LHSCst = dyn_cast<ConstantSDNode>(
LHS))
405if (
auto *RHSCst = dyn_cast<ConstantSDNode>(
RHS))
406returnMatch(LHSCst, RHSCst);
408// TODO: Add support for vector UNDEF cases? 409if (
LHS.getOpcode() !=
RHS.getOpcode() ||
414EVT SVT =
LHS.getValueType().getScalarType();
415for (
unsigned i = 0, e =
LHS.getNumOperands(); i != e; ++i) {
418bool LHSUndef = AllowUndefs && LHSOp.
isUndef();
419bool RHSUndef = AllowUndefs && RHSOp.
isUndef();
420auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
421auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
422if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
424if (!AllowTypeMismatch && (LHSOp.
getValueType() != SVT ||
427if (!
Match(LHSCst, RHSCst))
449switch (VecReduceOpcode) {
454case ISD::VP_REDUCE_FADD:
455case ISD::VP_REDUCE_SEQ_FADD:
459case ISD::VP_REDUCE_FMUL:
460case ISD::VP_REDUCE_SEQ_FMUL:
463case ISD::VP_REDUCE_ADD:
466case ISD::VP_REDUCE_MUL:
469case ISD::VP_REDUCE_AND:
472case ISD::VP_REDUCE_OR:
475case ISD::VP_REDUCE_XOR:
478case ISD::VP_REDUCE_SMAX:
481case ISD::VP_REDUCE_SMIN:
484case ISD::VP_REDUCE_UMAX:
487case ISD::VP_REDUCE_UMIN:
490case ISD::VP_REDUCE_FMAX:
493case ISD::VP_REDUCE_FMIN:
496case ISD::VP_REDUCE_FMAXIMUM:
499case ISD::VP_REDUCE_FMINIMUM:
508#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \ 511#include "llvm/IR/VPIntrinsics.def" 519#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: 520#define VP_PROPERTY_BINARYOP return true; 521#define END_REGISTER_VP_SDNODE(VPSD) break; 522#include "llvm/IR/VPIntrinsics.def" 531case ISD::VP_REDUCE_ADD:
532case ISD::VP_REDUCE_MUL:
533case ISD::VP_REDUCE_AND:
534case ISD::VP_REDUCE_OR:
535case ISD::VP_REDUCE_XOR:
536case ISD::VP_REDUCE_SMAX:
537case ISD::VP_REDUCE_SMIN:
538case ISD::VP_REDUCE_UMAX:
539case ISD::VP_REDUCE_UMIN:
540case ISD::VP_REDUCE_FMAX:
541case ISD::VP_REDUCE_FMIN:
542case ISD::VP_REDUCE_FMAXIMUM:
543case ISD::VP_REDUCE_FMINIMUM:
544case ISD::VP_REDUCE_FADD:
545case ISD::VP_REDUCE_FMUL:
546case ISD::VP_REDUCE_SEQ_FADD:
547case ISD::VP_REDUCE_SEQ_FMUL:
552/// The operand position of the vector mask. 557#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \ 560#include "llvm/IR/VPIntrinsics.def" 564/// The operand position of the explicit vector length parameter. 569#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ 572#include "llvm/IR/VPIntrinsics.def" 578// FIXME: Return strict opcodes in case of fp exceptions. 582#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC: 583#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC; 584#define END_REGISTER_VP_SDNODE(VPOPC) break; 585#include "llvm/IR/VPIntrinsics.def" 594#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break; 595#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC: 596#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC; 597#include "llvm/IR/VPIntrinsics.def" 617// To perform this operation, we just need to swap the L and G bits of the 622 (OldL << 1) |
// New G bit 623 (OldG << 2));
// New L bit. 629Operation ^= 7;
// Flip L, G, E bits, but not U. 631Operation ^= 15;
// Flip all of the condition bits. 634Operation &= ~8;
// Don't let N and U bits get set. 648/// For an integer comparison, return 1 if the comparison is a signed operation 649/// and 2 if the result is an unsigned comparison. Return zero if the operation 650/// does not depend on the sign of the input (setne and seteq). 669bool IsInteger =
Type.isInteger();
671// Cannot fold a signed integer setcc with an unsigned integer setcc. 674unsignedOp = Op1 | Op2;
// Combine all of the condition bits. 676// If the N and U bits get set, then the resultant comparison DOES suddenly 677// care about orderedness, and it is true when ordered. 679Op &= ~16;
// Clear the U bit if the N bit is set. 681// Canonicalize illegal integer setcc's. 690bool IsInteger =
Type.isInteger();
692// Cannot fold a signed setcc with an unsigned setcc. 695// Combine all of the condition bits. 698// Canonicalize illegal integer setcc's. 713//===----------------------------------------------------------------------===// 714// SDNode Profile Support 715//===----------------------------------------------------------------------===// 717/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 722/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 723/// solely with their pointer. 725ID.AddPointer(VTList.
VTs);
728/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 731for (
constauto &
Op : Ops) {
732ID.AddPointer(
Op.getNode());
733ID.AddInteger(
Op.getResNo());
737/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 740for (
constauto &
Op : Ops) {
741ID.AddPointer(
Op.getNode());
742ID.AddInteger(
Op.getResNo());
753/// If this is an SDNode with special info, add this info to the NodeID data. 755switch (
N->getOpcode()) {
760default:
break;
// Normal nodes don't need extra info. 764ID.AddPointer(
C->getConstantIntValue());
765ID.AddBoolean(
C->isOpaque());
770ID.AddPointer(cast<ConstantFPSDNode>(
N)->getConstantFPValue());
786ID.AddInteger(cast<RegisterSDNode>(
N)->
getReg().
id());
789ID.AddPointer(cast<RegisterMaskSDNode>(
N)->getRegMask());
792ID.AddPointer(cast<SrcValueSDNode>(
N)->getValue());
796ID.AddInteger(cast<FrameIndexSDNode>(
N)->getIndex());
800if (cast<LifetimeSDNode>(
N)->hasOffset()) {
801ID.AddInteger(cast<LifetimeSDNode>(
N)->
getSize());
806ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getGuid());
807ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getIndex());
808ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getAttributes());
812ID.AddInteger(cast<JumpTableSDNode>(
N)->getIndex());
813ID.AddInteger(cast<JumpTableSDNode>(
N)->getTargetFlags());
818ID.AddInteger(CP->getAlign().value());
819ID.AddInteger(CP->getOffset());
820if (CP->isMachineConstantPoolEntry())
821 CP->getMachineCPVal()->addSelectionDAGCSEId(
ID);
823ID.AddPointer(CP->getConstVal());
824ID.AddInteger(CP->getTargetFlags());
836ID.AddInteger(LD->getMemoryVT().getRawBits());
837ID.AddInteger(LD->getRawSubclassData());
838ID.AddInteger(LD->getPointerInfo().getAddrSpace());
839ID.AddInteger(LD->getMemOperand()->getFlags());
844ID.AddInteger(ST->getMemoryVT().getRawBits());
845ID.AddInteger(ST->getRawSubclassData());
846ID.AddInteger(ST->getPointerInfo().getAddrSpace());
847ID.AddInteger(ST->getMemOperand()->getFlags());
866case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
873case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
880case ISD::VP_GATHER: {
888case ISD::VP_SCATTER: {
977// Handled by MemIntrinsicSDNode check after the switch. 979 }
// end switch (N->getOpcode()) 981// MemIntrinsic nodes could also have subclass data, address spaces, and flags 983if (
auto *MN = dyn_cast<MemIntrinsicSDNode>(
N)) {
984ID.AddInteger(MN->getRawSubclassData());
985ID.AddInteger(MN->getPointerInfo().getAddrSpace());
986ID.AddInteger(MN->getMemOperand()->getFlags());
987ID.AddInteger(MN->getMemoryVT().getRawBits());
991/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 995// Add the return value info. 997// Add the operand info. 1000// Handle SDNode leafs with special info. 1004//===----------------------------------------------------------------------===// 1005// SelectionDAG Class 1006//===----------------------------------------------------------------------===// 1008/// doNotCSE - Return true if CSE should not be performed for this node. 1010if (
N->getValueType(0) == MVT::Glue)
1011returntrue;
// Never CSE anything that produces a glue result. 1013switch (
N->getOpcode()) {
1017returntrue;
// Never CSE these nodes. 1020// Check that remaining values produced are not flags. 1021for (
unsigned i = 1, e =
N->getNumValues(); i != e; ++i)
1022if (
N->getValueType(i) == MVT::Glue)
1023returntrue;
// Never CSE anything that produces a glue result. 1028/// RemoveDeadNodes - This method deletes all unreachable nodes in the 1031// Create a dummy node (which is not added to allnodes), that adds a reference 1032// to the root node, preventing it from being deleted. 1037// Add all obviously-dead nodes to the DeadNodes worklist. 1039if (Node.use_empty())
1044// If the root changed (e.g. it was a dead load, update the root). 1048/// RemoveDeadNodes - This method deletes the unreachable nodes in the 1049/// given list, and any nodes that become unreachable as a result. 1052// Process the worklist, deleting the nodes and adding their uses to the 1054while (!DeadNodes.
empty()) {
1056// Skip to next node if we've already managed to delete the node. This could 1057// happen if replacing a node causes a node previously added to the node to 1063 DUL->NodeDeleted(
N,
nullptr);
1065// Take the node out of the appropriate CSE map. 1066 RemoveNodeFromCSEMaps(
N);
1068// Next, brutally remove the operand list. This is safe to do, as there are 1069// no cycles in the graph. 1075// Now that we removed this operand, see if there are no uses of it left. 1087// Create a dummy node that adds a reference to the root node, preventing 1088// it from being deleted. (This matters if the root is an operand of the 1096// First take this out of the appropriate CSE map. 1097 RemoveNodeFromCSEMaps(
N);
1099// Finally, remove uses due to operands of this node, remove from the 1100// AllNodes list, and delete the node. 1101 DeleteNodeNotInCSEMaps(
N);
1104void SelectionDAG::DeleteNodeNotInCSEMaps(
SDNode *
N) {
1105assert(
N->getIterator() != AllNodes.begin() &&
1106"Cannot delete the entry node!");
1107assert(
N->use_empty() &&
"Cannot delete a node that is not dead!");
1109// Drop all of the operands and decrement used node's use counts. 1116assert(!(V->isVariadic() && isParameter));
1118 ByvalParmDbgValues.push_back(V);
1120 DbgValues.push_back(V);
1121for (
constSDNode *Node : V->getSDNodes())
1123 DbgValMap[Node].push_back(V);
1128if (
I == DbgValMap.end())
1130for (
auto &Val:
I->second)
1131 Val->setIsInvalidated();
1135void SelectionDAG::DeallocateNode(
SDNode *
N) {
1136// If we have operands, deallocate them. 1141// Set the opcode to DELETED_NODE to help catch bugs when node 1142// memory is reallocated. 1143// FIXME: There are places in SDag that have grown a dependency on the opcode 1144// value in the released node. 1148// If any of the SDDbgValue nodes refer to this SDNode, invalidate 1149// them and forget about that node. 1152// Invalidate extra info. 1157/// VerifySDNode - Check the given SDNode. Aborts if it is invalid. 1159switch (
N->getOpcode()) {
1165EVT VT =
N->getValueType(0);
1166assert(
N->getNumValues() == 1 &&
"Too many results!");
1168"Wrong return type!");
1169assert(
N->getNumOperands() == 2 &&
"Wrong number of operands!");
1170assert(
N->getOperand(0).getValueType() ==
N->getOperand(1).getValueType() &&
1171"Mismatched operand types!");
1173"Wrong operand type!");
1175"Wrong return type size");
1179assert(
N->getNumValues() == 1 &&
"Too many results!");
1180assert(
N->getValueType(0).isVector() &&
"Wrong return type!");
1181assert(
N->getNumOperands() ==
N->getValueType(0).getVectorNumElements() &&
1182"Wrong number of operands!");
1183EVT EltVT =
N->getValueType(0).getVectorElementType();
1185assert((
Op.getValueType() == EltVT ||
1186 (EltVT.
isInteger() &&
Op.getValueType().isInteger() &&
1187 EltVT.
bitsLE(
Op.getValueType()))) &&
1188"Wrong operand type!");
1189assert(
Op.getValueType() ==
N->getOperand(0).getValueType() &&
1190"Operands must all have the same type");
1198/// Insert a newly allocated node into the DAG. 1200/// Handles insertion into the all nodes list and CSE map, as well as 1201/// verification and other common operations when a new node is allocated. 1202void SelectionDAG::InsertNode(
SDNode *
N) {
1203 AllNodes.push_back(
N);
1205N->PersistentId = NextPersistentId++;
1208for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1209 DUL->NodeInserted(
N);
1212/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 1213/// correspond to it. This is useful when we're about to delete or repurpose 1214/// the node. We don't want future request for structurally identical nodes 1215/// to return N anymore. 1216bool SelectionDAG::RemoveNodeFromCSEMaps(
SDNode *
N) {
1218switch (
N->getOpcode()) {
1221assert(CondCodeNodes[cast<CondCodeSDNode>(
N)->
get()] &&
1222"Cond code doesn't exist!");
1223 Erased = CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] !=
nullptr;
1224 CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] =
nullptr;
1227 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(
N)->getSymbol());
1231 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1236auto *MCSN = cast<MCSymbolSDNode>(
N);
1237 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1241EVT VT = cast<VTSDNode>(
N)->getVT();
1243 Erased = ExtendedValueTypeNodes.erase(VT);
1251// Remove it from the CSE Map. 1254 Erased = CSEMap.RemoveNode(
N);
1258// Verify that the node was actually in one of the CSE maps, unless it has a 1259// glue result (which cannot be CSE'd) or is one of the special cases that are 1260// not subject to CSE. 1261if (!Erased &&
N->getValueType(
N->getNumValues()-1) != MVT::Glue &&
1271/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 1272/// maps and modified in place. Add it back to the CSE maps, unless an identical 1273/// node already exists, in which case transfer all its users to the existing 1274/// node. This transfer can potentially trigger recursive merging. 1276SelectionDAG::AddModifiedNodeToCSEMaps(
SDNode *
N) {
1277// For node types that aren't CSE'd, just act as if no identical node 1280SDNode *Existing = CSEMap.GetOrInsertNode(
N);
1282// If there was already an existing matching node, use ReplaceAllUsesWith 1283// to replace the dead one with the existing one. This can cause 1284// recursive merging of other unrelated nodes down the line. 1288// N is now dead. Inform the listeners and delete it. 1289for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1290 DUL->NodeDeleted(
N, Existing);
1291 DeleteNodeNotInCSEMaps(
N);
1296// If the node doesn't already exist, we updated it. Inform listeners. 1297for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1298 DUL->NodeUpdated(
N);
1301/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1302/// were replaced with those specified. If this node is never memoized, 1303/// return null, otherwise return a pointer to the slot it would take. If a 1304/// node already exists with these operands, the slot will be non-null. 1316Node->intersectFlagsWith(
N->getFlags());
1320/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1321/// were replaced with those specified. If this node is never memoized, 1322/// return null, otherwise return a pointer to the slot it would take. If a 1323/// node already exists with these operands, the slot will be non-null. 1336Node->intersectFlagsWith(
N->getFlags());
1340/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 1341/// were replaced with those specified. If this node is never memoized, 1342/// return null, otherwise return a pointer to the slot it would take. If a 1343/// node already exists with these operands, the slot will be non-null. 1354Node->intersectFlagsWith(
N->getFlags());
1365// EntryNode could meaningfully have debug info if we can find it... 1367 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0,
DebugLoc(),
1370 InsertNode(&EntryNode);
1381 SDAGISelPass = PassPtr;
1385 LibInfo = LibraryInfo;
1391 FnVarLocs = VarLocs;
1395assert(!UpdateListeners &&
"Dangling registered DAGUpdateListeners");
1397 OperandRecycler.clear(OperandAllocator);
1405void SelectionDAG::allnodes_clear() {
1406assert(&*AllNodes.begin() == &EntryNode);
1407 AllNodes.remove(AllNodes.begin());
1408while (!AllNodes.empty())
1409 DeallocateNode(&AllNodes.front());
1411 NextPersistentId = 0;
1417SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1419switch (
N->getOpcode()) {
1424"debug location. Use another overload.");
1431constSDLoc &
DL,
void *&InsertPos) {
1432SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1434switch (
N->getOpcode()) {
1437// Erase debug location from the node if the node is used at several 1438// different places. Do not propagate one location to all uses as it 1439// will cause a worse single stepping debugging experience. 1440if (
N->getDebugLoc() !=
DL.getDebugLoc())
1444// When the node's point of use is located earlier in the instruction 1445// sequence than its prior point of use, update its debug info to the 1447if (
DL.getIROrder() &&
DL.getIROrder() <
N->getIROrder())
1448N->setDebugLoc(
DL.getDebugLoc());
1457 OperandRecycler.clear(OperandAllocator);
1458 OperandAllocator.
Reset();
1461 ExtendedValueTypeNodes.clear();
1462 ExternalSymbols.clear();
1463 TargetExternalSymbols.clear();
1466 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
nullptr);
1467 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
nullptr);
1469 EntryNode.UseList =
nullptr;
1470 InsertNode(&EntryNode);
1482std::pair<SDValue, SDValue>
1486"Strict no-op FP extend/round not allowed.");
1493return std::pair<SDValue, SDValue>(Res,
SDValue(Res.
getNode(), 1));
1497return VT.
bitsGT(
Op.getValueType()) ?
1503return VT.
bitsGT(
Op.getValueType()) ?
1509return VT.
bitsGT(
Op.getValueType()) ?
1517autoType =
Op.getValueType();
1521autoSize =
Op.getValueSizeInBits();
1532autoType =
Op.getValueType();
1536autoSize =
Op.getValueSizeInBits();
1547autoType =
Op.getValueType();
1551autoSize =
Op.getValueSizeInBits();
1569EVT OpVT =
Op.getValueType();
1571"Cannot getZeroExtendInReg FP types");
1573"getZeroExtendInReg type should be vector iff the operand " 1577"Vector element counts must match in getZeroExtendInReg");
1589EVT OpVT =
Op.getValueType();
1591"Cannot getVPZeroExtendInReg FP types");
1593"getVPZeroExtendInReg type and operand type should be vector!");
1595"Vector element counts must match in getZeroExtendInReg");
1606// Only unsigned pointer semantics are supported right now. In the future this 1607// might delegate to TLI to check pointer signedness. 1612// Only unsigned pointer semantics are supported right now. In the future this 1613// might delegate to TLI to check pointer signedness. 1621/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 1634returngetNode(ISD::VP_XOR,
DL, VT, Val, TrueValue, Mask, EVL);
1645returngetNode(ISD::VP_ZERO_EXTEND,
DL, VT,
Op, Mask, EVL);
1647returngetNode(ISD::VP_TRUNCATE,
DL, VT,
Op, Mask, EVL);
1674returngetConstant(*ConstantInt::get(*Context, Val),
DL, VT, isT, isO);
1678EVT VT,
bool isT,
bool isO) {
1684// Vector splats are explicit within the DAG, with ConstantSDNode holding the 1685// to-be-splatted scalar ConstantInt. 1686if (isa<VectorType>(Elt->
getType()))
1689// In some cases the vector type is legal but the element type is illegal and 1690// needs to be promoted, for example v8i8 on ARM. In this case, promote the 1691// inserted value (the type does not need to match the vector element type). 1692// Any extra bits introduced will be truncated away. 1701 Elt = ConstantInt::get(*
getContext(), NewVal);
1703// In other cases the element type is illegal and needs to be expanded, for 1704// example v2i64 on MIPS32. In this case, find the nearest legal type, split 1705// the value into n parts and use a vector type with n-times the elements. 1706// Then bitcast to the type requested. 1707// Legalizing constants too early makes the DAGCombiner's job harder so we 1708// only legalize if the DAG tells us we must produce legal types. 1716// For scalable vectors, try to use a SPLAT_VECTOR_PARTS node. 1720"Can only handle an even split!");
1724for (
unsigned i = 0; i != Parts; ++i)
1726 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1727 ViaEltVT, isT, isO));
1732unsigned ViaVecNumElts = VT.
getSizeInBits() / ViaEltSizeInBits;
1735// Check the temporary vector is the correct size. If this fails then 1736// getTypeToTransformTo() probably returned a type whose size (in bits) 1737// isn't a power-of-2 factor of the requested type size. 1743 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1744 ViaEltVT, isT, isO));
1746// EltParts is currently in little endian order. If we actually want 1747// big-endian order then reverse it now. 1749 std::reverse(EltParts.
begin(), EltParts.
end());
1751// The elements must be reversed when the element order is different 1752// to the endianness of the elements (because the BITCAST is itself a 1753// vector shuffle in this situation). However, we do not need any code to 1754// perform this reversal because getConstant() is producing a vector 1756// This situation occurs in MIPS MSA. 1768"APInt size does not match type size!");
1777if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1782N = newSDNode<ConstantSDNode>(isT, isO, Elt, VTs);
1783 CSEMap.InsertNode(
N, IP);
1803 IsTarget, IsOpaque);
1835EVT VT,
bool isTarget) {
1841// Vector splats are explicit within the DAG, with ConstantFPSDNode holding 1842// the to-be-splatted scalar ConstantFP. 1843if (isa<VectorType>(Elt->
getType()))
1846// Do the map lookup using the actual bit pattern for the floating point 1847// value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1848// we don't have issues with SNANs. 1856if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1861N = newSDNode<ConstantFPSDNode>(isTarget, Elt, VTs);
1862 CSEMap.InsertNode(
N, IP);
1876if (EltVT == MVT::f32)
1878if (EltVT == MVT::f64)
1880if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1881 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1893unsigned TargetFlags) {
1894assert((TargetFlags == 0 || isTargetGA) &&
1895"Cannot set target flags on target-independent globals");
1897// Truncate (with sign-extension) the offset value to the pointer size. 1913ID.AddInteger(TargetFlags);
1915if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
1918auto *
N = newSDNode<GlobalAddressSDNode>(
1919 Opc,
DL.getIROrder(),
DL.getDebugLoc(), GV, VTs,
Offset, TargetFlags);
1920 CSEMap.InsertNode(
N, IP);
1932if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1935auto *
N = newSDNode<FrameIndexSDNode>(FI, VTs, isTarget);
1936 CSEMap.InsertNode(
N, IP);
1942unsigned TargetFlags) {
1943assert((TargetFlags == 0 || isTarget) &&
1944"Cannot set target flags on target-independent jump tables");
1950ID.AddInteger(TargetFlags);
1952if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1955auto *
N = newSDNode<JumpTableSDNode>(JTI, VTs, isTarget, TargetFlags);
1956 CSEMap.InsertNode(
N, IP);
1970bool isTarget,
unsigned TargetFlags) {
1971assert((TargetFlags == 0 || isTarget) &&
1972"Cannot set target flags on target-independent globals");
1981ID.AddInteger(Alignment->value());
1984ID.AddInteger(TargetFlags);
1986if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1989auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VTs,
Offset, *Alignment,
1991 CSEMap.InsertNode(
N, IP);
2000bool isTarget,
unsigned TargetFlags) {
2001assert((TargetFlags == 0 || isTarget) &&
2002"Cannot set target flags on target-independent globals");
2009ID.AddInteger(Alignment->value());
2011C->addSelectionDAGCSEId(
ID);
2012ID.AddInteger(TargetFlags);
2014if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2017auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VTs,
Offset, *Alignment,
2019 CSEMap.InsertNode(
N, IP);
2029if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2032auto *
N = newSDNode<BasicBlockSDNode>(
MBB);
2033 CSEMap.InsertNode(
N, IP);
2040 ValueTypeNodes.size())
2047N = newSDNode<VTSDNode>(VT);
2055N = newSDNode<ExternalSymbolSDNode>(
false,
Sym, 0,
getVTList(VT));
2070unsigned TargetFlags) {
2072 TargetExternalSymbols[std::pair<std::string, unsigned>(
Sym, TargetFlags)];
2074N = newSDNode<ExternalSymbolSDNode>(
true,
Sym, TargetFlags,
getVTList(VT));
2080if ((
unsigned)
Cond >= CondCodeNodes.size())
2081 CondCodeNodes.resize(
Cond+1);
2083if (!CondCodeNodes[
Cond]) {
2084auto *
N = newSDNode<CondCodeSDNode>(
Cond);
2085 CondCodeNodes[
Cond] =
N;
2095"APInt size does not match type size!");
2126constAPInt &StepVal) {
2140/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that 2141/// point at N1 to point at N2 and indices that point at N2 to point at N1. 2150"Must have the same number of vector elements as mask elements!");
2152"Invalid VECTOR_SHUFFLE");
2154// Canonicalize shuffle undef, undef -> undef 2158// Validate that all indices in Mask are within the range of the elements 2159// input to the shuffle. 2160int NElts = Mask.size();
2162 [&](
int M) {
return M < (NElts * 2) && M >= -1; }) &&
2163"Index out of range");
2165// Copy the mask so we can do any needed cleanup. 2168// Canonicalize shuffle v, v -> v, undef 2171for (
int i = 0; i != NElts; ++i)
2172if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2175// Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 2180// If shuffling a splat, try to blend the splat instead. We do this here so 2181// that even when this arises during lowering we don't have to re-handle it. 2188for (
int i = 0; i < NElts; ++i) {
2189if (MaskVec[i] <
Offset || MaskVec[i] >= (
Offset + NElts))
2192// If this input comes from undef, mark it as such. 2193if (UndefElements[MaskVec[i] -
Offset]) {
2198// If we can blend a non-undef lane, use that instead. 2199if (!UndefElements[i])
2203if (
auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2204 BlendSplat(N1BV, 0);
2205if (
auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2206 BlendSplat(N2BV, NElts);
2209// Canonicalize all index into lhs, -> shuffle lhs, undef 2210// Canonicalize all index into rhs, -> shuffle rhs, undef 2211bool AllLHS =
true, AllRHS =
true;
2213for (
int i = 0; i != NElts; ++i) {
2214if (MaskVec[i] >= NElts) {
2219 }
elseif (MaskVec[i] >= 0) {
2223if (AllLHS && AllRHS)
2225if (AllLHS && !N2Undef)
2231// Reset our undef status after accounting for the mask. 2233// Re-check whether both sides ended up undef. 2237// If Identity shuffle return that node. 2238bool Identity =
true, AllSame =
true;
2239for (
int i = 0; i != NElts; ++i) {
2240if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity =
false;
2241if (MaskVec[i] != MaskVec[0]) AllSame =
false;
2243if (Identity && NElts)
2246// Shuffling a constant splat doesn't change the result. 2250// Look through any bitcasts. We check that these don't change the number 2251// (and size) of elements and just changes their types. 2253 V = V->getOperand(0);
2255// A splat should always show up as a build vector node. 2256if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2259// If this is a splat of an undef, shuffling it is also undef. 2266// We only have a splat which can skip shuffles if there is a splatted 2267// value and no undef lanes rearranged by the shuffle. 2269// Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the 2270// number of elements match or the value splatted is a zero constant. 2275// If the shuffle itself creates a splat, build the vector directly. 2276if (AllSame && SameNumElts) {
2277EVT BuildVT = BV->getValueType(0);
2281// We may have jumped through bitcasts, so the type of the 2282// BUILD_VECTOR may not match the type of the shuffle. 2294for (
int i = 0; i != NElts; ++i)
2295ID.AddInteger(MaskVec[i]);
2298if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
2301// Allocate the mask array for the node out of the BumpPtrAllocator, since 2302// SDNode doesn't have access to it. This memory will be "leaked" when 2303// the node is deallocated, but recovered when the NodeAllocator is released. 2304int *MaskAlloc = OperandAllocator.
Allocate<
int>(NElts);
2307auto *
N = newSDNode<ShuffleVectorSDNode>(VTs, dl.
getIROrder(),
2309 createOperands(
N, Ops);
2311 CSEMap.InsertNode(
N, IP);
2332ID.AddInteger(Reg.id());
2334if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2337auto *
N = newSDNode<RegisterSDNode>(Reg, VTs);
2339 CSEMap.InsertNode(
N, IP);
2347ID.AddPointer(RegMask);
2349if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2352auto *
N = newSDNode<RegisterMaskSDNode>(RegMask);
2353 CSEMap.InsertNode(
N, IP);
2368ID.AddPointer(Label);
2370if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2375 createOperands(
N, Ops);
2377 CSEMap.InsertNode(
N, IP);
2383 int64_t
Offset,
bool isTarget,
2384unsigned TargetFlags) {
2392ID.AddInteger(TargetFlags);
2394if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2397auto *
N = newSDNode<BlockAddressSDNode>(Opc, VTs, BA,
Offset, TargetFlags);
2398 CSEMap.InsertNode(
N, IP);
2409if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2412auto *
N = newSDNode<SrcValueSDNode>(V);
2413 CSEMap.InsertNode(
N, IP);
2424if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2427auto *
N = newSDNode<MDNodeSDNode>(MD);
2428 CSEMap.InsertNode(
N, IP);
2434if (VT == V.getValueType())
2441unsigned SrcAS,
unsigned DestAS) {
2446ID.AddInteger(SrcAS);
2447ID.AddInteger(DestAS);
2450if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
2454 VTs, SrcAS, DestAS);
2455 createOperands(
N, Ops);
2457 CSEMap.InsertNode(
N, IP);
2466/// getShiftAmountOperand - Return the specified value casted to 2467/// the target's desired shift amount type. 2469EVT OpTy =
Op.getValueType();
2483// Collect all of the subvectors 2484 std::deque<SDValue> Subvectors = {Op1};
2485for (
unsignedI = 0;
I < ScaleFactor;
I++) {
2487 Subvectors.push_back(
2491// Flatten the subvector tree 2492while (Subvectors.size() > 1) {
2493 Subvectors.push_back(
2495 Subvectors.pop_front();
2496 Subvectors.pop_front();
2499assert(Subvectors.size() == 1 &&
2500"There should only be one subvector after tree flattening");
2502return Subvectors[0];
2505/// Given a store node \p StoreNode, return true if it is safe to fold that node 2506/// into \p FPNode, which expands to a library call with output pointers. 2513// Skip FPNode use by StoreNode (that's the use we want to fold into FPNode). 2515if (
Op.getNode() != FPNode)
2519while (!Worklist.
empty()) {
2528// Reached the FPNode (would result in a cycle). 2529// OR Reached CALLSEQ_START (would result in nested call sequences). 2534// Defer looking into call sequences (so we can check we're outside one). 2535// We still need to look through these for the predecessor check. 2544// True if we're outside a call sequence and don't have the FPNode as a 2545// predecessor. No cycles or nested call sequences possible. 2552 std::optional<unsigned> CallRetResNo) {
2554EVT VT = Node->getValueType(0);
2555unsigned NumResults = Node->getNumValues();
2561auto getVecDesc = [&]() ->
VecDescconst * {
2562for (
boolMasked : {
false,
true}) {
2571// For vector types, we must find a vector mapping for the libcall. 2573if (VT.
isVector() && !(VD = getVecDesc()))
2576// Find users of the node that store the results (and share input chains). The 2577// destination pointers can be used instead of creating stack allocations. 2583auto *ST = cast<StoreSDNode>(
User);
2584SDValue StoreValue = ST->getValue();
2585unsigned ResNo = StoreValue.
getResNo();
2586// Ensure the store corresponds to an output pointer. 2587if (CallRetResNo == ResNo)
2589// Ensure the store to the default address space and not atomic or volatile. 2590if (!ST->isSimple() || ST->getAddressSpace() != 0)
2592// Ensure all store chains are the same (so they don't alias). 2593if (StoresInChain && ST->getChain() != StoresInChain)
2595// Ensure the store is properly aligned. 2601// 1. Creating cyclic dependencies. 2602// 2. Expanding the node to a call within a call sequence. 2605 ResultStores[ResNo] = ST;
2606 StoresInChain = ST->getChain();
2610auto AddArgListEntry = [&](
SDValue Node,
Type *Ty) {
2614 Args.push_back(Entry);
2617// Pass the arguments. 2618for (
constSDValue &
Op : Node->op_values()) {
2619EVT ArgVT =
Op.getValueType();
2621 AddArgListEntry(
Op, ArgTy);
2624// Pass the output pointers. 2628if (ResNo == CallRetResNo)
2630EVT ResVT = Node->getValueType(ResNo);
2632 ResultPtrs[ResNo] = ResultPtr;
2638// Pass the vector mask (if required). 2645Type *RetType = CallRetResNo.has_value()
2646 ? Node->getValueType(*CallRetResNo).getTypeForEVT(Ctx)
2658if (ResNo == CallRetResNo) {
2664// Replace store with the library call. 2666 PtrInfo = ST->getPointerInfo();
2672getLoad(Node->getValueType(ResNo),
DL, CallChain, ResultPtr, PtrInfo);
2676if (CallRetResNo && !Node->hasAnyUseOfValue(*CallRetResNo)) {
2677// FIXME: Find a way to avoid updating the root. This is needed for x86, 2678// which uses a floating-point stack. If (for example) the node to be 2679// expanded has two results one floating-point which is returned by the 2680// call, and one integer result, returned via an output pointer. If only the 2681// integer result is used then the `CopyFromReg` for the FP result may be 2682// optimized out. This prevents an FP stack pop from being emitted for it. 2683// Setting the root like this ensures there will be a use of the 2684// `CopyFromReg` chain, and ensures the FP pop will be emitted. 2688// Ensure the new root is reachable from the results. 2698constValue *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2699EVT VT = Node->getValueType(0);
2700SDValue Tmp1 = Node->getOperand(0);
2701SDValue Tmp2 = Node->getOperand(1);
2702constMaybeAlign MA(Node->getConstantOperandVal(3));
2717// Increment the pointer, VAList, to the next vaarg 2722// Store the incremented VAList to the legalized pointer 2725// Load the actual argument out of the pointer VAList 2732// This defaults to loading a pointer from the input and storing it to the 2733// output, returning the chain. 2734constValue *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2735constValue *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2746Align RedAlign = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2754// See if we can choose a smaller ABI alignment in cases where it's an 2755// illegal vector type that will get broken down. 2756if (RedAlign > StackAlign) {
2759unsigned NumIntermediates;
2761 NumIntermediates, RegisterVT);
2763Align RedAlign2 = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2764if (RedAlign2 < RedAlign)
2765 RedAlign = RedAlign2;
2768// If the stack is not realignable, the alignment should be limited to the 2770 RedAlign = std::min(RedAlign, StackAlign);
2782// The stack id gives an indication of whether the object is scalable or 2783// not, so it's safe to pass in the minimum size here. 2785false,
nullptr, StackID);
2800"Don't know how to choose the maximum size when creating a stack " 2809AlignAlign = std::max(
DL.getPrefTypeAlign(Ty1),
DL.getPrefTypeAlign(Ty2));
2817auto GetUndefBooleanConstant = [&]() {
2822// ZeroOrOne / ZeroOrNegative require specific values for the high bits, 2823// so we cannot use getUNDEF(). Return zero instead. 2827// These setcc operations always fold. 2850// For EQ and NE, we can always pick a value for the undef to make the 2851// predicate pass or fail, so we can return undef. 2852// Matches behavior in llvm::ConstantFoldCompareInstruction. 2853// icmp eq/ne X, undef -> undef. 2856return GetUndefBooleanConstant();
2858// If both operands are undef, we can return undef for int comparison. 2859// icmp undef, undef -> undef. 2861return GetUndefBooleanConstant();
2863// icmp X, X -> true/false 2864// icmp X, undef -> true/false because undef could be X. 2870constAPInt &C2 = N2C->getAPIntValue();
2872constAPInt &C1 = N1C->getAPIntValue();
2879auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2880auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2882if (N1CFP && N2CFP) {
2887return GetUndefBooleanConstant();
2892return GetUndefBooleanConstant();
2898return GetUndefBooleanConstant();
2903return GetUndefBooleanConstant();
2908return GetUndefBooleanConstant();
2914return GetUndefBooleanConstant();
2939// Ensure that the constant occurs on the RHS. 2943returngetSetCC(dl, VT, N2, N1, SwappedCond);
2944 }
elseif ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2946// If an operand is known to be a nan (or undef that could be a nan), we can 2948// Choosing NaN for the undef will always make unordered comparison succeed 2949// and ordered comparison fails. 2950// Matches behavior in llvm::ConstantFoldCompareInstruction. 2954case 0:
// Known false. 2956case 1:
// Known true. 2959return GetUndefBooleanConstant();
2963// Could not fold it. 2967/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 2968/// use this predicate to simplify operations downstream. 2970unsignedBitWidth =
Op.getScalarValueSizeInBits();
2974/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 2975/// this predicate to simplify operations downstream. Mask is known to be zero 2976/// for bits that V cannot have. 2978unsignedDepth)
const{
2982/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in 2983/// DemandedElts. We use this predicate to simplify operations downstream. 2984/// Mask is known to be zero for bits that V cannot have. 2986constAPInt &DemandedElts,
2987unsignedDepth)
const{
2991/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in 2992/// DemandedElts. We use this predicate to simplify operations downstream. 2994unsignedDepth/* = 0 */)
const{
2998/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. 3000unsignedDepth)
const{
3005constAPInt &DemandedElts,
3006unsignedDepth)
const{
3007EVT VT =
Op.getValueType();
3014for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
3015if (!DemandedElts[EltIdx])
3016continue;
// Don't query elements that are not demanded. 3019 KnownZeroElements.
setBit(EltIdx);
3021return KnownZeroElements;
3024/// isSplatValue - Return true if the vector V has the same value 3025/// across all DemandedElts. For scalable vectors, we don't know the 3026/// number of lanes at compile time. Instead, we use a 1 bit APInt 3027/// to represent a conservative value for all lanes; that is, that 3028/// one bit value is implicitly splatted across all lanes. 3031unsigned Opcode = V.getOpcode();
3032EVT VT = V.getValueType();
3035"scalable demanded bits are ignored");
3038returnfalse;
// No demanded elts, better to assume we don't know anything. 3041returnfalse;
// Limit search depth. 3043// Deal with some common cases here that work for both fixed and scalable 3047 UndefElts = V.getOperand(0).isUndef()
3056APInt UndefLHS, UndefRHS;
3061 UndefElts = UndefLHS | UndefRHS;
3079// We don't support other cases than those above for scalable vectors at 3091for (
unsigned i = 0; i != NumElts; ++i) {
3097if (!DemandedElts[i])
3106// Check if this is a shuffle node doing a splat or a shuffle of a splat. 3109ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
3110for (
int i = 0; i != (int)NumElts; ++i) {
3116if (!DemandedElts[i])
3118if (M < (
int)NumElts)
3121 DemandedRHS.
setBit(M - NumElts);
3124// If we aren't demanding either op, assume there's no splat. 3125// If we are demanding both ops, assume there's no splat. 3130// See if the demanded elts of the source op is a splat or we only demand 3131// one element, which should always be a splat. 3132// TODO: Handle source ops splats with undefs. 3133auto CheckSplatSrc = [&](
SDValue Src,
constAPInt &SrcElts) {
3135return (SrcElts.popcount() == 1) ||
3137 (SrcElts & SrcUndefs).
isZero());
3139if (!DemandedLHS.
isZero())
3140return CheckSplatSrc(V.getOperand(0), DemandedLHS);
3141return CheckSplatSrc(V.getOperand(1), DemandedRHS);
3144// Offset the demanded elts by the subvector index. 3146// We don't support scalable vectors at the moment. 3147if (Src.getValueType().isScalableVector())
3150unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3162// Widen the demanded elts by the src element count. 3164// We don't support scalable vectors at the moment. 3165if (Src.getValueType().isScalableVector())
3167unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3169APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
3171 UndefElts = UndefSrcElts.
trunc(NumElts);
3178EVT SrcVT = Src.getValueType();
3182// Ignore bitcasts from unsupported types. 3183// TODO: Add fp support? 3187// Bitcast 'small element' vector to 'large element' vector. 3188if ((
BitWidth % SrcBitWidth) == 0) {
3189// See if each sub element is a splat. 3190unsigned Scale =
BitWidth / SrcBitWidth;
3192APInt ScaledDemandedElts =
3194for (
unsignedI = 0;
I != Scale; ++
I) {
3198 SubDemandedElts &= ScaledDemandedElts;
3201// TODO: Add support for merging sub undef elements. 3202if (!SubUndefElts.
isZero())
3214/// Helper wrapper to main isSplatValue function. 3216EVT VT = V.getValueType();
3220// Since the number of lanes in a scalable vector is unknown at compile time, 3221// we track one bit which is implicitly broadcast to all lanes. This means 3222// that all lanes in a scalable vector are considered demanded. 3226 (AllowUndefs || !UndefElts);
3232EVT VT = V.getValueType();
3233unsigned Opcode = V.getOpcode();
3237// Since the number of lanes in a scalable vector is unknown at compile time, 3238// we track one bit which is implicitly broadcast to all lanes. This means 3239// that all lanes in a scalable vector are considered demanded. 3245// DemandedElts and UndefElts are ignored for scalable vectors, since 3246// the only supported cases are SPLAT_VECTOR nodes. 3249// Handle case where all demanded elements are UNDEF. 3254 SplatIdx = (UndefElts & DemandedElts).
countr_one();
3265// Check if this is a shuffle node doing a splat. 3266// TODO - remove this and rely purely on SelectionDAG::isSplatValue, 3267// getTargetVShiftNode currently struggles without the splat source. 3268auto *SVN = cast<ShuffleVectorSDNode>(V);
3271intIdx = SVN->getSplatIndex();
3272int NumElts = V.getValueType().getVectorNumElements();
3273 SplatIdx =
Idx % NumElts;
3274return V.getOperand(
Idx / NumElts);
3299std::optional<ConstantRange>
3301unsignedDepth)
const{
3304"Unknown shift node");
3305// Shifting more than the bitwidth is not valid. 3306unsignedBitWidth = V.getScalarValueSizeInBits();
3308if (
auto *Cst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
3309constAPInt &ShAmt = Cst->getAPIntValue();
3315if (
auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1))) {
3316constAPInt *MinAmt =
nullptr, *MaxAmt =
nullptr;
3317for (
unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3318if (!DemandedElts[i])
3320auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3322 MinAmt = MaxAmt =
nullptr;
3325constAPInt &ShAmt = SA->getAPIntValue();
3328if (!MinAmt || MinAmt->
ugt(ShAmt))
3330if (!MaxAmt || MaxAmt->ult(ShAmt))
3333assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3334"Failed to find matching min/max shift amounts");
3335if (MinAmt && MaxAmt)
3339// Use computeKnownBits to find a hidden constant/knownbits (usually type 3340// legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc. 3348std::optional<uint64_t>
3350unsignedDepth)
const{
3353"Unknown shift node");
3354if (std::optional<ConstantRange> AmtRange =
3356if (
constAPInt *ShAmt = AmtRange->getSingleElement())
3357return ShAmt->getZExtValue();
3361std::optional<uint64_t>
3363EVT VT = V.getValueType();
3370std::optional<uint64_t>
3372unsignedDepth)
const{
3375"Unknown shift node");
3376if (std::optional<ConstantRange> AmtRange =
3378return AmtRange->getUnsignedMin().getZExtValue();
3382std::optional<uint64_t>
3384EVT VT = V.getValueType();
3391std::optional<uint64_t>
3393unsignedDepth)
const{
3396"Unknown shift node");
3397if (std::optional<ConstantRange> AmtRange =
3399return AmtRange->getUnsignedMax().getZExtValue();
3403std::optional<uint64_t>
3405EVT VT = V.getValueType();
3412/// Determine which bits of Op are known to be either zero or one and return 3413/// them in Known. For vectors, the known bits are those that are shared by 3414/// every vector element. 3416EVT VT =
Op.getValueType();
3418// Since the number of lanes in a scalable vector is unknown at compile time, 3419// we track one bit which is implicitly broadcast to all lanes. This means 3420// that all lanes in a scalable vector are considered demanded. 3427/// Determine which bits of Op are known to be either zero or one and return 3428/// them in Known. The DemandedElts argument allows us to only collect the known 3429/// bits that are shared by the requested vector elements. 3431unsignedDepth)
const{
3432unsignedBitWidth =
Op.getScalarValueSizeInBits();
3436if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
3437// We know all of the bits for a constant! 3440if (
auto *
C = dyn_cast<ConstantFPSDNode>(
Op)) {
3441// We know all of the bits for a constant fp! 3446return Known;
// Limit search depth. 3450assert((!
Op.getValueType().isFixedLengthVector() ||
3451 NumElts ==
Op.getValueType().getVectorNumElements()) &&
3452"Unexpected vector size");
3455return Known;
// No demanded elts, better to assume we don't know anything. 3457unsigned Opcode =
Op.getOpcode();
3465"Expected SPLAT_VECTOR implicit truncation");
3466// Implicitly truncate the bits to match the official semantics of 3472unsigned ScalarSize =
Op.getOperand(0).getScalarValueSizeInBits();
3474"Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3481constAPInt &Step =
Op.getConstantOperandAPInt(0);
3490constAPInt MinNumElts =
3496 .
umul_ov(MinNumElts, Overflow);
3500constAPInt MaxValue = (MaxNumElts - 1).
umul_ov(Step, Overflow);
3508assert(!
Op.getValueType().isScalableVector());
3509// Collect the known bits that are shared by every demanded vector element. 3512if (!DemandedElts[i])
3518// BUILD_VECTOR can implicitly truncate sources, we must handle this. 3521"Expected BUILD_VECTOR implicit truncation");
3525// Known bits are the values that are shared by every demanded element. 3528// If we don't know any bits, early out. 3534assert(!
Op.getValueType().isScalableVector());
3535// Collect the known bits that are shared by every vector element referenced 3537APInt DemandedLHS, DemandedRHS;
3541 DemandedLHS, DemandedRHS))
3544// Known bits are the values that are shared by every demanded element. 3551// If we don't know any bits, early out. 3563constAPInt &Multiplier =
Op.getConstantOperandAPInt(0);
3568if (
Op.getValueType().isScalableVector())
3570// Split DemandedElts and test each of the demanded subvectors. 3572EVT SubVectorVT =
Op.getOperand(0).getValueType();
3575for (
unsigned i = 0; i != NumSubVectors; ++i) {
3577 DemandedElts.
extractBits(NumSubVectorElts, i * NumSubVectorElts);
3583// If we don't know any bits, early out. 3590if (
Op.getValueType().isScalableVector())
3592// Demand any elements from the subvector and the remainder from the src its 3599APInt DemandedSrcElts = DemandedElts;
3604if (!!DemandedSubElts) {
3609if (!!DemandedSrcElts) {
3616// Offset the demanded elts by the subvector index. 3618// Bail until we can represent demanded elements for scalable vectors. 3619if (
Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3622unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3628if (
Op.getValueType().isScalableVector())
3630// We know about scalar_to_vector as much as we know about it source, 3631// which becomes the first element of otherwise unknown vector. 3632if (DemandedElts != 1)
3643if (
Op.getValueType().isScalableVector())
3650// Ignore bitcasts from unsupported types. 3654// Fast handling of 'identity' bitcasts. 3662// Bitcast 'small element' vector to 'large element' scalar/vector. 3663if ((
BitWidth % SubBitWidth) == 0) {
3666// Collect known bits for the (larger) output by collecting the known 3667// bits from each set of sub elements and shift these into place. 3668// We need to separately call computeKnownBits for each set of 3669// sub elements as the knownbits for each is likely to be different. 3670unsigned SubScale =
BitWidth / SubBitWidth;
3671APInt SubDemandedElts(NumElts * SubScale, 0);
3672for (
unsigned i = 0; i != NumElts; ++i)
3674 SubDemandedElts.
setBit(i * SubScale);
3676for (
unsigned i = 0; i != SubScale; ++i) {
3679unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3680 Known.
insertBits(Known2, SubBitWidth * Shifts);
3684// Bitcast 'large element' scalar/vector to 'small element' vector. 3686assert(
Op.getValueType().isVector() &&
"Expected bitcast to vector");
3688// Collect known bits for the (smaller) output by collecting the known 3689// bits from the overlapping larger input elements and extracting the 3690// sub sections we actually care about. 3691unsigned SubScale = SubBitWidth /
BitWidth;
3692APInt SubDemandedElts =
3697for (
unsigned i = 0; i != NumElts; ++i)
3698if (DemandedElts[i]) {
3699unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3702// If we don't know any bits, early out. 3730bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3731// TODO: SelfMultiply can be poison, but not undef. 3734Op.getOperand(0), DemandedElts,
false,
Depth + 1);
3737// If the multiplication is known not to overflow, the product of a number 3738// with itself is non-negative. Only do this if we didn't already computed 3739// the opposite value for the sign bit. 3740if (
Op->getFlags().hasNoSignedWrap() &&
3741Op.getOperand(0) ==
Op.getOperand(1) &&
3778assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3781bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3782if (
Op.getResNo() == 0)
3789assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3792bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3793if (
Op.getResNo() == 0)
3826// If we don't know any bits, early out. 3831// Only known if known in both the LHS and RHS. 3836// If we don't know any bits, early out. 3841// Only known if known in both the LHS and RHS. 3846if (
Op.getResNo() != 1)
3848// The boolean result conforms to getBooleanContents. 3849// If we know the result of a setcc has the top bits zero, use this info. 3850// We know that we have an integer-based boolean since these operations 3851// are only available for integer. 3861unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
3862// If we know the result of a setcc has the top bits zero, use this info. 3873bool NUW =
Op->getFlags().hasNoUnsignedWrap();
3874bool NSW =
Op->getFlags().hasNoSignedWrap();
3880// Minimum shift low bits are known zero. 3881if (std::optional<uint64_t> ShMinAmt =
3890Op->getFlags().hasExact());
3892// Minimum shift high bits are known zero. 3893if (std::optional<uint64_t> ShMinAmt =
3901Op->getFlags().hasExact());
3906unsigned Amt =
C->getAPIntValue().urem(
BitWidth);
3908// For fshl, 0-shift returns the 1st arg. 3909// For fshr, 0-shift returns the 2nd arg. 3912 DemandedElts,
Depth + 1);
3916// fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) 3917// fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) 3937assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3939// Collect lo/hi source values and concatenate. 3940unsigned LoBits =
Op.getOperand(0).getScalarValueSizeInBits();
3941unsigned HiBits =
Op.getOperand(1).getScalarValueSizeInBits();
3944 Known = Known2.
concat(Known);
3946// Collect shift amount. 3953else// if (Opcode == ISD::SRL_PARTS) 3956// TODO: Minimum shift low/high bits are known zero. 3958if (
Op.getResNo() == 0)
3966EVTEVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3973// If we have a known 1, its position is our upper bound. 3982// If we have a known 1, its position is our upper bound. 3990// If we know some of the bits are zero, they can't be one. 3996// Parity returns 0 everywhere but the LSB. 4004 ? cast<MaskedGatherSDNode>(
Op)->getExtensionType()
4005 : cast<MaskedLoadSDNode>(
Op)->getExtensionType();
4007EVT MemVT = cast<MemSDNode>(
Op)->getMemoryVT();
4017// Determine any common known bits from the loaded constant pool value. 4020 !
Op.getValueType().isScalableVector()) {
4021// If its a vector splat, then we can (quickly) reuse the scalar path. 4022// NOTE: We assume all elements match and none are UNDEF. 4029// TODO - do we need to handle different bitwidths? 4031// Iterate across all vector elements finding common known bits. 4034for (
unsigned i = 0; i != NumElts; ++i) {
4035if (!DemandedElts[i])
4038if (
auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4044if (
auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4045APIntValue = CFP->getValueAPF().bitcastToAPInt();
4056if (
auto *CInt = dyn_cast<ConstantInt>(Cst)) {
4058 }
elseif (
auto *CFP = dyn_cast<ConstantFP>(Cst)) {
4064 }
elseif (
Op.getResNo() == 0) {
4065KnownBits Known0(!LD->getMemoryVT().isScalableVT()
4066 ? LD->getMemoryVT().getFixedSizeInBits()
4068EVT VT =
Op.getValueType();
4069// Fill in any known bits from range information. There are 3 types being 4070// used. The results VT (same vector elt size as BitWidth), the loaded 4071// MemoryVT (which may or may not be vector) and the range VTs original 4072// type. The range matadata needs the full range (i.e 4073// MemoryVT().getSizeInBits()), which is truncated to the correct elt size 4074// if it is know. These are then extended to the original VT sizes below. 4075if (
constMDNode *MD = LD->getRanges()) {
4078// Handle truncation to the first demanded element. 4079// TODO: Figure out which demanded elements are covered 4086if (LD->getMemoryVT().isVector())
4087 Known0 = Known0.
trunc(LD->getMemoryVT().getScalarSizeInBits());
4089// Extend the Known bits from memory to the size of the result. 4104if (
Op.getValueType().isScalableVector())
4106EVT InVT =
Op.getOperand(0).getValueType();
4118if (
Op.getValueType().isScalableVector())
4120EVT InVT =
Op.getOperand(0).getValueType();
4123// If the sign bit is known to be zero or one, then sext will extend 4124// it to the top bits, else it will just zext. 4130// If the sign bit is known to be zero or one, then sext will extend 4131// it to the top bits, else it will just zext. 4136if (
Op.getValueType().isScalableVector())
4138EVT InVT =
Op.getOperand(0).getValueType();
4155EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
4158 Known.
Zero |= (~InMask);
4159 Known.
One &= (~Known.Zero);
4163unsigned LogOfAlign =
Log2(cast<AssertAlignSDNode>(
Op)->
getAlign());
4166// TODO: Should use maximum with source 4167// If a node is guaranteed to be aligned, set low zero bits accordingly as 4168// well as clearing one bits. 4174// All bits are zero except the low bit. 4183Op.getOpcode() ==
ISD::ADD, Flags.hasNoSignedWrap(),
4184 Flags.hasNoUnsignedWrap(), Known, Known2);
4191if (
Op.getResNo() == 1) {
4192// If we know the result of a setcc has the top bits zero, use this info. 4202"We only compute knownbits for the difference here.");
4204// With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in. 4208// Borrow has bit width 1 4209 Borrow = Borrow.
trunc(1);
4223if (
Op.getResNo() == 1) {
4224// If we know the result of a setcc has the top bits zero, use this info. 4234assert(
Op.getResNo() == 0 &&
"We only compute knownbits for the sum here.");
4236// With ADDE and UADDO_CARRY, a carry bit may be added in. 4239// Can't track carry from glue, set carry to unknown. 4243// Carry has bit width 1 4244 Carry = Carry.
trunc(1);
4280constunsigned Index =
Op.getConstantOperandVal(1);
4281constunsigned EltBitWidth =
Op.getValueSizeInBits();
4283// Remove low part of known bits mask 4287// Remove high part of known bit mask 4288 Known = Known.
trunc(EltBitWidth);
4295// computeKnownBits not yet implemented for scalable vectors. 4301// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know 4302// anything about the extended bits. 4304 Known = Known.
trunc(EltBitWidth);
4306// If we know the element index, just demand that vector element, else for 4307// an unknown element index, ignore DemandedElts and demand them all. 4309auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4310if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4320if (
Op.getValueType().isScalableVector())
4323// If we know the element index, split the demand between the 4324// source vector and the inserted element, otherwise assume we need 4325// the original demanded vector elements and the value. 4329bool DemandedVal =
true;
4330APInt DemandedVecElts = DemandedElts;
4331auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4332if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4333unsigned EltIdx = CEltNo->getZExtValue();
4334 DemandedVal = !!DemandedElts[EltIdx];
4343if (!!DemandedVecElts) {
4361 Known = Known2.
abs();
4386// If we have a clamp pattern, we know that the number of sign bits will be 4387// the minimum of the clamp min/max range. 4394if (CstLow && CstHigh) {
4399constAPInt &ValueHigh = CstHigh->getAPIntValue();
4400if (ValueLow.
sle(ValueHigh)) {
4403unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
4422// For SMAX, if CstLow is non-negative we know the result will be 4423// non-negative and thus all sign bits are 0. 4424// TODO: There's an equivalent of this for smin with negative constant for 4426if (IsMax && CstLow) {
4449// FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT. 4450EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
4455if (
Op.getResNo() == 1) {
4456// The boolean result conforms to getBooleanContents. 4457// If we know the result of a setcc has the top bits zero, use this info. 4458// We know that we have an integer-based boolean since these operations 4459// are only available for integer. 4482 cast<AtomicSDNode>(
Op)->getMemoryVT().getScalarSizeInBits();
4483// If we are looking at the loaded value. 4484if (
Op.getResNo() == 0) {
4506// TODO: Probably okay to remove after audit; here to reduce change size 4507// in initial enablement patch for scalable vectors 4508if (
Op.getValueType().isScalableVector())
4511// Allow the target to implement this method for its nodes. 4519/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind. 4535// X + 0 never overflow 4539// If both operands each have at least two sign bits, the addition 4544// TODO: Add ConstantRange::signedAddMayOverflow handling. 4550// X + 0 never overflow 4554// mulhi + 1 never overflow 4565// Fallback to ConstantRange::unsignedAddMayOverflow handling. 4573// X - 0 never overflow 4577// If both operands each have at least two sign bits, the subtraction 4591// X - 0 never overflow 4604// X * 0 and X * 1 never overflow. 4617// X * 0 and X * 1 never overflow. 4621// Get the size of the result. 4624// Sum of the sign bits. 4627// If we have enough sign bits, then there's no overflow. 4632// The overflow occurs when the true multiplication of the 4633// the operands is the minimum negative number. 4636// If one of the operands is non-negative, then there's no 4647returnfalse;
// Limit search depth. 4652// Is the constant a known power of 2? 4654returnC->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2();
4658// A left-shift of a constant one will have exactly one bit set because 4659// shifting the bit off the end is undefined. 4662if (
C &&
C->getAPIntValue() == 1)
4668// Similarly, a logical right-shift of a constant sign-bit will have exactly 4672if (
C &&
C->getAPIntValue().isSignMask())
4681// Are all operands of a build vector constant powers of two? 4684 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4685 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4690// Is the operand of a splat vector a constant power of two? 4693if (
C->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2())
4696// vscale(power-of-two) is a power-of-two for some targets 4711// Looking for `x & -x` pattern: 4715// x & -x -> non-zero pow2 4716// so if we find the pattern return whether we know `x` is non-zero. 4724// More could be done here, though the above checks are enough 4725// to handle some common cases. 4731return C1->getValueAPF().getExactLog2Abs() >= 0;
4740EVT VT =
Op.getValueType();
4742// Since the number of lanes in a scalable vector is unknown at compile time, 4743// we track one bit which is implicitly broadcast to all lanes. This means 4744// that all lanes in a scalable vector are considered demanded. 4752unsignedDepth)
const{
4753EVT VT =
Op.getValueType();
4758unsigned FirstAnswer = 1;
4760if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
4761constAPInt &Val =
C->getAPIntValue();
4766return 1;
// Limit search depth. 4769return 1;
// No demanded elts, better to assume we don't know anything. 4771unsigned Opcode =
Op.getOpcode();
4775 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getSizeInBits();
4778 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getSizeInBits();
4784// Check if the sign bits of source go down as far as the truncated value. 4785unsigned NumSrcBits =
Op.getOperand(0).getValueSizeInBits();
4787if (NumSrcSignBits > (NumSrcBits - VTBits))
4788return NumSrcSignBits - (NumSrcBits - VTBits);
4795if (!DemandedElts[i])
4799// BUILD_VECTOR can implicitly truncate sources, we handle this specially 4800// for constant nodes to ensure we only look at the sign bits. 4802APIntT =
C->getAPIntValue().trunc(VTBits);
4803 Tmp2 =
T.getNumSignBits();
4807if (
SrcOp.getValueSizeInBits() != VTBits) {
4809"Expected BUILD_VECTOR implicit truncation");
4810unsigned ExtraBits =
SrcOp.getValueSizeInBits() - VTBits;
4811 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4814 Tmp = std::min(Tmp, Tmp2);
4819// Collect the minimum number of sign bits that are shared by every vector 4820// element referenced by the shuffle. 4821APInt DemandedLHS, DemandedRHS;
4825 DemandedLHS, DemandedRHS))
4828 Tmp = std::numeric_limits<unsigned>::max();
4833 Tmp = std::min(Tmp, Tmp2);
4835// If we don't know anything, early out and try computeKnownBits fall-back. 4838assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
4849// Ignore bitcasts from unsupported types.. 4853// Fast handling of 'identity' bitcasts. 4854if (VTBits == SrcBits)
4859// Bitcast 'large element' scalar/vector to 'small element' vector. 4860if ((SrcBits % VTBits) == 0) {
4863unsigned Scale = SrcBits / VTBits;
4864APInt SrcDemandedElts =
4867// Fast case - sign splat can be simply split across the small elements. 4872// Slow case - determine how far the sign extends into each sub-element. 4874for (
unsigned i = 0; i != NumElts; ++i)
4875if (DemandedElts[i]) {
4876unsigned SubOffset = i % Scale;
4877 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4878 SubOffset = SubOffset * VTBits;
4879if (Tmp <= SubOffset)
4881 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
4889// FP_TO_SINT_SAT produces a signed value that fits in the saturating VT. 4890 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getScalarSizeInBits();
4891return VTBits - Tmp + 1;
4893 Tmp = VTBits -
Op.getOperand(0).getScalarValueSizeInBits();
4896// Max of the input and what this extends. 4897 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getScalarSizeInBits();
4900return std::max(Tmp, Tmp2);
4905EVT SrcVT = Src.getValueType();
4912// SRA X, C -> adds C sign bits. 4913if (std::optional<uint64_t> ShAmt =
4915 Tmp = std::min<uint64_t>(Tmp + *ShAmt, VTBits);
4918if (std::optional<ConstantRange> ShAmtRange =
4920uint64_t MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
4921uint64_t MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
4922// Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are 4923// shifted out, then we can compute the number of sign bits for the 4924// operand being extended. A future improvement could be to pass along the 4925// "shifted left by" information in the recursive calls to 4926// ComputeKnownSignBits. Allowing us to handle this more generically. 4929EVT ExtVT = Ext.getValueType();
4930SDValue Extendee = Ext.getOperand(0);
4934if (SizeDifference <= MinShAmt) {
4935 Tmp = SizeDifference +
4938return Tmp - MaxShAmt;
4941// shl destroys sign bits, ensure it doesn't shift out all sign bits. 4944return Tmp - MaxShAmt;
4949caseISD::XOR:
// NOT is handled here. 4950// Logical binary ops preserve the number of sign bits at the worst. 4954 FirstAnswer = std::min(Tmp, Tmp2);
4955// We computed what we know about the sign bits as our first 4956// answer. Now proceed to the generic code that uses 4957// computeKnownBits, and pick whichever answer is better. 4964if (Tmp == 1)
return 1;
// Early out. 4966return std::min(Tmp, Tmp2);
4969if (Tmp == 1)
return 1;
// Early out. 4971return std::min(Tmp, Tmp2);
4975// If we have a clamp pattern, we know that the number of sign bits will be 4976// the minimum of the clamp min/max range. 4983if (CstLow && CstHigh) {
4988 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4989return std::min(Tmp, Tmp2);
4993// Fallback - just get the minimum number of sign bits of the operands. 4996return 1;
// Early out. 4998return std::min(Tmp, Tmp2);
5004return 1;
// Early out. 5006return std::min(Tmp, Tmp2);
5009// sub_carry(x,x,c) -> 0/-1 (sext carry) 5010if (
Op.getResNo() == 0 &&
Op.getOperand(0) ==
Op.getOperand(1))
5021if (
Op.getResNo() != 1)
5023// The boolean result conforms to getBooleanContents. Fall through. 5024// If setcc returns 0/-1, all bits are sign bits. 5025// We know that we have an integer-based boolean since these operations 5026// are only available for integer. 5035unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
5036// If setcc returns 0/-1, all bits are sign bits. 5046// If we're rotating an 0/-1 value, then it stays an 0/-1 value. 5052unsigned RotAmt =
C->getAPIntValue().urem(VTBits);
5054// Handle rotate right by N like a rotate left by 32-N. 5056 RotAmt = (VTBits - RotAmt) % VTBits;
5058// If we aren't rotating out all of the known-in sign bits, return the 5059// number that are left. This handles rotl(sext(x), 1) for example. 5060if (Tmp > (RotAmt + 1))
return (Tmp - RotAmt);
5065// Add can have at most one carry bit. Thus we know that the output 5066// is, at worst, one more bit than the inputs. 5068if (Tmp == 1)
return 1;
// Early out. 5070// Special case decrementing a value (ADD X, -1): 5073if (CRHS->isAllOnes()) {
5077// If the input is known to be 0 or 1, the output is 0/-1, which is all 5079if ((Known.
Zero | 1).isAllOnes())
5082// If we are subtracting one from a positive number, there is no carry 5083// out of the result. 5089if (Tmp2 == 1)
return 1;
// Early out. 5090return std::min(Tmp, Tmp2) - 1;
5093if (Tmp2 == 1)
return 1;
// Early out. 5098if (CLHS->isZero()) {
5101// If the input is known to be 0 or 1, the output is 0/-1, which is all 5103if ((Known.
Zero | 1).isAllOnes())
5106// If the input is known to be positive (the sign bit is known clear), 5107// the output of the NEG has the same number of sign bits as the input. 5111// Otherwise, we treat this like a SUB. 5114// Sub can have at most one carry bit. Thus we know that the output 5115// is, at worst, one more bit than the inputs. 5117if (Tmp == 1)
return 1;
// Early out. 5118return std::min(Tmp, Tmp2) - 1;
5120// The output of the Mul can be at most twice the valid bits in the inputs. 5122if (SignBitsOp0 == 1)
5125if (SignBitsOp1 == 1)
5127unsigned OutValidBits =
5128 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5129return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5135return 1;
// Early out. 5137return std::min(Tmp, Tmp2);
5139// The sign bit is the LHS's sign bit, except when the result of the 5140// remainder is zero. The magnitude of the result should be less than or 5141// equal to the magnitude of the LHS. Therefore, the result should have 5142// at least as many sign bits as the left hand side. 5145// Check if the sign bits of source go down as far as the truncated value. 5146unsigned NumSrcBits =
Op.getOperand(0).getScalarValueSizeInBits();
5148if (NumSrcSignBits > (NumSrcBits - VTBits))
5149return NumSrcSignBits - (NumSrcBits - VTBits);
5157constint Items =
Op.getOperand(0).getValueSizeInBits() /
BitWidth;
5159// Get reverse index (starting from 1), Op1 value indexes elements from 5160// little end. Sign starts at big end. 5161constint rIndex = Items - 1 -
Op.getConstantOperandVal(1);
5163// If the sign portion ends in our element the subtraction gives correct 5164// result. Otherwise it gives either negative or > bitwidth result 5170// If we know the element index, split the demand between the 5171// source vector and the inserted element, otherwise assume we need 5172// the original demanded vector elements and the value. 5176bool DemandedVal =
true;
5177APInt DemandedVecElts = DemandedElts;
5178auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
5179if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
5180unsigned EltIdx = CEltNo->getZExtValue();
5181 DemandedVal = !!DemandedElts[EltIdx];
5184 Tmp = std::numeric_limits<unsigned>::max();
5186// TODO - handle implicit truncation of inserted elements. 5190 Tmp = std::min(Tmp, Tmp2);
5192if (!!DemandedVecElts) {
5194 Tmp = std::min(Tmp, Tmp2);
5196assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
5204// ComputeNumSignBits not yet implemented for scalable vectors. 5207constunsignedBitWidth =
Op.getValueSizeInBits();
5208constunsigned EltBitWidth =
Op.getOperand(0).getScalarValueSizeInBits();
5211// If BitWidth > EltBitWidth the value is anyext:ed, and we do not know 5212// anything about sign bits. But if the sizes match we can derive knowledge 5213// about sign bits from the vector operand. 5217// If we know the element index, just demand that vector element, else for 5218// an unknown element index, ignore DemandedElts and demand them all. 5220auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
5221if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
5228// Offset the demanded elts by the subvector index. 5230// Bail until we can represent demanded elements for scalable vectors. 5231if (Src.getValueType().isScalableVector())
5234unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5241// Determine the minimum number of sign bits across all demanded 5242// elts of the input vectors. Early out if the result is already 1. 5243 Tmp = std::numeric_limits<unsigned>::max();
5244EVT SubVectorVT =
Op.getOperand(0).getValueType();
5247for (
unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5249 DemandedElts.
extractBits(NumSubVectorElts, i * NumSubVectorElts);
5253 Tmp = std::min(Tmp, Tmp2);
5255assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
5261// Demand any elements from the subvector and the remainder from the src its 5268APInt DemandedSrcElts = DemandedElts;
5271 Tmp = std::numeric_limits<unsigned>::max();
5272if (!!DemandedSubElts) {
5275return 1;
// early-out 5277if (!!DemandedSrcElts) {
5279 Tmp = std::min(Tmp, Tmp2);
5281assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
5286if (
constMDNode *Ranges = LD->getRanges()) {
5287if (DemandedElts != 1)
5292switch (LD->getExtensionType()) {
5327 Tmp = cast<AtomicSDNode>(
Op)->getMemoryVT().getScalarSizeInBits();
5328// If we are looking at the loaded value. 5329if (
Op.getResNo() == 0) {
5331return 1;
// early-out 5333return VTBits - Tmp + 1;
5339return VTBits - Tmp + 1;
5348// If we are looking at the loaded value of the SDNode. 5349if (
Op.getResNo() == 0) {
5350// Handle LOADX separately here. EXTLOAD case will fallthrough. 5352unsigned ExtType = LD->getExtensionType();
5356 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5357return VTBits - Tmp + 1;
5359 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5363// We only need to handle vectors - computeKnownBits should handle 5365Type *CstTy = Cst->getType();
5370for (
unsigned i = 0; i != NumElts; ++i) {
5371if (!DemandedElts[i])
5374if (
auto *CInt = dyn_cast<ConstantInt>(Elt)) {
5376 Tmp = std::min(Tmp,
Value.getNumSignBits());
5379if (
auto *CFP = dyn_cast<ConstantFP>(Elt)) {
5380APIntValue = CFP->getValueAPF().bitcastToAPInt();
5381 Tmp = std::min(Tmp,
Value.getNumSignBits());
5385// Unknown type. Conservatively assume no bits match sign bit. 5396// Allow the target to implement this method for its nodes. 5401// TODO: This can probably be removed once target code is audited. This 5402// is here purely to reduce patch size and review complexity. 5407 FirstAnswer = std::max(FirstAnswer, NumBits);
5411// Finally, if we can prove that the top bits of the result are 0's or 1's, 5412// use this information. 5418unsignedDepth)
const{
5420returnOp.getScalarValueSizeInBits() - SignBits + 1;
5424constAPInt &DemandedElts,
5425unsignedDepth)
const{
5427returnOp.getScalarValueSizeInBits() - SignBits + 1;
5431unsignedDepth)
const{
5432// Early out for FREEZE. 5436EVT VT =
Op.getValueType();
5444constAPInt &DemandedElts,
5446unsignedDepth)
const{
5447unsigned Opcode =
Op.getOpcode();
5449// Early out for FREEZE. 5454returnfalse;
// Limit search depth. 5471// NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements - 5472// this shouldn't affect the result. 5474if (!DemandedElts[i])
5487APInt DemandedLHS, DemandedRHS;
5488auto *SVN = cast<ShuffleVectorSDNode>(
Op);
5490 DemandedElts, DemandedLHS, DemandedRHS,
5491/*AllowUndefElts=*/false))
5493if (!DemandedLHS.
isZero() &&
5497if (!DemandedRHS.
isZero() &&
5504// TODO: Search for noundef attributes from library functions. 5506// TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef. 5509// Allow the target to implement this method for its nodes. 5517// If Op can't create undef/poison and none of its operands are undef/poison 5518// then Op is never undef/poison. 5519// NOTE: TargetNodes can handle this in themselves in 5520// isGuaranteedNotToBeUndefOrPoisonForTargetNode or let 5521// TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it. 5525 return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);
5531unsignedDepth)
const{
5532EVT VT =
Op.getValueType();
5542unsignedDepth)
const{
5543if (ConsiderFlags &&
Op->hasPoisonGeneratingFlags())
5546unsigned Opcode =
Op.getOpcode();
5584// Integer setcc cannot create undef or poison. 5585if (
Op.getOperand(0).getValueType().isInteger())
5588// FP compares are more complicated. They can create poison for nan/infinity 5589// based on options and flags. The options and flags also cause special 5590// nonan condition codes to be used. Those condition codes may be preserved 5591// even if the nonan flag is dropped somewhere. 5593ISD::CondCode CCCode = cast<CondCodeSDNode>(
Op.getOperand(CCOp))->get();
5594if (((
unsigned)CCCode & 0x10U))
5606// No poison except from flags (which is handled above) 5612// If the max shift amount isn't in range, then the shift can 5619// Check if we demand any upper (undef) elements. 5624// Ensure that the element index is in bounds. 5625EVT VecVT =
Op.getOperand(0).getValueType();
5636// Check for any demanded shuffle element that is undef. 5637auto *SVN = cast<ShuffleVectorSDNode>(
Op);
5639if (Elt < 0 && DemandedElts[
Idx])
5645// Allow the target to implement this method for its nodes. 5653// Be conservative and return true. 5658unsigned Opcode =
Op.getOpcode();
5660returnOp->getFlags().hasDisjoint() ||
5673// If we're told that NaNs won't happen, assume they won't. 5678returnfalse;
// Limit search depth. 5680// If the value is a constant, we can obviously see if it is a NaN or not. 5682return !
C->getValueAPF().isNaN() ||
5683 (SNaN && !
C->getValueAPF().isSignaling());
5686unsigned Opcode =
Op.getOpcode();
5707// TODO: Need isKnownNeverInfinity 5755// TODO: Refine on operand 5762// Only one needs to be known not-nan, since it will be returned if the 5763// other ends up being one. 5771// This can return a NaN if either operand is an sNaN, or if both operands 5780// TODO: Does this quiet or return the origina NaN as-is? 5806assert(
Op.getValueType().isFloatingPoint() &&
5807"Floating point type expected");
5809// If the value is a constant, we can obviously see if it is a zero or not. 5816returnfalse;
// Limit search depth. 5818assert(!
Op.getValueType().isFloatingPoint() &&
5819"Floating point types unsupported - use isKnownNeverZeroFloat");
5821// If the value is a constant, we can obviously see if it is a zero or not. 5826// TODO: Recognize more cases here. Most of the cases are also incomplete to 5828switch (
Op.getOpcode()) {
5842if (
Op->getFlags().hasNoSignedWrap() ||
Op->getFlags().hasNoUnsignedWrap())
5845// 1 << X is never zero. 5848// If max shift cnt of known ones is non-zero, result is non-zero. 5860// For smin/smax: If either operand is known negative/positive 5861// respectively we don't need the other to be known at all. 5906if (
Op->getFlags().hasExact())
5911// If max shift cnt of known ones is non-zero, result is non-zero. 5920// div exact can only produce a zero if the dividend is zero. 5921// TODO: For udiv this is also true if Op1 u<= Op0 5922if (
Op->getFlags().hasExact())
5927if (
Op->getFlags().hasNoUnsignedWrap())
5931// TODO: There are a lot more cases we can prove for add. 5938 std::optional<bool> ne =
5945if (
Op->getFlags().hasNoSignedWrap() ||
Op->getFlags().hasNoUnsignedWrap())
5956constAPInt &Multiplier =
Op.getConstantOperandAPInt(0);
5970return !C1->isNegative();
5976// Check the obvious case. 5977if (
A ==
B)
returntrue;
5979// For negative and positive zero. 5982if (CA->isZero() && CB->isZero())
returntrue;
5984// Otherwise they may not be equal. 5988// Only bits set in Mask must be negated, other bits may be arbitrary. 5991return V.getOperand(0);
5993// Handle any_extend (not (truncate X)) pattern, where Mask only sets 5994// bits in the non-extended part. 5998SDValue ExtArg = V.getOperand(0);
6009// Match masked merge pattern (X & ~M) op (Y & M) 6010// Including degenerate case (X & ~M) op M 6017 NotOperand = NotOperand->getOperand(0);
6019if (
Other == NotOperand)
6022return NotOperand ==
Other->getOperand(0) ||
6023 NotOperand ==
Other->getOperand(1);
6035return MatchNoCommonBitsPattern(
A->getOperand(0),
A->getOperand(1),
B) ||
6036 MatchNoCommonBitsPattern(
A->getOperand(1),
A->getOperand(0),
B);
6040// FIXME: unify with llvm::haveNoCommonBitsSet. 6042assert(
A.getValueType() ==
B.getValueType() &&
6043"Values must have the same type");
6053if (cast<ConstantSDNode>(Step)->
isZero())
6062int NumOps = Ops.
size();
6063assert(NumOps != 0 &&
"Can't build an empty vector!");
6065"BUILD_VECTOR cannot be used with scalable types");
6067"Incorrect element count in BUILD_VECTOR!");
6069// BUILD_VECTOR of UNDEFs is UNDEF. 6073// BUILD_VECTOR of seq extract/insert from the same vector + type is Identity. 6075bool IsIdentity =
true;
6076for (
int i = 0; i != NumOps; ++i) {
6079 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
6080 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
6081 Ops[i].getConstantOperandAPInt(1) != i) {
6085 IdentitySrc = Ops[i].getOperand(0);
6093/// Try to simplify vector concatenation to an input value, undef, or build 6098assert(!Ops.
empty() &&
"Can't concatenate an empty list of vectors!");
6101return Ops[0].getValueType() ==
Op.getValueType();
6103"Concatenation of vectors with inconsistent value types!");
6106"Incorrect element count in vector concatenation!");
6111// Concat of UNDEFs is UNDEF. 6115// Scan the operands and look for extract operations from a single source 6116// that correspond to insertion at the same location via this concatenation: 6117// concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ... 6119bool IsIdentity =
true;
6120for (
unsigned i = 0, e = Ops.
size(); i != e; ++i) {
6122unsigned IdentityIndex = i *
Op.getValueType().getVectorMinNumElements();
6124Op.getOperand(0).getValueType() != VT ||
6125 (IdentitySrc &&
Op.getOperand(0) != IdentitySrc) ||
6126Op.getConstantOperandVal(1) != IdentityIndex) {
6130assert((!IdentitySrc || IdentitySrc ==
Op.getOperand(0)) &&
6131"Unexpected identity source vector for concat of extracts");
6132 IdentitySrc =
Op.getOperand(0);
6135assert(IdentitySrc &&
"Failed to set source vector of extracts");
6139// The code below this point is only designed to work for fixed width 6140// vectors, so we bail out for now. 6144// A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be 6145// simplified to one big BUILD_VECTOR. 6146// FIXME: Add support for SCALAR_TO_VECTOR as well. 6150EVT OpVT =
Op.getValueType();
6159// BUILD_VECTOR requires all inputs to be of the same type, find the 6160// maximum type and extend them all. 6162 SVT = (SVT.
bitsLT(
Op.getValueType()) ?
Op.getValueType() : SVT);
6180/// Gets or creates the specified node. 6186if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
6189auto *
N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
6190 CSEMap.InsertNode(
N, IP);
6210// Constant fold unary operations with a vector integer or float operand. 6213// FIXME: Entirely reasonable to perform folding of other unary 6214// operations here as the need arises. 6254"STEP_VECTOR can only be used with scalable types");
6257"Unexpected step operand");
6268return N1;
// Factor, merge or concat of one node? No need. 6270// Attempt to simplify BUILD_VECTOR. 6280if (N1.
getValueType() == VT)
return N1;
// noop conversion. 6283"Vector element count mismatch!");
6295// [us]itofp(undef) = 0, because the result value is bounded. 6301"Invalid SIGN_EXTEND!");
6303"SIGN_EXTEND result type type should be vector iff the operand " 6308"Vector element count mismatch!");
6317// sext(undef) = 0, because the top bits will all be the same. 6322"Invalid ZERO_EXTEND!");
6324"ZERO_EXTEND result type type should be vector iff the operand " 6329"Vector element count mismatch!");
6337// zext(undef) = 0, because the top bits will be zero. 6340// Skip unnecessary zext_inreg pattern: 6341// (zext (trunc x)) -> x iff the upper bits are known zero. 6342// TODO: Remove (zext (trunc (and x, c))) exception which some targets 6343// use to recognise zext_inreg patterns. 6360"Invalid ANY_EXTEND!");
6362"ANY_EXTEND result type type should be vector iff the operand " 6367"Vector element count mismatch!");
6375// (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 6381// (ext (trunc x)) -> x 6392"Invalid TRUNCATE!");
6394"TRUNCATE result type type should be vector iff the operand " 6399"Vector element count mismatch!");
6405// If the source is smaller than the dest, we still need an extend. 6422assert(VT.
isVector() &&
"This DAG node is restricted to vector types.");
6424"The input must be the same size or smaller than the result.");
6427"The destination vector type must have fewer lanes than the input.");
6437"BSWAP types must be a multiple of 16 bits!");
6440// bswap(bswap(X)) -> X. 6451"Cannot BITCAST between types of different sizes!");
6452if (VT == N1.
getValueType())
return N1;
// noop conversion. 6453if (OpOpcode ==
ISD::BITCAST)
// bitconv(bitconv(x)) -> bitconv(x) 6464"Illegal SCALAR_TO_VECTOR node!");
6467// scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 6475// Negation of an unknown bag of bits is still completely undefined. 6483if (OpOpcode ==
ISD::FNEG)
// abs(-X) -> abs(X) 6514// FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow 6521"Wrong operand type!");
6528if (VT != MVT::Glue) {
// Don't CSE glue producing nodes 6532if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
6533 E->intersectFlagsWith(Flags);
6537N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
6539 createOperands(
N, Ops);
6540 CSEMap.InsertNode(
N, IP);
6542N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
6543 createOperands(
N, Ops);
6577if (!C2.getBoolValue())
6581if (!C2.getBoolValue())
6585if (!C2.getBoolValue())
6589if (!C2.getBoolValue())
6611// Handle constant folding with UNDEF. 6612// TODO: Handle more cases. 6614bool IsUndef1,
constAPInt &C2,
6616if (!(IsUndef1 || IsUndef2))
6619// Fold and(x, undef) -> 0 6620// Fold mul(x, undef) -> 0 6634auto *C2 = dyn_cast<ConstantSDNode>(N2);
6637 int64_t
Offset = C2->getSExtValue();
6653// If a divisor is zero/undef or any element of a divisor vector is 6654// zero/undef, the whole op is undef. 6655assert(Ops.
size() == 2 &&
"Div/rem should have 2 operands");
6662 [](
SDValue V) { return V.isUndef() ||
6663 isNullConstant(V); });
6664// TODO: Handle signed overflow. 6666// TODO: Handle oversized shifts. 6675// If the opcode is a target-specific ISD node, there's nothing we can 6676// do here and the operand rules may not line up with the below, so 6678// We can't create a scalar CONCAT_VECTORS so skip it. It will break 6679// for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by 6680// foldCONCAT_VECTORS in getNode before this is called. 6684unsigned NumOps = Ops.
size();
6691// Handle unary special cases. 6695// Constant fold unary operations with an integer constant operand. Even 6696// opaque constant will be folded, because the folding of unary operations 6697// doesn't create new constants with different values. Nevertheless, the 6698// opaque flag is preserved during folding to prevent future folding with 6700if (
auto *
C = dyn_cast<ConstantSDNode>(N1)) {
6701constAPInt &Val =
C->getAPIntValue();
6705C->isTargetOpcode(),
C->isOpaque());
6712C->isTargetOpcode(),
C->isOpaque());
6714// Some targets like RISCV prefer to sign extend some types. 6717C->isTargetOpcode(),
C->isOpaque());
6719C->isTargetOpcode(),
C->isOpaque());
6754// This can return overflow, underflow, or inexact; we don't care. 6755// FIXME need to be more flexible about rounding mode. 6765if (VT == MVT::f16 &&
C->getValueType(0) == MVT::i16)
6767if (VT == MVT::f32 &&
C->getValueType(0) == MVT::i32)
6769if (VT == MVT::f64 &&
C->getValueType(0) == MVT::i64)
6771if (VT == MVT::f128 &&
C->getValueType(0) == MVT::i128)
6777// Constant fold unary operations with a floating point constant operand. 6778if (
auto *
C = dyn_cast<ConstantFPSDNode>(N1)) {
6779APFloat V =
C->getValueAPF();
// make copy 6807// This can return overflow, underflow, or inexact; we don't care. 6808// FIXME need to be more flexible about rounding mode. 6817// FIXME need to be more flexible about rounding mode. 6827// This can return overflow, underflow, or inexact; we don't care. 6828// FIXME need to be more flexible about rounding mode. 6835if (VT == MVT::i16 &&
C->getValueType(0) == MVT::f16)
6838if (VT == MVT::i16 &&
C->getValueType(0) == MVT::bf16)
6841if (VT == MVT::i32 &&
C->getValueType(0) == MVT::f32)
6844if (VT == MVT::i64 &&
C->getValueType(0) == MVT::f64)
6850// Early-out if we failed to constant fold a bitcast. 6855// Handle binops special cases. 6860if (
auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6861if (
auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
6862if (C1->isOpaque() || C2->isOpaque())
6865 std::optional<APInt> FoldAttempt =
6866FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
6872"Can't fold vectors ops with scalar operands");
6877// fold (add Sym, c) -> Sym+c 6884// fold (sext_in_reg c1) -> c2 6886EVTEVT = cast<VTSDNode>(Ops[1])->getVT();
6895if (
auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6896constAPInt &Val = C1->getAPIntValue();
6897return SignExtendInReg(Val, VT);
6902llvm::EVT OpVT = Ops[0].getOperand(0).getValueType();
6909constAPInt &Val = cast<ConstantSDNode>(
Op)->getAPIntValue();
6910 ScalarOps.
push_back(SignExtendInReg(Val, OpVT));
6916 isa<ConstantSDNode>(Ops[0].getOperand(0)))
6918 SignExtendInReg(Ops[0].getConstantOperandAPInt(0),
6923// This is for vector folding only from here on. 6929// See if we can fold through any bitcasted integer ops. 6931 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6936auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
6937auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
6944if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
6945 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
6949 Opcode, RawBits1[
I], UndefElts1[
I], RawBits2[
I], UndefElts2[
I]);
6955// We have constant folded, but we might need to cast this again back 6956// to the original (possibly legalized) type. 6960 BVEltVT = BV1->getOperand(0).getValueType();
6963 BVEltVT = BV2->getOperand(0).getValueType();
6969 DstBits, RawBits, DstUndefs,
6972for (
unsignedI = 0, E = DstBits.
size();
I != E; ++
I) {
6983// Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)). 6984// (shl step_vector(C0), C1) -> (step_vector(C0 << C1)) 6990 ? Ops[0].getConstantOperandAPInt(0) * RHSVal
6991 : Ops[0].getConstantOperandAPInt(0) << RHSVal;
6996auto IsScalarOrSameVectorSize = [NumElts](
constSDValue &
Op) {
6997return !
Op.getValueType().isVector() ||
6998Op.getValueType().getVectorElementCount() == NumElts;
7001auto IsBuildVectorSplatVectorOrUndef = [](
constSDValue &
Op) {
7007// All operands must be vector types with the same number of elements as 7008// the result type and must be either UNDEF or a build/splat vector 7010if (!
llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
7014// If we are comparing vectors, then the result needs to be a i1 boolean that 7015// is then extended back to the legal result type depending on how booleans 7023// Find legal integer scalar type for constant promotion and 7024// ensure that its scalar size is at least as large as source. 7032// For scalable vector types we know we're dealing with SPLAT_VECTORs. We 7033// only have one operand to check. For fixed-length vector types we may have 7034// a combination of BUILD_VECTOR and SPLAT_VECTOR. 7037// Constant fold each scalar lane separately. 7039for (
unsignedI = 0;
I != NumVectorElts;
I++) {
7042EVT InSVT =
Op.getValueType().getScalarType();
7056// Build vector (integer) scalar operands may need implicit 7057// truncation - do this before constant folding. 7059// Don't create illegally-typed nodes unless they're constants or undef 7060// - if we fail to constant fold we can't guarantee the (dead) nodes 7061// we're creating will be cleaned up before being visited for 7064 !isa<ConstantSDNode>(ScalarOp) &&
7074// Constant fold the scalar operands. 7077// Scalar folding only succeeded if the result is a constant or UNDEF. 7082// Legalize the (integer) scalar constant if necessary. We only do 7083// this once we know the folding succeeded, since otherwise we would 7084// get a node with illegal type which has a user. 7086 ScalarResult =
getNode(ExtendCode,
DL, LegalSVT, ScalarResult);
7099// TODO: Add support for unary/ternary fp opcodes. 7103// TODO: We don't do any constant folding for strict FP opcodes here, but we 7104// should. That will require dealing with a potentially non-default 7105// rounding mode, checking the "opStatus" return value from the APFloat 7106// math calculations, and possibly other variations. 7111if (N1CFP && N2CFP) {
7151// This can return overflow, underflow, or inexact; we don't care. 7152// FIXME need to be more flexible about rounding mode. 7160// -0.0 - undef --> undef (consistent with "fneg undef") 7162if (N1C && N1C->getValueAPF().isNegZero() && N2.
isUndef())
7170// If both operands are undef, the result is undef. If 1 operand is undef, 7171// the result is NaN. This should match the behavior of the IR optimizer. 7183// There's no need to assert on a byte-aligned pointer. All pointers are at 7184// least byte aligned. 7191ID.AddInteger(
A.value());
7194if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
7198 newSDNode<AssertAlignSDNode>(
DL.getIROrder(),
DL.getDebugLoc(), VTs,
A);
7199 createOperands(
N, {Val});
7201 CSEMap.InsertNode(
N, IP);
7214returngetNode(Opcode,
DL, VT, N1, N2, Flags);
7223// binop(const, nonconst) -> binop(nonconst, const) 7228if ((N1C && !N2C) || (N1CFP && !N2CFP))
7232// binop(splat(x), step_vector) -> binop(step_vector, splat(x)) 7242"Operand is DELETED_NODE!");
7246auto *N1C = dyn_cast<ConstantSDNode>(N1);
7247auto *N2C = dyn_cast<ConstantSDNode>(N2);
7249// Don't allow undefs in vector splats - we might be returning N2 when folding 7258 N2.
getValueType() == MVT::Other &&
"Invalid token factor!");
7259// Fold trivial token factors. 7262if (N1 == N2)
return N1;
7265// Attempt to simplify BUILD_VECTOR. 7280 N1.
getValueType() == VT &&
"Binary operator types must match!");
7281// (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 7282// worth handling here. 7283if (N2CV && N2CV->
isZero())
7285if (N2CV && N2CV->
isAllOnes())
// X & -1 -> X 7294 N1.
getValueType() == VT &&
"Binary operator types must match!");
7295// (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 7296// it's worth handling here. 7297if (N2CV && N2CV->
isZero())
7306 N1.
getValueType() == VT &&
"Binary operator types must match!");
7311constAPInt &N2CImm = N2C->getAPIntValue();
7327 N1.
getValueType() == VT &&
"Binary operator types must match!");
7329// fold (add_sat x, y) -> (or x, y) for bool types. 7332// fold (sub_sat x, y) -> (and x, ~y) for bool types. 7340"Types of operands of UCMP/SCMP must match");
7342"Operands and return type of must both be scalars or vectors");
7346"Result and operands must have the same number of elements");
7354 N1.
getValueType() == VT &&
"Binary operator types must match!");
7360 N1.
getValueType() == VT &&
"Binary operator types must match!");
7368 N1.
getValueType() == VT &&
"Binary operator types must match!");
7376 N1.
getValueType() == VT &&
"Binary operator types must match!");
7387 N1.
getValueType() == VT &&
"Binary operator types must match!");
7391caseISD::FCOPYSIGN:
// N1 and result must match. N1/N2 need not match. 7395"Invalid FCOPYSIGN!");
7400constAPInt &ShiftImm = N2C->getAPIntValue();
7412"Shift operators return type must be the same as their first arg");
7414"Shifts only work on integers");
7416"Vector shift amounts must be in the same as their first arg");
7417// Verify that the shift amount VT is big enough to hold valid shift 7418// amounts. This catches things like trying to shift an i1024 value by an 7419// i8, which is easy to fall into in generic code that uses 7420// TLI.getShiftAmount(). 7423"Invalid use of small shift amount with oversized value!");
7425// Always fold shifts of i1 values so the code generator doesn't need to 7426// handle them. Since we know the size of the shift has to be less than the 7427// size of the value, the shift/rotate count is guaranteed to be zero. 7430if (N2CV && N2CV->
isZero())
7436 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
7438if (N1.
getValueType() == VT)
return N1;
// noop conversion. 7442EVTEVT = cast<VTSDNode>(N2)->getVT();
7445"Cannot *_EXTEND_INREG FP types");
7447"AssertSExt/AssertZExt type should be the vector element type " 7448"rather than the vector type!");
7454EVTEVT = cast<VTSDNode>(N2)->getVT();
7457"Cannot *_EXTEND_INREG FP types");
7459"SIGN_EXTEND_INREG type should be vector iff the operand " 7463"Vector element counts must match in SIGN_EXTEND_INREG");
7465if (
EVT == VT)
return N1;
// Not actually extending 7473"FP_TO_*INT_SAT type should be vector iff the operand type is " 7477"Vector element counts must match in FP_TO_*INT_SAT");
7478assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
7479"Type to saturate to must be a scalar.");
7486"The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ 7487 element type of the vector.");
7489// Extract from an undefined value or using an undefined index is undefined. 7493// EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length 7494// vectors. For scalable vectors we will provide appropriate support for 7495// dealing with arbitrary indices. 7500// EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 7501// expanding copies of large vectors from registers. This only works for 7502// fixed length vectors, since we need to know the exact number of 7513// EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while 7514// lowering is expanding large vector constants. 7519"BUILD_VECTOR used for scalable vectors");
7525// If the vector element type is not legal, the BUILD_VECTOR operands 7526// are promoted and implicitly truncated, and the result implicitly 7527// extended. Make that explicit here. 7533// EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 7534// operations are lowered to scalars. 7536// If the indices are the same, return the inserted element else 7537// if the indices are known different, extract the element from 7538// the original vector. 7556// EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed 7557// when vector types are scalarized and v1iX is legal. 7558// vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx). 7559// Here we are completely ignoring the extract element index (N2), 7560// which is fine for fixed width vectors, since any index other than 0 7561// is undefined anyway. However, this cannot be ignored for scalable 7562// vectors - in theory we could support this, but we don't want to do this 7563// without a profitability check. 7572assert(N2C && (
unsigned)N2C->getZExtValue() < 2 &&
"Bad EXTRACT_ELEMENT!");
7576"Wrong types for EXTRACT_ELEMENT!");
7578// EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 7579// 64-bit integers into 32-bit parts. Instead of building the extract of 7580// the BUILD_PAIR, only to have legalize rip it apart, just do it now. 7584// EXTRACT_ELEMENT of a constant int is also very common. 7587unsigned Shift = ElementSize * N2C->getZExtValue();
7588constAPInt &Val = N1C->getAPIntValue();
7595"Extract subvector VTs must be vectors!");
7597"Extract subvector VTs must have the same element type!");
7599"Cannot extract a scalable vector from a fixed length vector!");
7602"Extract subvector must be from larger vector to smaller vector!");
7603assert(N2C &&
"Extract subvector index must be a constant");
7607"Extract subvector overflow!");
7608assert(N2C->getAPIntValue().getBitWidth() ==
7610"Constant index for EXTRACT_SUBVECTOR has an invalid size");
7612// Trivial extraction. 7616// EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. 7620// EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of 7621// the concat have the same type as the extract. 7625return N1.
getOperand(N2C->getZExtValue() / Factor);
7628// EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created 7629// during shuffle legalization. 7637// Perform trivial constant folding. 7641// Canonicalize an UNDEF to the RHS, even over a constant. 7648returngetUNDEF(VT);
// fold op(undef, arg2) -> undef 7661// Fold a bunch of operators when the RHS is undef. 7666// Handle undef ^ undef -> 0 special case. This is a common 7676returngetUNDEF(VT);
// fold op(arg1, undef) -> undef 7689// Memoize this node if possible. 7693if (VT != MVT::Glue) {
7697if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
7698 E->intersectFlagsWith(Flags);
7702N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7704 createOperands(
N, Ops);
7705 CSEMap.InsertNode(
N, IP);
7707N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7708 createOperands(
N, Ops);
7722returngetNode(Opcode,
DL, VT, N1, N2, N3, Flags);
7731"Operand is DELETED_NODE!");
7732// Perform various simplifications. 7742if (N1CFP && N2CFP && N3CFP) {
7756// Attempt to simplify BUILD_VECTOR. 7771"SETCC operands must have the same type!");
7773"SETCC type should be vector iff the operand type is vector!");
7776"SETCC vector element counts must match!");
7777// Use FoldSetCC to simplify SETCC's. 7780// Vector constant folding. 7796if (cast<ConstantSDNode>(N3)->
isZero())
7802// INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except 7803// for scalable vectors where we will generate appropriate code to 7804// deal with out-of-bounds cases correctly. 7809// Undefined index can be assumed out-of-bounds, so that's UNDEF too. 7813// If the inserted element is an UNDEF, just use the input vector. 7820// Inserting undef into undef is still undef. 7826"Dest and insert subvector source types must match!");
7828"Insert subvector VTs must be vectors!");
7830"Insert subvector VTs must have the same element type!");
7832"Cannot insert a scalable vector into a fixed length vector!");
7835"Insert subvector must be from smaller vector to larger vector!");
7836assert(isa<ConstantSDNode>(N3) &&
7837"Insert subvector index must be constant");
7841"Insert subvector overflow!");
7844"Constant index for INSERT_SUBVECTOR has an invalid size");
7846// Trivial insertion. 7850// If this is an insert of an extracted vector into an undef vector, we 7851// can just use the input to the extract. 7858// Fold bit_convert nodes from a type to themselves. 7862case ISD::VP_TRUNCATE:
7863case ISD::VP_SIGN_EXTEND:
7864case ISD::VP_ZERO_EXTEND:
7865// Don't create noop casts. 7873assert(VT == VecVT &&
"Vector and result type don't match.");
7875"All inputs must be vectors.");
7876assert(VecVT == PassthruVT &&
"Vector and passthru types don't match.");
7878"Vector and mask must have same number of elements.");
7887// Memoize node if it doesn't produce a glue result. 7891if (VT != MVT::Glue) {
7895if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
7896 E->intersectFlagsWith(Flags);
7900N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7902 createOperands(
N, Ops);
7903 CSEMap.InsertNode(
N, IP);
7905N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7906 createOperands(
N, Ops);
7918SDValue Ops[] = { N1, N2, N3, N4 };
7919returngetNode(Opcode,
DL, VT, Ops, Flags);
7927returngetNode(Opcode,
DL, VT, N1, N2, N3, N4, Flags);
7933SDValue Ops[] = { N1, N2, N3, N4, N5 };
7934returngetNode(Opcode,
DL, VT, Ops, Flags);
7943returngetNode(Opcode,
DL, VT, N1, N2, N3, N4, N5, Flags);
7946/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 7947/// the incoming stack arguments to be loaded from the stack. 7951// Include the original chain at the beginning of the list. When this is 7952// used by target LowerCall hooks, this helps legalize find the 7953// CALLSEQ_BEGIN node. 7956// Add a chain value for each stack argument. 7960if (FI->getIndex() < 0)
7963// Build a tokenfactor for all the chains. 7967/// getMemsetValue - Vectorized representation of the memset value 7975assert(
C->getAPIntValue().getBitWidth() == 8);
7980return DAG.
getConstant(Val, dl, VT,
false, IsOpaque);
7985assert(
Value.getValueType() == MVT::i8 &&
"memset with non-byte fill value?");
7992// Use a multiplication with 0x010101... to extend the input to the 8001if (VT !=
Value.getValueType())
8007/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 8008/// used when a memcpy is turned into a memset when the source is a constant 8013// Handle vector with all elements zero. 8014if (Slice.
Array ==
nullptr) {
8017if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
8032unsigned NumVTBytes = NumVTBits / 8;
8033unsigned NumBytes = std::min(NumVTBytes,
unsigned(Slice.
Length));
8035APInt Val(NumVTBits, 0);
8037for (
unsigned i = 0; i != NumBytes; ++i)
8040for (
unsigned i = 0; i != NumBytes; ++i)
8041 Val |= (
uint64_t)(
unsignedchar)Slice[i] << (NumVTBytes-i-1)*8;
8044// If the "cost" of materializing the integer immediate is less than the cost 8045// of a load, then it is cost effective to turn the load into the immediate. 8060APInt(
Base.getValueSizeInBits().getFixedValue(),
8061Offset.getKnownMinValue()));
8072EVT BasePtrVT =
Ptr.getValueType();
8076/// Returns true if memcpy source is constant data. 8081G = cast<GlobalAddressSDNode>(Src);
8082elseif (Src.getOpcode() ==
ISD::ADD &&
8085G = cast<GlobalAddressSDNode>(Src.getOperand(0));
8086 SrcDelta = Src.getConstantOperandVal(1);
8092 SrcDelta +
G->getOffset());
8097// On Darwin, -Os means optimize for size without hurting performance, so 8098// only really optimize for size when -Oz (MinSize) is used. 8108assert(OutLoadChains.
size() &&
"Missing loads in memcpy inlining");
8109assert(OutStoreChains.
size() &&
"Missing stores in memcpy inlining");
8111for (
unsigned i =
From; i < To; ++i) {
8113 GluedLoadChains.
push_back(OutLoadChains[i]);
8116// Chain for all loads. 8120for (
unsigned i =
From; i < To; ++i) {
8121StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
8123 ST->getBasePtr(), ST->getMemoryVT(),
8124 ST->getMemOperand());
8134// Turn a memcpy of undef to nop. 8135// FIXME: We need to honor volatile even is Src is undef. 8139// Expand memcpy to a series of load and store ops if the size operand falls 8140// below a certain threshold. 8141// TODO: In the AlwaysInline case, if the size is big then generate a loop 8142// rather than maybe a humongous number of loads and stores. 8146 std::vector<EVT> MemOps;
8147bool DstAlignCanChange =
false;
8153 DstAlignCanChange =
true;
8155if (!SrcAlign || Alignment > *SrcAlign)
8156 SrcAlign = Alignment;
8157assert(SrcAlign &&
"SrcAlign must be set");
8159// If marked as volatile, perform a copy even when marked as constant. 8161bool isZeroConstant = CopyFromConstant && Slice.
Array ==
nullptr;
8165/*IsZeroMemset*/true, isVol)
8167 *SrcAlign, isVol, CopyFromConstant);
8173if (DstAlignCanChange) {
8174Type *Ty = MemOps[0].getTypeForEVT(
C);
8175Align NewAlign =
DL.getABITypeAlign(Ty);
8177// Don't promote to an alignment that would require dynamic stack 8178// realignment which may conflict with optimizations such as tail call 8181if (!
TRI->hasStackRealignment(MF))
8183 NewAlign = std::min(NewAlign, *StackAlign);
8185if (NewAlign > Alignment) {
8186// Give the stack frame object a larger alignment if needed. 8189 Alignment = NewAlign;
8193// Prepare AAInfo for loads/stores after lowering this memcpy. 8197constValue *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.
V);
8199 BatchAA && SrcVal &&
8207unsigned NumMemOps = MemOps.
size();
8209for (
unsigned i = 0; i != NumMemOps; ++i) {
8215// Issuing an unaligned load / store pair that overlaps with the previous 8216// pair. Adjust the offset accordingly. 8217assert(i == NumMemOps-1 && i != 0);
8218 SrcOff -= VTSize -
Size;
8219 DstOff -= VTSize -
Size;
8222if (CopyFromConstant &&
8224// It's unlikely a store of a vector immediate can be done in a single 8225// instruction. It would require a load from a constantpool first. 8226// We only handle zero vectors here. 8227// FIXME: Handle other cases where store of vector immediate is done in 8228// a single instruction. 8230if (SrcOff < Slice.
Length) {
8232 SubSlice.
move(SrcOff);
8234// This is an out-of-bounds access and hence UB. Pretend we read zero. 8235 SubSlice.
Array =
nullptr;
8237 SubSlice.
Length = VTSize;
8240if (
Value.getNode()) {
8244 DstPtrInfo.
getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
8249if (!Store.getNode()) {
8250// The type might not be legal for the target. This should only happen 8251// if the type is smaller than a legal type, as on PPC, so the right 8252// thing to do is generate a LoadExt/StoreTrunc pair. These simplify 8253// to Load/Store if NVT==VT. 8254// FIXME does the case above also need this? 8258bool isDereferenceable =
8261if (isDereferenceable)
8276 DstPtrInfo.
getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
8286unsigned NumLdStInMemcpy = OutStoreChains.
size();
8288if (NumLdStInMemcpy) {
8289// It may be that memcpy might be converted to memset if it's memcpy 8290// of constants. In such a case, we won't have loads and stores, but 8291// just stores. In the absence of loads, there is nothing to gang up. 8293// If target does not care, just leave as it. 8294for (
unsigned i = 0; i < NumLdStInMemcpy; ++i) {
8299// Ld/St less than/equal limit set by target. 8300if (NumLdStInMemcpy <= GluedLdStLimit) {
8302 NumLdStInMemcpy, OutLoadChains,
8305unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
8306unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
8307unsigned GlueIter = 0;
8309for (
unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
8310unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
8311unsigned IndexTo = NumLdStInMemcpy - GlueIter;
8314 OutLoadChains, OutStoreChains);
8315 GlueIter += GluedLdStLimit;
8319if (RemainingLdStInMemcpy) {
8321 RemainingLdStInMemcpy, OutLoadChains,
8333bool isVol,
bool AlwaysInline,
8337// Turn a memmove of undef to nop. 8338// FIXME: We need to honor volatile even is Src is undef. 8342// Expand memmove to a series of load and store ops if the size operand falls 8343// below a certain threshold. 8347 std::vector<EVT> MemOps;
8348bool DstAlignCanChange =
false;
8354 DstAlignCanChange =
true;
8356if (!SrcAlign || Alignment > *SrcAlign)
8357 SrcAlign = Alignment;
8358assert(SrcAlign &&
"SrcAlign must be set");
8368if (DstAlignCanChange) {
8369Type *Ty = MemOps[0].getTypeForEVT(
C);
8370Align NewAlign =
DL.getABITypeAlign(Ty);
8372// Don't promote to an alignment that would require dynamic stack 8373// realignment which may conflict with optimizations such as tail call 8376if (!
TRI->hasStackRealignment(MF))
8378 NewAlign = std::min(NewAlign, *StackAlign);
8380if (NewAlign > Alignment) {
8381// Give the stack frame object a larger alignment if needed. 8384 Alignment = NewAlign;
8388// Prepare AAInfo for loads/stores after lowering this memmove. 8398unsigned NumMemOps = MemOps.
size();
8399for (
unsigned i = 0; i < NumMemOps; i++) {
8404bool isDereferenceable =
8407if (isDereferenceable)
8413 SrcPtrInfo.
getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
8420for (
unsigned i = 0; i < NumMemOps; i++) {
8426 Chain, dl, LoadValues[i],
8428 DstPtrInfo.
getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
8436/// Lower the call to 'memset' intrinsic function into a series of store 8439/// \param DAG Selection DAG where lowered code is placed. 8440/// \param dl Link to corresponding IR location. 8441/// \param Chain Control flow dependency. 8442/// \param Dst Pointer to destination memory location. 8443/// \param Src Value of byte to write into the memory. 8444/// \param Size Number of bytes to write. 8445/// \param Alignment Alignment of the destination in bytes. 8446/// \param isVol True if destination is volatile. 8447/// \param AlwaysInline Makes sure no function call is generated. 8448/// \param DstPtrInfo IR information on the memory pointer. 8449/// \returns New head in the control flow, if lowering was successful, empty 8450/// SDValue otherwise. 8452/// The function tries to replace 'llvm.memset' intrinsic with several store 8453/// operations and value calculation code. This is usually profitable for small 8454/// memory size or when the semantic requires inlining. 8460// Turn a memset of undef to nop. 8461// FIXME: We need to honor volatile even is Src is undef. 8465// Expand memset to a series of load/store ops if the size operand 8466// falls below a certain threshold. 8468 std::vector<EVT> MemOps;
8469bool DstAlignCanChange =
false;
8475 DstAlignCanChange =
true;
8485if (DstAlignCanChange) {
8488Align NewAlign =
DL.getABITypeAlign(Ty);
8490// Don't promote to an alignment that would require dynamic stack 8491// realignment which may conflict with optimizations such as tail call 8494if (!
TRI->hasStackRealignment(MF))
8496 NewAlign = std::min(NewAlign, *StackAlign);
8498if (NewAlign > Alignment) {
8499// Give the stack frame object a larger alignment if needed. 8502 Alignment = NewAlign;
8508unsigned NumMemOps = MemOps.size();
8510// Find the largest store and generate the bit pattern for it. 8511EVT LargestVT = MemOps[0];
8512for (
unsigned i = 1; i < NumMemOps; i++)
8513if (MemOps[i].bitsGT(LargestVT))
8514 LargestVT = MemOps[i];
8517// Prepare AAInfo for loads/stores after lowering this memset. 8521for (
unsigned i = 0; i < NumMemOps; i++) {
8525// Issuing an unaligned load / store pair that overlaps with the previous 8526// pair. Adjust the offset accordingly. 8527assert(i == NumMemOps-1 && i != 0);
8528 DstOff -= VTSize -
Size;
8531// If this store is smaller than the largest store see whether we can get 8532// the smaller value for free with a truncate or extract vector element and 8535if (VT.
bitsLT(LargestVT)) {
8548// Target which can combine store(extractelement VectorTy, Idx) can get 8549// the smaller value for free. 8556assert(
Value.getValueType() == VT &&
"Value with wrong type.");
8573// Lowering memcpy / memset / memmove intrinsics to calls is only valid if all 8574// pointer operands can be losslessly bitcasted to pointers of address space 0 8583Align Alignment,
bool isVol,
bool AlwaysInline,
constCallInst *CI,
8587// Check to see if we should lower the memcpy to loads and stores first. 8588// For cases within the target-specified limits, this is the best choice. 8591// Memcpy with size zero? Just return the original chain. 8592if (ConstantSize->
isZero())
8596 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), Alignment,
8597 isVol,
false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
8598if (Result.getNode())
8602// Then check to see if we should lower the memcpy with target-specific 8603// code. If the target chooses to do this, this is the next best. 8606 *
this, dl, Chain, Dst, Src,
Size, Alignment, isVol, AlwaysInline,
8607 DstPtrInfo, SrcPtrInfo);
8608if (Result.getNode())
8612// If we really need inline code and the target declined to provide it, 8613// use a (potentially long) sequence of loads and stores. 8615assert(ConstantSize &&
"AlwaysInline requires a constant size!");
8617 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), Alignment,
8618 isVol,
true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
8624// FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 8625// memcpy is not guaranteed to be safe. libc memcpys aren't required to 8626// respect volatile, so they may do things like read or write memory 8627// beyond the given memory regions. But fixing this isn't easy, and most 8628// people don't care. 8630// Emit a library call. 8634 Entry.Node = Dst; Args.push_back(Entry);
8635 Entry.Node = Src; Args.push_back(Entry);
8638 Entry.Node =
Size; Args.push_back(Entry);
8639// FIXME: pass in SDLoc 8641bool IsTailCall =
false;
8642if (OverrideTailCall.has_value()) {
8643 IsTailCall = *OverrideTailCall;
8645bool LowersToMemcpy =
8650 ReturnsFirstArg && LowersToMemcpy);
8656 Dst.getValueType().getTypeForEVT(*
getContext()),
8663 std::pair<SDValue,SDValue> CallResult = TLI->
LowerCallTo(CLI);
8664return CallResult.second;
8669Type *SizeTy,
unsigned ElemSz,
8673// Emit a library call. 8678 Args.push_back(Entry);
8681 Args.push_back(Entry);
8685 Args.push_back(Entry);
8689if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8703 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8704return CallResult.second;
8710 std::optional<bool> OverrideTailCall,
8715// Check to see if we should lower the memmove to loads and stores first. 8716// For cases within the target-specified limits, this is the best choice. 8719// Memmove with size zero? Just return the original chain. 8720if (ConstantSize->
isZero())
8724 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), Alignment,
8725 isVol,
false, DstPtrInfo, SrcPtrInfo, AAInfo);
8726if (Result.getNode())
8730// Then check to see if we should lower the memmove with target-specific 8731// code. If the target chooses to do this, this is the next best. 8735 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
8736if (Result.getNode())
8743// FIXME: If the memmove is volatile, lowering it to plain libc memmove may 8744// not be safe. See memcpy above for more details. 8746// Emit a library call. 8750 Entry.Node = Dst; Args.push_back(Entry);
8751 Entry.Node = Src; Args.push_back(Entry);
8754 Entry.Node =
Size; Args.push_back(Entry);
8755// FIXME: pass in SDLoc 8758bool IsTailCall =
false;
8759if (OverrideTailCall.has_value()) {
8760 IsTailCall = *OverrideTailCall;
8762bool LowersToMemmove =
8767 ReturnsFirstArg && LowersToMemmove);
8773 Dst.getValueType().getTypeForEVT(*
getContext()),
8780 std::pair<SDValue,SDValue> CallResult = TLI->
LowerCallTo(CLI);
8781return CallResult.second;
8786Type *SizeTy,
unsigned ElemSz,
8790// Emit a library call. 8795 Args.push_back(Entry);
8798 Args.push_back(Entry);
8802 Args.push_back(Entry);
8806if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8820 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8821return CallResult.second;
8826bool isVol,
bool AlwaysInline,
8830// Check to see if we should lower the memset to stores first. 8831// For cases within the target-specified limits, this is the best choice. 8834// Memset with size zero? Just return the original chain. 8835if (ConstantSize->
isZero())
8840 isVol,
false, DstPtrInfo, AAInfo);
8842if (Result.getNode())
8846// Then check to see if we should lower the memset with target-specific 8847// code. If the target chooses to do this, this is the next best. 8850 *
this, dl, Chain, Dst, Src,
Size, Alignment, isVol, AlwaysInline, DstPtrInfo);
8851if (Result.getNode())
8855// If we really need inline code and the target declined to provide it, 8856// use a (potentially long) sequence of loads and stores. 8858assert(ConstantSize &&
"AlwaysInline requires a constant size!");
8861 isVol,
true, DstPtrInfo, AAInfo);
8863"getMemsetStores must return a valid sequence when AlwaysInline");
8869// Emit a library call. 8874// FIXME: pass in SDLoc 8879// Helper function to create an Entry from Node and Type. 8880constauto CreateEntry = [](
SDValue Node,
Type *Ty) {
8888// If zeroing out and bzero is present, use it. 8892 Args.push_back(CreateEntry(
Size,
DL.getIntPtrType(Ctx)));
8899 Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
8900 Args.push_back(CreateEntry(
Size,
DL.getIntPtrType(Ctx)));
8902 Dst.getValueType().getTypeForEVT(Ctx),
8907bool LowersToMemset =
8909// If we're going to use bzero, make sure not to tail call unless the 8910// subsequent return doesn't need a value, as bzero doesn't return the first 8911// arg unlike memset. 8918 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8919return CallResult.second;
8924Type *SizeTy,
unsigned ElemSz,
8927// Emit a library call. 8932 Args.push_back(Entry);
8936 Args.push_back(Entry);
8940 Args.push_back(Entry);
8944if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8958 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8959return CallResult.second;
8971if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
8972 cast<AtomicSDNode>(E)->refineAlignment(MMO);
8977 VTList, MemVT, MMO);
8978 createOperands(
N, Ops);
8980 CSEMap.InsertNode(
N, IP);
8994returngetAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
9013"Invalid Atomic Op");
9020returngetAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
9030returngetAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
9033/// getMergeValues - Create a MERGE_VALUES node from the given operands. 9050if (
Size.hasValue() && !
Size.getValue())
9067 (Opcode <= (
unsigned)std::numeric_limits<int>::max() &&
9069"Opcode is not a memory-accessing opcode!");
9071// Memoize the node unless it returns a glue result. 9073if (VTList.
VTs[VTList.
NumVTs-1] != MVT::Glue) {
9076ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
9077 Opcode, dl.
getIROrder(), VTList, MemVT, MMO));
9082if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9083 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
9088 VTList, MemVT, MMO);
9089 createOperands(
N, Ops);
9091 CSEMap.InsertNode(
N, IP);
9094 VTList, MemVT, MMO);
9095 createOperands(
N, Ops);
9116ID.AddInteger(FrameIndex);
9120if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
9125 createOperands(
N, Ops);
9126 CSEMap.InsertNode(
N, IP);
9142ID.AddInteger(Index);
9144if (
SDNode *E = FindNodeOrInsertPos(
ID, Dl, IP))
9147auto *
N = newSDNode<PseudoProbeSDNode>(
9149 createOperands(
N, Ops);
9150 CSEMap.InsertNode(
N, IP);
9157/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 9158/// MachinePointerInfo record from it. This is particularly useful because the 9159/// code generator has many cases where it doesn't bother passing in a 9160/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 9164// If this is FI+Offset, we can model it. 9169// If this is (FI+Offset1)+Offset2, we can model it. 9171 !isa<ConstantSDNode>(
Ptr.getOperand(1)) ||
9172 !isa<FrameIndexSDNode>(
Ptr.getOperand(0)))
9175int FI = cast<FrameIndexSDNode>(
Ptr.getOperand(0))->getIndex();
9178Offset + cast<ConstantSDNode>(
Ptr.getOperand(1))->getSExtValue());
9181/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 9182/// MachinePointerInfo record from it. This is particularly useful because the 9183/// code generator has many cases where it doesn't bother passing in a 9184/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 9188// If the 'Offset' value isn't a constant, we can't handle this. 9189if (
ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
9204"Invalid chain type");
9208// If we don't have a PtrInfo, infer the trivial frame index case to simplify 9216 Alignment, AAInfo, Ranges);
9227assert(VT == MemVT &&
"Non-extending load from different memory type!");
9231"Should only be an extending load, not truncating!");
9233"Cannot convert from FP to Int or Int -> FP!");
9235"Cannot use an ext load to convert to or from a vector!");
9238"Cannot use an ext load to change the number of vector elements!");
9250ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
9251 dl.
getIROrder(), VTs, AM, ExtType, MemVT, MMO));
9255if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9256 cast<LoadSDNode>(E)->refineAlignment(MMO);
9260 ExtType, MemVT, MMO);
9261 createOperands(
N, Ops);
9263 CSEMap.InsertNode(
N, IP);
9277 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
9295 MemVT, Alignment, MMOFlags, AAInfo);
9310assert(LD->getOffset().isUndef() &&
"Load is already a indexed load!");
9311// Don't propagate the invariant or dereferenceable flags. 9313 LD->getMemOperand()->getFlags() &
9316 LD->getChain(),
Base,
Offset, LD->getPointerInfo(),
9317 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
9343"Invalid chain type");
9351ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
9356if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9357 cast<StoreSDNode>(E)->refineAlignment(MMO);
9362 createOperands(
N, Ops);
9364 CSEMap.InsertNode(
N, IP);
9377"Invalid chain type");
9398"Invalid chain type");
9403"Should only be a truncating store, not extending!");
9405"Can't do FP-INT conversion!");
9407"Cannot use trunc store to convert to or from a vector!");
9410"Cannot use trunc store to change the number of vector elements!");
9418ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
9423if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9424 cast<StoreSDNode>(E)->refineAlignment(MMO);
9429 createOperands(
N, Ops);
9431 CSEMap.InsertNode(
N, IP);
9442assert(ST->getOffset().isUndef() &&
"Store is already a indexed store!");
9447ID.AddInteger(ST->getMemoryVT().getRawBits());
9448ID.AddInteger(ST->getRawSubclassData());
9449ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9450ID.AddInteger(ST->getMemOperand()->getFlags());
9452if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
9456 ST->isTruncatingStore(), ST->getMemoryVT(),
9457 ST->getMemOperand());
9458 createOperands(
N, Ops);
9460 CSEMap.InsertNode(
N, IP);
9472constMDNode *Ranges,
bool IsExpanding) {
9477// If we don't have a PtrInfo, infer the trivial frame index case to simplify 9485 Alignment, AAInfo, Ranges);
9486returngetLoadVP(AM, ExtType, VT, dl, Chain,
Ptr,
Offset, Mask, EVL, MemVT,
9505ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
9506 dl.
getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9510if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9511 cast<VPLoadSDNode>(E)->refineAlignment(MMO);
9515 ExtType, IsExpanding, MemVT, MMO);
9516 createOperands(
N, Ops);
9518 CSEMap.InsertNode(
N, IP);
9534 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
9543 Mask, EVL, VT, MMO, IsExpanding);
9552constAAMDNodes &AAInfo,
bool IsExpanding) {
9555 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo,
nullptr,
9565 EVL, MemVT, MMO, IsExpanding);
9571auto *LD = cast<VPLoadSDNode>(OrigLoad);
9572assert(LD->getOffset().isUndef() &&
"Load is already a indexed load!");
9573// Don't propagate the invariant or dereferenceable flags. 9575 LD->getMemOperand()->getFlags() &
9579 LD->getVectorLength(), LD->getPointerInfo(),
9580 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),
9581nullptr, LD->isExpandingLoad());
9588bool IsCompressing) {
9598ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9599 dl.
getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9603if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9604 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9608 IsTruncating, IsCompressing, MemVT, MMO);
9609 createOperands(
N, Ops);
9611 CSEMap.InsertNode(
N, IP);
9624bool IsCompressing) {
9645bool IsCompressing) {
9652/*IsTruncating*/false, IsCompressing);
9655"Should only be a truncating store, not extending!");
9658"Cannot use trunc store to convert to or from a vector!");
9661"Cannot use trunc store to change the number of vector elements!");
9665SDValue Ops[] = {Chain, Val,
Ptr, Undef, Mask, EVL};
9669ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9674if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9675 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9681 createOperands(
N, Ops);
9683 CSEMap.InsertNode(
N, IP);
9693auto *ST = cast<VPStoreSDNode>(OrigStore);
9694assert(ST->getOffset().isUndef() &&
"Store is already an indexed store!");
9696SDValue Ops[] = {ST->getChain(), ST->getValue(),
Base,
9697Offset, ST->getMask(), ST->getVectorLength()};
9700ID.AddInteger(ST->getMemoryVT().getRawBits());
9701ID.AddInteger(ST->getRawSubclassData());
9702ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9703ID.AddInteger(ST->getMemOperand()->getFlags());
9705if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
9708auto *
N = newSDNode<VPStoreSDNode>(
9710 ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());
9711 createOperands(
N, Ops);
9713 CSEMap.InsertNode(
N, IP);
9733ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9734DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9738if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
9739 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
9744 newSDNode<VPStridedLoadSDNode>(
DL.getIROrder(),
DL.getDebugLoc(), VTs, AM,
9745 ExtType, IsExpanding, MemVT, MMO);
9746 createOperands(
N, Ops);
9747 CSEMap.InsertNode(
N, IP);
9761 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
9770 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9779bool IsTruncating,
bool IsCompressing) {
9789ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9790DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9793if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
9794 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9797auto *
N = newSDNode<VPStridedStoreSDNode>(
DL.getIROrder(),
DL.getDebugLoc(),
9798 VTs, AM, IsTruncating,
9799 IsCompressing, MemVT, MMO);
9800 createOperands(
N, Ops);
9802 CSEMap.InsertNode(
N, IP);
9814bool IsCompressing) {
9821/*IsTruncating*/false, IsCompressing);
9824"Should only be a truncating store, not extending!");
9827"Cannot use trunc store to convert to or from a vector!");
9830"Cannot use trunc store to change the number of vector elements!");
9834SDValue Ops[] = {Chain, Val,
Ptr, Undef, Stride, Mask, EVL};
9838ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9842if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
9843 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9846auto *
N = newSDNode<VPStridedStoreSDNode>(
DL.getIROrder(),
DL.getDebugLoc(),
9848 IsCompressing, SVT, MMO);
9849 createOperands(
N, Ops);
9851 CSEMap.InsertNode(
N, IP);
9861assert(Ops.
size() == 6 &&
"Incompatible number of operands");
9866ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
9871if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9872 cast<VPGatherSDNode>(E)->refineAlignment(MMO);
9877 VT, MMO, IndexType);
9878 createOperands(
N, Ops);
9880assert(
N->getMask().getValueType().getVectorElementCount() ==
9881N->getValueType(0).getVectorElementCount() &&
9882"Vector width mismatch between mask and data");
9883assert(
N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9884N->getValueType(0).getVectorElementCount().isScalable() &&
9885"Scalable flags of index and data do not match");
9887N->getIndex().getValueType().getVectorElementCount(),
9888N->getValueType(0).getVectorElementCount()) &&
9889"Vector width mismatch between index and data");
9890assert(isa<ConstantSDNode>(
N->getScale()) &&
9891N->getScale()->getAsAPIntVal().isPowerOf2() &&
9892"Scale should be a constant power of 2");
9894 CSEMap.InsertNode(
N, IP);
9905assert(Ops.
size() == 7 &&
"Incompatible number of operands");
9910ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
9915if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9916 cast<VPScatterSDNode>(E)->refineAlignment(MMO);
9920 VT, MMO, IndexType);
9921 createOperands(
N, Ops);
9923assert(
N->getMask().getValueType().getVectorElementCount() ==
9924N->getValue().getValueType().getVectorElementCount() &&
9925"Vector width mismatch between mask and data");
9927N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9928N->getValue().getValueType().getVectorElementCount().isScalable() &&
9929"Scalable flags of index and data do not match");
9931N->getIndex().getValueType().getVectorElementCount(),
9932N->getValue().getValueType().getVectorElementCount()) &&
9933"Vector width mismatch between index and data");
9934assert(isa<ConstantSDNode>(
N->getScale()) &&
9935N->getScale()->getAsAPIntVal().isPowerOf2() &&
9936"Scale should be a constant power of 2");
9938 CSEMap.InsertNode(
N, IP);
9953"Unindexed masked load with an offset!");
9960ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9961 dl.
getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
9965if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9966 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
9970 AM, ExtTy, isExpanding, MemVT, MMO);
9971 createOperands(
N, Ops);
9973 CSEMap.InsertNode(
N, IP);
9984assert(LD->getOffset().isUndef() &&
"Masked load is already a indexed load!");
9986Offset, LD->getMask(), LD->getPassThru(),
9987 LD->getMemoryVT(), LD->getMemOperand(), AM,
9988 LD->getExtensionType(), LD->isExpandingLoad());
9996bool IsCompressing) {
9998"Invalid chain type");
10001"Unindexed masked store with an offset!");
10008ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
10009 dl.
getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
10013if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
10014 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
10019 IsTruncating, IsCompressing, MemVT, MMO);
10020 createOperands(
N, Ops);
10022 CSEMap.InsertNode(
N, IP);
10033assert(ST->getOffset().isUndef() &&
10034"Masked store is already a indexed store!");
10036 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
10037 AM, ST->isTruncatingStore(), ST->isCompressingStore());
10045assert(Ops.
size() == 6 &&
"Incompatible number of operands");
10050ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
10051 dl.
getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
10055if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
10056 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
10061 VTs, MemVT, MMO, IndexType, ExtTy);
10062 createOperands(
N, Ops);
10064assert(
N->getPassThru().getValueType() ==
N->getValueType(0) &&
10065"Incompatible type of the PassThru value in MaskedGatherSDNode");
10066assert(
N->getMask().getValueType().getVectorElementCount() ==
10067N->getValueType(0).getVectorElementCount() &&
10068"Vector width mismatch between mask and data");
10069assert(
N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10070N->getValueType(0).getVectorElementCount().isScalable() &&
10071"Scalable flags of index and data do not match");
10073N->getIndex().getValueType().getVectorElementCount(),
10074N->getValueType(0).getVectorElementCount()) &&
10075"Vector width mismatch between index and data");
10076assert(isa<ConstantSDNode>(
N->getScale()) &&
10077N->getScale()->getAsAPIntVal().isPowerOf2() &&
10078"Scale should be a constant power of 2");
10080 CSEMap.InsertNode(
N, IP);
10092assert(Ops.
size() == 6 &&
"Incompatible number of operands");
10097ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
10098 dl.
getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
10102if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
10103 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
10108 VTs, MemVT, MMO, IndexType, IsTrunc);
10109 createOperands(
N, Ops);
10111assert(
N->getMask().getValueType().getVectorElementCount() ==
10112N->getValue().getValueType().getVectorElementCount() &&
10113"Vector width mismatch between mask and data");
10115N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10116N->getValue().getValueType().getVectorElementCount().isScalable() &&
10117"Scalable flags of index and data do not match");
10119N->getIndex().getValueType().getVectorElementCount(),
10120N->getValue().getValueType().getVectorElementCount()) &&
10121"Vector width mismatch between index and data");
10122assert(isa<ConstantSDNode>(
N->getScale()) &&
10123N->getScale()->getAsAPIntVal().isPowerOf2() &&
10124"Scale should be a constant power of 2");
10126 CSEMap.InsertNode(
N, IP);
10137assert(Ops.
size() == 7 &&
"Incompatible number of operands");
10142ID.AddInteger(getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
10143 dl.
getIROrder(), VTs, MemVT, MMO, IndexType));
10147if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
10148 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
10153 VTs, MemVT, MMO, IndexType);
10154 createOperands(
N, Ops);
10156assert(
N->getMask().getValueType().getVectorElementCount() ==
10157N->getIndex().getValueType().getVectorElementCount() &&
10158"Vector width mismatch between mask and data");
10159assert(isa<ConstantSDNode>(
N->getScale()) &&
10160N->getScale()->getAsAPIntVal().isPowerOf2() &&
10161"Scale should be a constant power of 2");
10162assert(
N->getInc().getValueType().isInteger() &&
"Non integer update value");
10164 CSEMap.InsertNode(
N, IP);
10179ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
10184if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
10189 createOperands(
N, Ops);
10191 CSEMap.InsertNode(
N, IP);
10206ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
10211if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
10216 createOperands(
N, Ops);
10218 CSEMap.InsertNode(
N, IP);
10226// select undef, T, F --> T (if T is a constant), otherwise F 10227// select, ?, undef, F --> F 10228// select, ?, T, undef --> T 10236// select true, T, F --> T 10237// select false, T, F --> F 10241// select ?, T, T --> T 10249// shift undef, Y --> 0 (can always assume that the undef value is 0) 10252// shift X, undef --> undef (because it may shift by the bitwidth) 10261// shift X, C >= bitwidth(X) --> undef 10262// All vector elements must be too big (or undef) to avoid partial undefs. 10264return !Val || Val->getAPIntValue().uge(
X.getScalarValueSizeInBits());
10269// shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined). 10270if (
X.getValueType().getScalarType() == MVT::i1)
10278// If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand 10279// (an undef operand can be chosen to be Nan/Inf), then the result of this 10280// operation is poison. That result can be relaxed to undef. 10283bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
10285bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
10288if (Flags.hasNoNaNs() && (HasNan ||
X.isUndef() ||
Y.isUndef()))
10291if (Flags.hasNoInfs() && (HasInf ||
X.isUndef() ||
Y.isUndef()))
10314if (Opcode ==
ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
10329switch (Ops.
size()) {
10332case 2:
returngetNode(Opcode,
DL, VT, Ops[0], Ops[1]);
10333case 3:
returngetNode(Opcode,
DL, VT, Ops[0], Ops[1], Ops[2]);
10337// Copy from an SDUse array into an SDValue array for use with 10338// the regular getNode logic. 10348returngetNode(Opcode,
DL, VT, Ops, Flags);
10353unsigned NumOps = Ops.
size();
10356case 1:
returngetNode(Opcode,
DL, VT, Ops[0], Flags);
10357case 2:
returngetNode(Opcode,
DL, VT, Ops[0], Ops[1], Flags);
10358case 3:
returngetNode(Opcode,
DL, VT, Ops[0], Ops[1], Ops[2], Flags);
10363for (
constauto &
Op : Ops)
10365"Operand is DELETED_NODE!");
10371// Attempt to simplify BUILD_VECTOR. 10380assert(NumOps == 5 &&
"SELECT_CC takes 5 operands!");
10382"LHS and RHS of condition must have same type!");
10384"True and False arms of SelectCC must have same type!");
10386"select_cc node must be of same type as true and false value!");
10390"Expected select_cc with vector result to have the same sized " 10391"comparison type!");
10394assert(NumOps == 5 &&
"BR_CC takes 5 operands!");
10396"LHS/RHS of comparison should match types!");
10400// If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR 10402 Opcode = ISD::VP_XOR;
10405// If it is VP_MUL mask operation then turn it to VP_AND 10407 Opcode = ISD::VP_AND;
10409case ISD::VP_REDUCE_MUL:
10410// If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND 10412 Opcode = ISD::VP_REDUCE_AND;
10414case ISD::VP_REDUCE_ADD:
10415// If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR 10417 Opcode = ISD::VP_REDUCE_XOR;
10419case ISD::VP_REDUCE_SMAX:
10420case ISD::VP_REDUCE_UMIN:
10421// If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to 10424 Opcode = ISD::VP_REDUCE_AND;
10426case ISD::VP_REDUCE_SMIN:
10427case ISD::VP_REDUCE_UMAX:
10428// If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to 10431 Opcode = ISD::VP_REDUCE_OR;
10439if (VT != MVT::Glue) {
10444if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
10445 E->intersectFlagsWith(Flags);
10449N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
10450 createOperands(
N, Ops);
10452 CSEMap.InsertNode(
N, IP);
10454N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
10455 createOperands(
N, Ops);
10475returngetNode(Opcode,
DL, VTList, Ops, Flags);
10484for (
constauto &
Op : Ops)
10486"Operand is DELETED_NODE!");
10495"Invalid add/sub overflow op!");
10497 Ops[0].getValueType() == Ops[1].getValueType() &&
10498 Ops[0].getValueType() == VTList.
VTs[0] &&
10499"Binary operator types must match!");
10500SDValue N1 = Ops[0], N2 = Ops[1];
10503// (X +- 0) -> X with zero-overflow. 10505/*AllowTruncation*/true);
10506if (N2CV && N2CV->
isZero()) {
10516// {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)} 10522// {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)} 10538"Invalid add/sub overflow op!");
10540 Ops[0].getValueType() == Ops[1].getValueType() &&
10541 Ops[0].getValueType() == VTList.
VTs[0] &&
10542 Ops[2].getValueType() == VTList.
VTs[1] &&
10543"Binary operator types must match!");
10549 VTList.
VTs[0] == Ops[0].getValueType() &&
10550 VTList.
VTs[0] == Ops[1].getValueType() &&
10551"Binary operator types must match!");
10557unsigned OutWidth = Width * 2;
10561 Val = Val.
sext(OutWidth);
10564 Val = Val.
zext(OutWidth);
10579 VTList.
VTs[0] == Ops[0].getValueType() &&
"frexp type mismatch");
10595"Invalid STRICT_FP_EXTEND!");
10597 Ops[1].getValueType().isFloatingPoint() &&
"Invalid FP cast!");
10599"STRICT_FP_EXTEND result type should be vector iff the operand " 10603 Ops[1].getValueType().getVectorElementCount()) &&
10604"Vector element count mismatch!");
10606"Invalid fpext node, dst <= src!");
10609assert(VTList.
NumVTs == 2 && Ops.
size() == 3 &&
"Invalid STRICT_FP_ROUND!");
10611"STRICT_FP_ROUND result type should be vector iff the operand " 10615 Ops[1].getValueType().getVectorElementCount()) &&
10616"Vector element count mismatch!");
10618 Ops[1].getValueType().isFloatingPoint() &&
10619 VTList.
VTs[0].
bitsLT(Ops[1].getValueType()) &&
10621 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
10622"Invalid STRICT_FP_ROUND!");
10625// FIXME: figure out how to safely handle things like 10626// int foo(int x) { return 1 << (x & 255); } 10627// int bar() { return foo(256); } 10632 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
10633returngetNode(Opcode,
DL, VT, N1, N2, N3.getOperand(0));
10635if (
ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
10636// If the and is only masking out bits that cannot effect the shift, 10637// eliminate the and. 10639if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
10640returngetNode(Opcode,
DL, VT, N1, N2, N3.getOperand(0));
10646// Memoize the node unless it returns a glue result. 10648if (VTList.
VTs[VTList.
NumVTs-1] != MVT::Glue) {
10652if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
10653 E->intersectFlagsWith(Flags);
10657N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTList);
10658 createOperands(
N, Ops);
10659 CSEMap.InsertNode(
N, IP);
10661N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTList);
10662 createOperands(
N, Ops);
10691SDValue Ops[] = { N1, N2, N3 };
10697SDValue Ops[] = { N1, N2, N3, N4 };
10704SDValue Ops[] = { N1, N2, N3, N4, N5 };
10712returnmakeVTList(&(*EVTs.insert(VT).first), 1);
10727 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, 2);
10728 VTListMap.InsertNode(Result, IP);
10730return Result->getSDVTList();
10747 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, 3);
10748 VTListMap.InsertNode(Result, IP);
10750return Result->getSDVTList();
10769 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, 4);
10770 VTListMap.InsertNode(Result, IP);
10772return Result->getSDVTList();
10776unsigned NumVTs = VTs.
size();
10778ID.AddInteger(NumVTs);
10779for (
unsigned index = 0; index < NumVTs; index++) {
10780ID.AddInteger(VTs[index].getRawBits());
10788 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, NumVTs);
10789 VTListMap.InsertNode(Result, IP);
10791return Result->getSDVTList();
10795/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 10796/// specified operands. If the resultant node already exists in the DAG, 10797/// this does not modify the specified node, instead it returns the node that 10798/// already exists. If the resultant node does not exist in the DAG, the 10799/// input node is returned. As a degenerate case, if you specify the same 10800/// input operands as the node already has, the input node is returned. 10802assert(
N->getNumOperands() == 1 &&
"Update with wrong number of operands");
10804// Check to see if there is no change. 10805if (
Op ==
N->getOperand(0))
returnN;
10807// See if the modified node already exists. 10808void *InsertPos =
nullptr;
10809if (
SDNode *Existing = FindModifiedNodeSlot(
N,
Op, InsertPos))
10812// Nope it doesn't. Remove the node from its current place in the maps. 10814if (!RemoveNodeFromCSEMaps(
N))
10815 InsertPos =
nullptr;
10817// Now we update the operands. 10818N->OperandList[0].set(
Op);
10821// If this gets put into a CSE map, add it. 10822if (InsertPos) CSEMap.InsertNode(
N, InsertPos);
10827assert(
N->getNumOperands() == 2 &&
"Update with wrong number of operands");
10829// Check to see if there is no change. 10830if (Op1 ==
N->getOperand(0) && Op2 ==
N->getOperand(1))
10831returnN;
// No operands changed, just return the input node. 10833// See if the modified node already exists. 10834void *InsertPos =
nullptr;
10835if (
SDNode *Existing = FindModifiedNodeSlot(
N, Op1, Op2, InsertPos))
10838// Nope it doesn't. Remove the node from its current place in the maps. 10840if (!RemoveNodeFromCSEMaps(
N))
10841 InsertPos =
nullptr;
10843// Now we update the operands. 10844if (
N->OperandList[0] != Op1)
10845N->OperandList[0].set(Op1);
10846if (
N->OperandList[1] != Op2)
10847N->OperandList[1].set(Op2);
10850// If this gets put into a CSE map, add it. 10851if (InsertPos) CSEMap.InsertNode(
N, InsertPos);
10857SDValue Ops[] = { Op1, Op2, Op3 };
10864SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10871SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10877unsigned NumOps = Ops.
size();
10878assert(
N->getNumOperands() == NumOps &&
10879"Update with wrong number of operands");
10881// If no operands changed just return the input node. 10882if (std::equal(Ops.
begin(), Ops.
end(),
N->op_begin()))
10885// See if the modified node already exists. 10886void *InsertPos =
nullptr;
10887if (
SDNode *Existing = FindModifiedNodeSlot(
N, Ops, InsertPos))
10890// Nope it doesn't. Remove the node from its current place in the maps. 10892if (!RemoveNodeFromCSEMaps(
N))
10893 InsertPos =
nullptr;
10895// Now we update the operands. 10896for (
unsigned i = 0; i != NumOps; ++i)
10897if (
N->OperandList[i] != Ops[i])
10898N->OperandList[i].set(Ops[i]);
10901// If this gets put into a CSE map, add it. 10902if (InsertPos) CSEMap.InsertNode(
N, InsertPos);
10906/// DropOperands - Release the operands and set this node to have 10909// Unlike the code in MorphNodeTo that does this, we don't need to 10910// watch for dead nodes here. 10919if (NewMemRefs.
empty()) {
10924// Check if we can avoid allocating by storing a single reference directly. 10925if (NewMemRefs.
size() == 1) {
10926N->MemRefs = NewMemRefs[0];
10932 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.
size());
10934N->MemRefs = MemRefsBuffer;
10935N->NumMemRefs =
static_cast<int>(NewMemRefs.
size());
10938/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 10966SDValue Ops[] = { Op1, Op2, Op3 };
11006// Reset the NodeID to -1. 11007 New->setNodeId(-1);
11015/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away 11016/// the line number information on the merged node since it is not possible to 11017/// preserve the information that operation is associated with multiple lines. 11018/// This will make the debugger working better at -O0, were there is a higher 11019/// probability having other instructions associated with that line. 11021/// For IROrder, we keep the smaller of the two 11027unsigned Order = std::min(
N->getIROrder(), OLoc.
getIROrder());
11028N->setIROrder(Order);
11032/// MorphNodeTo - This *mutates* the specified node to have the specified 11033/// return type, opcode, and operands. 11035/// Note that MorphNodeTo returns the resultant node. If there is already a 11036/// node of the specified opcode and operands, it returns that node instead of 11037/// the current one. Note that the SDLoc need not be the same. 11039/// Using MorphNodeTo is faster than creating a new node and swapping it in 11040/// with ReplaceAllUsesWith both because it often avoids allocating a new 11041/// node, and because it doesn't require CSE recalculation for any of 11042/// the node's users. 11044/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. 11045/// As a consequence it isn't appropriate to use from within the DAG combiner or 11046/// the legalizer which maintain worklists that would need to be updated when 11047/// deleting things. 11050// If an identical node already exists, use it. 11052if (VTs.
VTs[VTs.
NumVTs-1] != MVT::Glue) {
11056return UpdateSDLocOnMergeSDNode(ON,
SDLoc(
N));
11059if (!RemoveNodeFromCSEMaps(
N))
11062// Start the morphing. 11064N->ValueList = VTs.
VTs;
11067// Clear the operands list, updating used nodes to remove this from their 11068// use list. Keep track of any operands that become dead as a result. 11074if (Used->use_empty())
11075 DeadNodeSet.
insert(Used);
11078// For MachineNode, initialize the memory references information. 11080 MN->clearMemRefs();
11082// Swap for an appropriately sized array from the recycler. 11084 createOperands(
N, Ops);
11086// Delete any nodes that are still dead after adding the uses for the 11088if (!DeadNodeSet.
empty()) {
11097 CSEMap.InsertNode(
N, IP);
// Memoize the new node. 11102unsigned OrigOpc = Node->getOpcode();
11107#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 11108 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; 11109#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 11110 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; 11111#include "llvm/IR/ConstrainedOps.def" 11114assert(Node->getNumValues() == 2 &&
"Unexpected number of results!");
11116// We're taking this node out of the chain, so we need to re-link things. 11117SDValue InputChain = Node->getOperand(0);
11122for (
unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
11128// MorphNodeTo can operate in two ways: if an existing node with the 11129// specified operands exists, it can just return it. Otherwise, it 11130// updates the node in place to have the requested operands. 11132// If we updated the node in place, reset the node ID. To the isel, 11133// this should be just like a newly allocated machine node. 11143/// getMachineNode - These are used for target selectors to create a new node 11144/// with specified return type(s), MachineInstr opcode, and operands. 11146/// Note that getMachineNode returns the resultant node. If there is already a 11147/// node of the specified opcode and operands, it returns that node instead of 11148/// the current one. 11173SDValue Ops[] = { Op1, Op2, Op3 };
11195SDValue Ops[] = { Op1, Op2, Op3 };
11219SDValue Ops[] = { Op1, Op2, Op3 };
11240bool DoCSE = VTs.
VTs[VTs.
NumVTs-1] != MVT::Glue;
11248if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
11249return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E,
DL));
11253// Allocate a new MachineSDNode. 11254N = newSDNode<MachineSDNode>(~Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
11255 createOperands(
N, Ops);
11258 CSEMap.InsertNode(
N, IP);
11265/// getTargetExtractSubreg - A convenience function for creating 11266/// TargetOpcode::EXTRACT_SUBREG nodes. 11271 VT, Operand, SRIdxVal);
11275/// getTargetInsertSubreg - A convenience function for creating 11276/// TargetOpcode::INSERT_SUBREG nodes. 11281 VT, Operand, Subreg, SRIdxVal);
11285/// getNodeIfExists - Get the specified node if it's already available, or 11286/// else return NULL. 11298if (VTList.
VTs[VTList.
NumVTs - 1] != MVT::Glue) {
11303 E->intersectFlagsWith(Flags);
11310/// doesNodeExist - Check if a node exists without modifying its flags. 11313if (VTList.
VTs[VTList.
NumVTs - 1] != MVT::Glue) {
11317if (FindNodeOrInsertPos(
ID,
SDLoc(), IP))
11323/// getDbgValue - Creates a SDDbgValue node. 11327SDNode *
N,
unsigned R,
bool IsIndirect,
11329assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11330"Expected inlined-at fields to agree");
11333 {}, IsIndirect,
DL, O,
11334/*IsVariadic=*/false);
11342assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11343"Expected inlined-at fields to agree");
11346/*IsIndirect=*/false,
DL, O,
11347/*IsVariadic=*/false);
11356assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11357"Expected inlined-at fields to agree");
11361/// FrameIndex with dependencies 11368assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11369"Expected inlined-at fields to agree");
11372 Dependencies, IsIndirect,
DL, O,
11373/*IsVariadic=*/false);
11378unsigned VReg,
bool IsIndirect,
11380assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11381"Expected inlined-at fields to agree");
11384 {}, IsIndirect,
DL, O,
11385/*IsVariadic=*/false);
11392unsigned O,
bool IsVariadic) {
11393assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11394"Expected inlined-at fields to agree");
11401unsigned OffsetInBits,
unsigned SizeInBits,
11402bool InvalidateDbg) {
11405assert(FromNode && ToNode &&
"Can't modify dbg values");
11408// TODO: assert(From != To && "Redundant dbg value transfer"); 11409// TODO: assert(FromNode != ToNode && "Intranode dbg value transfer"); 11410if (
From == To || FromNode == ToNode)
11422if (Dbg->isInvalidated())
11425// TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value"); 11427// Create a new location ops vector that is equal to the old vector, but 11428// with each instance of FromLocOp replaced with ToLocOp. 11429bool Changed =
false;
11430auto NewLocOps = Dbg->copyLocationOps();
11432 NewLocOps.begin(), NewLocOps.end(),
11434 bool Match = Op == FromLocOp;
11439// Ignore this SDDbgValue if we didn't find a matching location. 11444auto *Expr = Dbg->getExpression();
11445// If a fragment is requested, update the expression. 11447// When splitting a larger (e.g., sign-extended) value whose 11448// lower bits are described with an SDDbgValue, do not attempt 11449// to transfer the SDDbgValue to the upper bits. 11450if (
auto FI = Expr->getFragmentInfo())
11451if (OffsetInBits + SizeInBits > FI->SizeInBits)
11460auto AdditionalDependencies = Dbg->getAdditionalDependencies();
11461// Clone the SDDbgValue and move it to To. 11463 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
11464 Dbg->getDebugLoc(), std::max(ToNode->
getIROrder(), Dbg->getOrder()),
11465 Dbg->isVariadic());
11468if (InvalidateDbg) {
11469// Invalidate value and indicate the SDDbgValue should not be emitted. 11470 Dbg->setIsInvalidated();
11471 Dbg->setIsEmitted();
11477"Transferred DbgValues should depend on the new SDNode");
11483if (!
N.getHasDebugValue())
11486auto GetLocationOperand = [](
SDNode *Node,
unsigned ResNo) {
11487if (
auto *FISDN = dyn_cast<FrameIndexSDNode>(Node))
11494if (DV->isInvalidated())
11496switch (
N.getOpcode()) {
11502if (!isa<ConstantSDNode>(N0)) {
11503bool RHSConstant = isa<ConstantSDNode>(N1);
11506Offset =
N.getConstantOperandVal(1);
11507// We are not allowed to turn indirect debug values variadic, so 11508// don't salvage those. 11509if (!RHSConstant && DV->isIndirect())
11512// Rewrite an ADD constant node into a DIExpression. Since we are 11513// performing arithmetic to compute the variable's *value* in the 11514// DIExpression, we need to mark the expression with a 11515// DW_OP_stack_value. 11516auto *DIExpr = DV->getExpression();
11517auto NewLocOps = DV->copyLocationOps();
11518bool Changed =
false;
11519size_t OrigLocOpsSize = NewLocOps.size();
11520for (
size_t i = 0; i < OrigLocOpsSize; ++i) {
11521// We're not given a ResNo to compare against because the whole 11522// node is going away. We know that any ISD::ADD only has one 11523// result, so we can assume any node match is using the result. 11525 NewLocOps[i].getSDNode() != &
N)
11533// Convert to a variadic expression (if not already). 11534// convertToVariadicExpression() returns a const pointer, so we use 11535// a temporary const variable here. 11536constauto *TmpDIExpr =
11544 NewLocOps.push_back(
RHS);
11550assert(Changed &&
"Salvage target doesn't use N");
11553 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
11555auto AdditionalDependencies = DV->getAdditionalDependencies();
11557 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
11558 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
11560 DV->setIsInvalidated();
11561 DV->setIsEmitted();
11563 N0.
getNode()->dumprFull(
this);
11564dbgs() <<
" into " << *DIExpr <<
'\n');
11571TypeSize ToSize =
N.getValueSizeInBits(0);
11575auto NewLocOps = DV->copyLocationOps();
11576bool Changed =
false;
11577for (
size_t i = 0; i < NewLocOps.size(); ++i) {
11579 NewLocOps[i].getSDNode() != &
N)
11586assert(Changed &&
"Salvage target doesn't use N");
11591 DV->getAdditionalDependencies(), DV->isIndirect(),
11592 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
11595 DV->setIsInvalidated();
11596 DV->setIsEmitted();
11598dbgs() <<
" into " << *DbgExpression <<
'\n');
11605assert((!Dbg->getSDNodes().empty() ||
11608 return Op.getKind() == SDDbgOperand::FRAMEIX;
11610"Salvaged DbgValue should depend on a new SDNode");
11615/// Creates a SDDbgLabel node. 11618assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(
DL) &&
11619"Expected inlined-at fields to agree");
11625/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 11626/// pointed to by a use iterator is deleted, increment the use iterator 11627/// so that it doesn't dangle. 11634// Increment the iterator as needed. 11635while (UI != UE &&
N == UI->
getUser())
11643 :
SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
11646}
// end anonymous namespace 11648/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 11649/// This can cause recursive merging of nodes in the DAG. 11651/// This version assumes From has a single result value. 11656"Cannot replace with this method!");
11659// Preserve Debug Values 11661// Preserve extra info. 11664// Iterate over all the existing uses of From. New uses will be added 11665// to the beginning of the use list, which we avoid visiting. 11666// This specifically avoids visiting uses of From that arise while the 11667// replacement is happening, because any such uses would be the result 11668// of CSE: If an existing node looks like From after one of its operands 11669// is replaced by To, we don't want to replace of all its users with To 11670// too. See PR3018 for more info. 11672 RAUWUpdateListener Listener(*
this, UI, UE);
11676// This node is about to morph, remove its old self from the CSE maps. 11677 RemoveNodeFromCSEMaps(
User);
11679// A user can appear in a use list multiple times, and when this 11680// happens the uses are usually next to each other in the list. 11681// To help reduce the number of CSE recomputations, process all 11682// the uses of this user that we can find this way. 11690// Now that we have modified User, add it back to the CSE maps. If it 11691// already exists there, recursively merge the results together. 11692 AddModifiedNodeToCSEMaps(
User);
11695// If we just RAUW'd the root, take note. 11700/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 11701/// This can cause recursive merging of nodes in the DAG. 11703/// This version assumes that for each value of From, there is a 11704/// corresponding value in To in the same position with the same type. 11708for (
unsigned i = 0, e =
From->getNumValues(); i != e; ++i)
11711"Cannot use this version of ReplaceAllUsesWith!");
11714// Handle the trivial case. 11718// Preserve Debug Info. Only do this if there's a use. 11719for (
unsigned i = 0, e =
From->getNumValues(); i != e; ++i)
11720if (
From->hasAnyUseOfValue(i)) {
11721assert((i < To->getNumValues()) &&
"Invalid To location");
11724// Preserve extra info. 11727// Iterate over just the existing users of From. See the comments in 11728// the ReplaceAllUsesWith above. 11730 RAUWUpdateListener Listener(*
this, UI, UE);
11734// This node is about to morph, remove its old self from the CSE maps. 11735 RemoveNodeFromCSEMaps(
User);
11737// A user can appear in a use list multiple times, and when this 11738// happens the uses are usually next to each other in the list. 11739// To help reduce the number of CSE recomputations, process all 11740// the uses of this user that we can find this way. 11749// Now that we have modified User, add it back to the CSE maps. If it 11750// already exists there, recursively merge the results together. 11751 AddModifiedNodeToCSEMaps(
User);
11754// If we just RAUW'd the root, take note. 11759/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 11760/// This can cause recursive merging of nodes in the DAG. 11762/// This version can replace From with any result values. To must match the 11763/// number and types of values returned by From. 11765if (
From->getNumValues() == 1)
// Handle the simple case efficiently. 11768for (
unsigned i = 0, e =
From->getNumValues(); i != e; ++i) {
11769// Preserve Debug Info. 11771// Preserve extra info. 11775// Iterate over just the existing users of From. See the comments in 11776// the ReplaceAllUsesWith above. 11778 RAUWUpdateListener Listener(*
this, UI, UE);
11782// This node is about to morph, remove its old self from the CSE maps. 11783 RemoveNodeFromCSEMaps(
User);
11785// A user can appear in a use list multiple times, and when this happens the 11786// uses are usually next to each other in the list. To help reduce the 11787// number of CSE and divergence recomputations, process all the uses of this 11788// user that we can find this way. 11789bool To_IsDivergent =
false;
11798if (To_IsDivergent !=
From->isDivergent())
11801// Now that we have modified User, add it back to the CSE maps. If it 11802// already exists there, recursively merge the results together. 11803 AddModifiedNodeToCSEMaps(
User);
11806// If we just RAUW'd the root, take note. 11811/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 11812/// uses of other values produced by From.getNode() alone. The Deleted 11813/// vector is handled the same way as for ReplaceAllUsesWith. 11815// Handle the really simple, really trivial case efficiently. 11816if (
From == To)
return;
11818// Handle the simple, trivial, case efficiently. 11819if (
From.getNode()->getNumValues() == 1) {
11824// Preserve Debug Info. 11828// Iterate over just the existing users of From. See the comments in 11829// the ReplaceAllUsesWith above. 11831 UE =
From.getNode()->use_end();
11832 RAUWUpdateListener Listener(*
this, UI, UE);
11835bool UserRemovedFromCSEMaps =
false;
11837// A user can appear in a use list multiple times, and when this 11838// happens the uses are usually next to each other in the list. 11839// To help reduce the number of CSE recomputations, process all 11840// the uses of this user that we can find this way. 11844// Skip uses of different values from the same node. 11845if (
Use.getResNo() !=
From.getResNo()) {
11850// If this node hasn't been modified yet, it's still in the CSE maps, 11851// so remove its old self from the CSE maps. 11852if (!UserRemovedFromCSEMaps) {
11853 RemoveNodeFromCSEMaps(
User);
11854 UserRemovedFromCSEMaps =
true;
11862// We are iterating over all uses of the From node, so if a use 11863// doesn't use the specific value, no changes are made. 11864if (!UserRemovedFromCSEMaps)
11867// Now that we have modified User, add it back to the CSE maps. If it 11868// already exists there, recursively merge the results together. 11869 AddModifiedNodeToCSEMaps(
User);
11872// If we just RAUW'd the root, take note. 11879/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 11880/// to record information about a use. 11887/// operator< - Sort Memos by User. 11888booloperator<(
const UseMemo &L,
const UseMemo &R) {
11889return (intptr_t)L.User < (intptr_t)R.User;
11892/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node 11893/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that 11894/// the node already has been taken care of recursively. 11899for (UseMemo &Memo :
Uses)
11901 Memo.User =
nullptr;
11909}
// end anonymous namespace 11911/// Return true if a glue output should propagate divergence information. 11913switch (
Node->getOpcode()) {
11927"Conflicting divergence information!");
11932for (
constauto &
Op :
N->ops()) {
11933EVT VT =
Op.getValueType();
11935// Skip Chain. It does not carry divergence. 11936if (VT != MVT::Other &&
Op.getNode()->isDivergent() &&
11948if (
N->SDNodeBits.IsDivergent != IsDivergent) {
11949N->SDNodeBits.IsDivergent = IsDivergent;
11952 }
while (!Worklist.
empty());
11955void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11957 Order.
reserve(AllNodes.size());
11959unsigned NOps =
N.getNumOperands();
11962 Order.push_back(&
N);
11964for (
size_tI = 0;
I != Order.size(); ++
I) {
11966for (
auto *U :
N->users()) {
11967unsigned &UnsortedOps = Degree[U];
11968if (0 == --UnsortedOps)
11969 Order.push_back(U);
11974#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS 11975void SelectionDAG::VerifyDAGDivergence() {
11976 std::vector<SDNode *> TopoOrder;
11977 CreateTopologicalOrder(TopoOrder);
11978for (
auto *
N : TopoOrder) {
11980"Divergence bit inconsistency detected");
11985/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 11986/// uses of other values produced by From.getNode() alone. The same value 11987/// may appear in both the From and To list. The Deleted vector is 11988/// handled the same way as for ReplaceAllUsesWith. 11992// Handle the simple, trivial case efficiently. 11999// Read up all the uses and make records of them. This helps 12000// processing new uses that are introduced during the 12001// replacement process. 12003for (
unsigned i = 0; i != Num; ++i) {
12004unsigned FromResNo =
From[i].getResNo();
12007if (
Use.getResNo() == FromResNo) {
12009Uses.push_back(Memo);
12014// Sort the uses, so that all the uses from a given User are together. 12016 RAUOVWUpdateListener Listener(*
this,
Uses);
12018for (
unsigned UseIndex = 0, UseIndexEnd =
Uses.size();
12019 UseIndex != UseIndexEnd; ) {
12020// We know that this user uses some value of From. If it is the right 12021// value, update it. 12023// If the node has been deleted by recursive CSE updates when updating 12024// another node, then just skip this entry. 12025if (
User ==
nullptr) {
12030// This node is about to morph, remove its old self from the CSE maps. 12031 RemoveNodeFromCSEMaps(
User);
12033// The Uses array is sorted, so all the uses for a given User 12034// are next to each other in the list. 12035// To help reduce the number of CSE recomputations, process all 12036// the uses of this user that we can find this way. 12038unsigned i =
Uses[UseIndex].Index;
12043 }
while (UseIndex != UseIndexEnd &&
Uses[UseIndex].
User ==
User);
12045// Now that we have modified User, add it back to the CSE maps. If it 12046// already exists there, recursively merge the results together. 12047 AddModifiedNodeToCSEMaps(
User);
12051/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 12052/// based on their topological order. It returns the maximum id and a vector 12053/// of the SDNodes* in assigned order by reference. 12055unsigned DAGSize = 0;
12057// SortedPos tracks the progress of the algorithm. Nodes before it are 12058// sorted, nodes after it are unsorted. When the algorithm completes 12059// it is at the end of the list. 12062// Visit all the nodes. Move nodes with no operands to the front of 12063// the list immediately. Annotate nodes that do have operands with their 12064// operand count. Before we do this, the Node Id fields of the nodes 12065// may contain arbitrary values. After, the Node Id fields for nodes 12066// before SortedPos will contain the topological sort index, and the 12067// Node Id fields for nodes At SortedPos and after will contain the 12068// count of outstanding operands. 12071unsigned Degree =
N.getNumOperands();
12073// A node with no uses, add it to the result array immediately. 12074N.setNodeId(DAGSize++);
12077 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
12078assert(SortedPos != AllNodes.end() &&
"Overran node list");
12081// Temporarily use the Node Id as scratch space for the degree count. 12082N.setNodeId(Degree);
12086// Visit all the nodes. As we iterate, move nodes into sorted order, 12087// such that by the time the end is reached all nodes will be sorted. 12091// N is in sorted position, so all its uses have one less operand 12092// that needs to be sorted. 12094unsigned Degree =
P->getNodeId();
12095assert(Degree != 0 &&
"Invalid node degree");
12098// All of P's operands are sorted, so P may sorted now. 12099P->setNodeId(DAGSize++);
12100if (
P->getIterator() != SortedPos)
12101 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(
P));
12102assert(SortedPos != AllNodes.end() &&
"Overran node list");
12105// Update P's outstanding operand count. 12106P->setNodeId(Degree);
12109if (Node.getIterator() == SortedPos) {
12113dbgs() <<
"Overran sorted position:\n";
12115dbgs() <<
"Checking if this is due to cycles\n";
12122assert(SortedPos == AllNodes.end() &&
12123"Topological sort incomplete!");
12125"First node in topological sort is not the entry token!");
12126assert(AllNodes.front().getNodeId() == 0 &&
12127"First node in topological sort has non-zero id!");
12128assert(AllNodes.front().getNumOperands() == 0 &&
12129"First node in topological sort has operands!");
12130assert(AllNodes.back().getNodeId() == (
int)DAGSize-1 &&
12131"Last node in topologic sort has unexpected id!");
12132assert(AllNodes.back().use_empty() &&
12133"Last node in topologic sort has users!");
12138/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 12139/// value is produced by SD. 12141for (
SDNode *SD : DB->getSDNodes()) {
12145 SD->setHasDebugValue(
true);
12147 DbgInfo->
add(DB, isParameter);
12154assert(isa<MemSDNode>(NewMemOpChain) &&
"Expected a memop node");
12156// The new memory operation must have the same position as the old load in 12157// terms of memory dependency. Create a TokenFactor for the old load and new 12158// memory operation and update uses of the old load's output chain to use that 12160if (OldChain == NewMemOpChain || OldChain.
use_empty())
12161return NewMemOpChain;
12164 OldChain, NewMemOpChain);
12172assert(isa<MemSDNode>(NewMemOp.
getNode()) &&
"Expected a memop node");
12180assert(isa<ExternalSymbolSDNode>(
Op) &&
"Node should be an ExternalSymbol");
12182auto *Symbol = cast<ExternalSymbolSDNode>(
Op)->getSymbol();
12186if (OutFunction !=
nullptr)
12194 std::string ErrorStr;
12196 ErrorFormatter <<
"Undefined external symbol ";
12197 ErrorFormatter <<
'"' << Symbol <<
'"';
12201//===----------------------------------------------------------------------===// 12203//===----------------------------------------------------------------------===// 12207return Const !=
nullptr && Const->isZero();
12216return Const !=
nullptr && Const->isZero() && !Const->isNegative();
12221return Const !=
nullptr && Const->isAllOnes();
12226return Const !=
nullptr && Const->isOne();
12231return Const !=
nullptr && Const->isMinSignedValue();
12235unsigned OperandNo) {
12236// NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity(). 12237// TODO: Target-specific opcodes could be added. 12239/*AllowTruncation*/true)) {
12240APInt Const = ConstV->getAPIntValue().trunc(V.getScalarValueSizeInBits());
12246return Const.isZero();
12248return Const.isOne();
12251return Const.isAllOnes();
12253return Const.isMinSignedValue();
12255return Const.isMaxSignedValue();
12260return OperandNo == 1 && Const.isZero();
12263return OperandNo == 1 && Const.isOne();
12268return ConstFP->isZero() &&
12269 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
12271return OperandNo == 1 && ConstFP->isZero() &&
12272 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
12274return ConstFP->isExactlyValue(1.0);
12276return OperandNo == 1 && ConstFP->isExactlyValue(1.0);
12279// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. 12280EVT VT = V.getValueType();
12282APFloat NeutralAF = !Flags.hasNoNaNs()
12284 : !Flags.hasNoInfs()
12290return ConstFP->isExactlyValue(NeutralAF);
12299 V = V.getOperand(0);
12304while (V.getOpcode() ==
ISD::BITCAST && V.getOperand(0).hasOneUse())
12305 V = V.getOperand(0);
12311 V = V.getOperand(0);
12317 V = V.getOperand(0);
12325unsigned NumBits = V.getScalarValueSizeInBits();
12328returnC && (
C->getAPIntValue().countr_one() >= NumBits);
12332bool AllowTruncation) {
12333EVT VT =
N.getValueType();
12342bool AllowTruncation) {
12346// SplatVectors can truncate their operands. Ignore that case here unless 12347// AllowTruncation is set. 12349EVT VecEltVT =
N->getValueType(0).getVectorElementType();
12350if (
auto *CN = dyn_cast<ConstantSDNode>(
N->getOperand(0))) {
12351EVT CVT = CN->getValueType(0);
12352assert(CVT.
bitsGE(VecEltVT) &&
"Illegal splat_vector element extension");
12353if (AllowTruncation || CVT == VecEltVT)
12360ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
12362// BuildVectors can truncate their operands. Ignore that case here unless 12363// AllowTruncation is set. 12364// TODO: Look into whether we should allow UndefElements in non-DemandedElts 12365if (CN && (UndefElements.
none() || AllowUndefs)) {
12367EVT NSVT =
N.getValueType().getScalarType();
12368assert(CVT.
bitsGE(NSVT) &&
"Illegal build vector element extension");
12369if (AllowTruncation || (CVT == NSVT))
12378EVT VT =
N.getValueType();
12386constAPInt &DemandedElts,
12394 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
12395// TODO: Look into whether we should allow UndefElements in non-DemandedElts 12396if (CN && (UndefElements.
none() || AllowUndefs))
12408// TODO: may want to use peekThroughBitcast() here. 12411returnC &&
C->isZero();
12417returnC &&
C->isOne();
12422unsignedBitWidth =
N.getScalarValueSizeInBits();
12424returnC &&
C->isAllOnes() &&
C->getValueSizeInBits(0) ==
BitWidth;
12433 :
SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
12439// We check here that the size of the memory operand fits within the size of 12440// the MMO. This is because the MMO might indicate only a possible address 12441// range instead of specifying the affected memory addresses precisely. 12448/// Profile - Gather unique data for the node. 12457 std::vector<EVT> VTs;
12466}
// end anonymous namespace 12468/// getValueTypeList - Return a pointer to the specified value type. 12470constEVT *SDNode::getValueTypeList(
MVT VT) {
12471static EVTArray SimpleVTArray;
12474return &SimpleVTArray.VTs[VT.
SimpleTy];
12477/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 12478/// indicated value. This method ignores uses of other values defined by this 12483// TODO: Only iterate over uses of a given value of the node 12485if (U.getResNo() ==
Value) {
12492// Found exactly the right number of uses? 12496/// hasAnyUseOfValue - Return true if there are any use of the indicated 12497/// value. This method ignores uses of other values defined by this operation. 12502if (U.getResNo() ==
Value)
12508/// isOnlyUserOf - Return true if this node is the only use of N. 12521/// Return true if the only users of N are contained in Nodes. 12534/// isOperand - Return true if this node is an operand of N. 12541 [
this](
SDValueOp) { return this == Op.getNode(); });
12544/// reachesChainWithoutSideEffects - Return true if this operand (which must 12545/// be a chain) reaches the specified operand without crossing any 12546/// side-effecting instructions on any chain path. In practice, this looks 12547/// through token factors and non-volatile loads. In order to remain efficient, 12548/// this only looks a couple of nodes in, it does not do an exhaustive search. 12550/// Note that we only need to examine chains when we're searching for 12551/// side-effects; SelectionDAG requires that all side-effects are represented 12552/// by chains, even if another operand would force a specific ordering. This 12553/// constraint is necessary to allow transformations like splitting loads. 12555unsignedDepth)
const{
12556if (*
this == Dest)
returntrue;
12558// Don't search too deeply, we just want to be able to see through 12559// TokenFactor's etc. 12560if (
Depth == 0)
returnfalse;
12562// If this is a token factor, all inputs to the TF happen in parallel. 12564// First, try a shallow search. 12566// We found the chain we want as an operand of this TokenFactor. 12567// Essentially, we reach the chain without side-effects if we could 12568// serialize the TokenFactor into a simple chain of operations with 12569// Dest as the last operation. This is automatically true if the 12570// chain has one use: there are no other ordering constraints. 12571// If the chain has more than one use, we give up: some other 12572// use of Dest might force a side-effect between Dest and the current 12577// Next, try a deep search: check whether every operand of the TokenFactor 12580 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
12584// Loads don't have side effects, look through them. 12585if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(*
this)) {
12586if (Ld->isUnordered())
12587return Ld->getChain().reachesChainWithoutSideEffects(Dest,
Depth-1);
12600 this->Flags &= Flags;
12606bool AllowPartials) {
12607// The pattern must end in an extract from index 0. 12612// Match against one of the candidate binary ops. 12615returnOp.getOpcode() ==
unsigned(BinOp);
12619// Floating-point reductions may require relaxed constraints on the final step 12620// of the reduction because they may reorder intermediate operations. 12621unsigned CandidateBinOp =
Op.getOpcode();
12622if (
Op.getValueType().isFloatingPoint()) {
12624switch (CandidateBinOp) {
12626if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
12634// Matching failed - attempt to see if we did enough stages that a partial 12635// reduction from a subvector is possible. 12636auto PartialReduction = [&](
SDValueOp,
unsigned NumSubElts) {
12637if (!AllowPartials || !
Op)
12639EVT OpVT =
Op.getValueType();
12649// At each stage, we're looking for something that looks like: 12650// %s = shufflevector <8 x i32> %op, <8 x i32> undef, 12651// <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, 12652// i32 undef, i32 undef, i32 undef, i32 undef> 12653// %a = binop <8 x i32> %op, %s 12654// Where the mask changes according to the stage. E.g. for a 3-stage pyramid, 12655// we expect something like: 12656// <4,5,6,7,u,u,u,u> 12657// <2,3,u,u,u,u,u,u> 12658// <1,u,u,u,u,u,u,u> 12659// While a partial reduction match would be: 12660// <2,3,u,u,u,u,u,u> 12661// <1,u,u,u,u,u,u,u> 12662unsigned Stages =
Log2_32(
Op.getValueType().getVectorNumElements());
12664for (
unsigned i = 0; i < Stages; ++i) {
12665unsigned MaskEnd = (1 << i);
12667if (
Op.getOpcode() != CandidateBinOp)
12668return PartialReduction(PrevOp, MaskEnd);
12677 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
12681// The first operand of the shuffle should be the same as the other operand 12684return PartialReduction(PrevOp, MaskEnd);
12686// Verify the shuffle has the expected (at this stage of the pyramid) mask. 12687for (
int Index = 0; Index < (int)MaskEnd; ++Index)
12688if (Shuffle->
getMaskElt(Index) != (int)(MaskEnd + Index))
12689return PartialReduction(PrevOp, MaskEnd);
12694// Handle subvector reductions, which tend to appear after the shuffle 12695// reduction stages. 12696while (
Op.getOpcode() == CandidateBinOp) {
12697unsigned NumElts =
Op.getValueType().getVectorNumElements();
12705unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12706if (NumSrcElts != (2 * NumElts))
12721EVT VT =
N->getValueType(0);
12727// If ResNE is 0, fully unroll the vector op. 12733if (
N->getNumValues() == 2) {
12736EVT VT1 =
N->getValueType(1);
12740for (i = 0; i != NE; ++i) {
12741for (
unsigned j = 0, e =
N->getNumOperands(); j != e; ++j) {
12745// A vector operand; extract a single element. 12756for (; i < ResNE; ++i) {
12768assert(
N->getNumValues() == 1 &&
12769"Can't unroll a vector with multiple results!");
12775for (i= 0; i != NE; ++i) {
12776for (
unsigned j = 0, e =
N->getNumOperands(); j != e; ++j) {
12780// A vector operand; extract a single element. 12785// A scalar operand; just use it as is. 12790switch (
N->getOpcode()) {
12816constauto *ASC = cast<AddrSpaceCastSDNode>(
N);
12818 ASC->getSrcAddressSpace(),
12819 ASC->getDestAddressSpace()));
12825for (; i < ResNE; ++i)
12834unsigned Opcode =
N->getOpcode();
12838"Expected an overflow opcode");
12840EVT ResVT =
N->getValueType(0);
12841EVT OvVT =
N->getValueType(1);
12846// If ResNE is 0, fully unroll the vector op. 12862for (
unsigned i = 0; i < NE; ++i) {
12863SDValue Res =
getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
12886if (LD->isVolatile() ||
Base->isVolatile())
12888// TODO: probably too restrictive for atomics, revisit 12889if (!LD->isSimple())
12891if (LD->isIndexed() ||
Base->isIndexed())
12893if (LD->getChain() !=
Base->getChain())
12895EVT VT = LD->getMemoryVT();
12903if (BaseLocDecomp.equalBaseIndex(LocDecomp, *
this,
Offset))
12904return (Dist * (int64_t)Bytes ==
Offset);
12908/// InferPtrAlignment - Infer alignment of a load / store address. Return 12909/// std::nullopt if it cannot be inferred. 12911// If this is a GlobalAddress + cst, return the alignment. 12913 int64_t GVOffset = 0;
12923// If this is a direct reference to a stack slot, use information about the 12924// stack slot's alignment. 12925int FrameIdx = INT_MIN;
12926 int64_t FrameOffset = 0;
12928 FrameIdx = FI->getIndex();
12930 isa<FrameIndexSDNode>(
Ptr.getOperand(0))) {
12932 FrameIdx = cast<FrameIndexSDNode>(
Ptr.getOperand(0))->getIndex();
12933 FrameOffset =
Ptr.getConstantOperandVal(1);
12936if (FrameIdx != INT_MIN) {
12941return std::nullopt;
12944/// Split the scalar node with EXTRACT_ELEMENT using the provided 12945/// VTs and return the low/high part. 12951"Split node must be a scalar type");
12956return std::make_pair(
Lo,
Hi);
12959/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type 12960/// which is split (or expanded) into two not necessarily identical pieces. 12962// Currently all types are split in half. 12969return std::make_pair(LoVT, HiVT);
12972/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a 12973/// type, dependent on an enveloping VT that has been split into two identical 12974/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size. 12977bool *HiIsEmpty)
const{
12980// custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) 12981// custom VL=9 with enveloping VL=8/8 yields 8/1 12982// custom VL=10 with enveloping VL=8/8 yields 8/2 12987"Mixing fixed width and scalable vectors when enveloping a type");
12994// Flag that hi type has zero storage size, but return split envelop type 12995// (this would be easier if vector types with zero elements were allowed). 13000return std::make_pair(LoVT, HiVT);
13003/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the 13005std::pair<SDValue, SDValue>
13010"Splitting vector with an invalid mixture of fixed and scalable " 13013N.getValueType().getVectorMinNumElements() &&
13014"More vector elements requested than available!");
13018// For scalable vectors it is safe to use LoVT.getVectorMinNumElements() 13019// (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales 13020// IDX with the runtime scaling factor of the result vector type. For 13021// fixed-width result vectors, that runtime scaling factor is 1. 13024return std::make_pair(
Lo,
Hi);
13029// Split the vector length parameter. 13030// %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts). 13031EVT VT =
N.getValueType();
13033"Expecting the mask to be an evenly-sized vector");
13041return std::make_pair(
Lo,
Hi);
13044/// Widen the vector up to the next power of two using INSERT_SUBVECTOR. 13046EVT VT =
N.getValueType();
13055unsigned Start,
unsigned Count,
13057EVT VT =
Op.getValueType();
13063for (
unsigned i = Start, e = Start + Count; i != e; ++i) {
13069// getAddressSpace - Return the address space this GlobalAddress belongs to. 13076return Val.MachineCPVal->getType();
13077return Val.ConstVal->getType();
13081unsigned &SplatBitSize,
13083unsigned MinSplatBits,
13084bool IsBigEndian)
const{
13088if (MinSplatBits > VecWidth)
13091// FIXME: The widths are based on this node's type, but build vectors can 13092// truncate their operands. 13093 SplatValue =
APInt(VecWidth, 0);
13094 SplatUndef =
APInt(VecWidth, 0);
13096// Get the bits. Bits with undefined values (when the corresponding element 13097// of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 13098// in SplatValue. If any of the values are not constant, give up and return 13101assert(NumOps > 0 &&
"isConstantSplat has 0-size build vector");
13104for (
unsigned j = 0; j < NumOps; ++j) {
13105unsigned i = IsBigEndian ? NumOps - 1 - j : j;
13107unsigned BitPos = j * EltWidth;
13110 SplatUndef.
setBits(BitPos, BitPos + EltWidth);
13111elseif (
auto *CN = dyn_cast<ConstantSDNode>(OpVal))
13112 SplatValue.
insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
13113elseif (
auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
13114 SplatValue.
insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
13119// The build_vector is all constants or undefs. Find the smallest element 13120// size that splats the vector. 13121 HasAnyUndefs = (SplatUndef != 0);
13123// FIXME: This does not work for vectors with elements less than 8 bits. 13124while (VecWidth > 8) {
13125// If we can't split in half, stop here. 13129unsigned HalfSize = VecWidth / 2;
13135// If the two halves do not match (ignoring undef bits), stop here. 13136if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
13137 MinSplatBits > HalfSize)
13140 SplatValue = HighValue | LowValue;
13141 SplatUndef = HighUndef & LowUndef;
13143 VecWidth = HalfSize;
13146// FIXME: The loop above only tries to split in halves. But if the input 13147// vector for example is <3 x i16> it wouldn't be able to detect a 13148// SplatBitSize of 16. No idea if that is a design flaw currently limiting 13149// optimizations. I guess that back in the days when this helper was created 13150// vectors normally was power-of-2 sized. 13152 SplatBitSize = VecWidth;
13159if (UndefElements) {
13160 UndefElements->
clear();
13161 UndefElements->
resize(NumOps);
13167for (
unsigned i = 0; i != NumOps; ++i) {
13168if (!DemandedElts[i])
13173 (*UndefElements)[i] =
true;
13174 }
elseif (!Splatted) {
13176 }
elseif (Splatted !=
Op) {
13182unsigned FirstDemandedIdx = DemandedElts.
countr_zero();
13184"Can only have a splat without a constant for all undefs.");
13201if (UndefElements) {
13202 UndefElements->
clear();
13203 UndefElements->
resize(NumOps);
13209// Set the undefs even if we don't find a sequence (like getSplatValue). 13211for (
unsignedI = 0;
I != NumOps; ++
I)
13213 (*UndefElements)[
I] =
true;
13215// Iteratively widen the sequence length looking for repetitions. 13216for (
unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
13217 Sequence.append(SeqLen,
SDValue());
13218for (
unsignedI = 0;
I != NumOps; ++
I) {
13219if (!DemandedElts[
I])
13221SDValue &SeqOp = Sequence[
I % SeqLen];
13228if (SeqOp && !SeqOp.
isUndef() && SeqOp !=
Op) {
13234if (!Sequence.empty())
13238assert(Sequence.empty() &&
"Failed to empty non-repeating sequence pattern");
13251return dyn_cast_or_null<ConstantSDNode>(
13257return dyn_cast_or_null<ConstantSDNode>(
getSplatValue(UndefElements));
13263return dyn_cast_or_null<ConstantFPSDNode>(
13269return dyn_cast_or_null<ConstantFPSDNode>(
getSplatValue(UndefElements));
13276 dyn_cast_or_null<ConstantFPSDNode>(
getSplatValue(UndefElements))) {
13279constAPFloat &APF = CN->getValueAPF();
13285return IntVal.exactLogBase2();
13291bool IsLittleEndian,
unsigned DstEltSizeInBits,
13293// Early-out if this contains anything but Undef/Constant/ConstantFP. 13299assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
13300"Invalid bitcast scale");
13302// Extract raw src bits. 13305BitVector SrcUndeElements(NumSrcOps,
false);
13307for (
unsignedI = 0;
I != NumSrcOps; ++
I) {
13310 SrcUndeElements.
set(
I);
13313auto *CInt = dyn_cast<ConstantSDNode>(
Op);
13314auto *CFP = dyn_cast<ConstantFPSDNode>(
Op);
13315assert((CInt || CFP) &&
"Unknown constant");
13316 SrcBitElements[
I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
13317 : CFP->getValueAPF().bitcastToAPInt();
13320// Recast to dst width. 13321recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
13322 SrcBitElements, UndefElements, SrcUndeElements);
13327unsigned DstEltSizeInBits,
13332unsigned NumSrcOps = SrcBitElements.
size();
13333unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
13334assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
13335"Invalid bitcast scale");
13336assert(NumSrcOps == SrcUndefElements.
size() &&
13337"Vector size mismatch");
13339unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
13340 DstUndefElements.
clear();
13341 DstUndefElements.
resize(NumDstOps,
false);
13344// Concatenate src elements constant bits together into dst element. 13345if (SrcEltSizeInBits <= DstEltSizeInBits) {
13346unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
13347for (
unsignedI = 0;
I != NumDstOps; ++
I) {
13348 DstUndefElements.
set(
I);
13349APInt &DstBits = DstBitElements[
I];
13350for (
unsigned J = 0; J != Scale; ++J) {
13351unsignedIdx = (
I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13352if (SrcUndefElements[
Idx])
13354 DstUndefElements.
reset(
I);
13355constAPInt &SrcBits = SrcBitElements[
Idx];
13357"Illegal constant bitwidths");
13358 DstBits.
insertBits(SrcBits, J * SrcEltSizeInBits);
13364// Split src element constant bits into dst elements. 13365unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
13366for (
unsignedI = 0;
I != NumSrcOps; ++
I) {
13367if (SrcUndefElements[
I]) {
13368 DstUndefElements.
set(
I * Scale, (
I + 1) * Scale);
13371constAPInt &SrcBits = SrcBitElements[
I];
13372for (
unsigned J = 0; J != Scale; ++J) {
13373unsignedIdx = (
I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13374APInt &DstBits = DstBitElements[
Idx];
13375 DstBits = SrcBits.
extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
13382unsigned Opc =
Op.getOpcode();
13389std::optional<std::pair<APInt, APInt>>
13393return std::nullopt;
13397return std::nullopt;
13404return std::nullopt;
13406for (
unsigned i = 2; i < NumOps; ++i) {
13408return std::nullopt;
13411if (Val != (Start + (Stride * i)))
13412return std::nullopt;
13415return std::make_pair(Start, Stride);
13419// Find the first non-undef value in the shuffle mask. 13424// If all elements are undefined, this shuffle can be considered a splat 13425// (although it should eventually get simplified away completely). 13429// Make sure all remaining elements are either undef or the same as the first 13431for (
intIdx = Mask[i]; i != e; ++i)
13432if (Mask[i] >= 0 && Mask[i] !=
Idx)
13437// Returns true if it is a constant integer BuildVector or constant integer, 13438// possibly hidden by a bitcast. 13443if (
auto *
C = dyn_cast<ConstantSDNode>(
N))
13444return AllowOpaques || !
C->isOpaque();
13449// Treat a GlobalAddress supporting constant offset folding as a 13450// constant integer. 13451if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
N))
13457 isa<ConstantSDNode>(
N.getOperand(0)))
13462// Returns true if it is a constant float BuildVector or constant float. 13464if (isa<ConstantFPSDNode>(
N))
13471 isa<ConstantFPSDNode>(
N.getOperand(0)))
13478bool AllowTruncation)
const{
13481return std::nullopt;
13483constAPInt &CVal = Const->getAPIntValue();
13490return std::nullopt;
13496return std::nullopt;
13504assert(!Node->OperandList &&
"Node already has operands");
13506"too many operands to fit into SDNode");
13507SDUse *Ops = OperandRecycler.allocate(
13510bool IsDivergent =
false;
13511for (
unsignedI = 0;
I != Vals.
size(); ++
I) {
13512 Ops[
I].setUser(Node);
13513 Ops[
I].setInitial(Vals[
I]);
13516// Skip Chain. It does not carry divergence. 13517if (VT != MVT::Other &&
13519 Ops[
I].
getNode()->isDivergent()) {
13524Node->OperandList = Ops;
13527Node->SDNodeBits.IsDivergent = IsDivergent;
13535while (Vals.
size() > Limit) {
13536unsigned SliceIdx = Vals.
size() - Limit;
13565// If flags allow, prefer positive zero since it's generally cheaper 13566// to materialize on most targets. 13572// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. 13584// Neutral element for fminimum is Inf or FLT_MAX, depending on FMF. 13597/// Helper used to make a call to a library function that has one argument of 13600/// Such functions include 'fegetmode', 'fesetenv' and some others, which are 13601/// used to get or set floating-point state. They have one argument of pointer 13602/// type, which points to the memory region containing bits of the 13603/// floating-point state. The value returned by such function is ignored in the 13606/// \param LibFunc Reference to library function (value of RTLIB::Libcall). 13607/// \param Ptr Pointer used to save/load state. 13608/// \param InChain Ingoing token chain. 13609/// \returns Outgoing chain token. 13618 Args.push_back(Entry);
13630assert(
From && To &&
"Invalid SDNode; empty source SDValue?");
13631autoI = SDEI.find(
From);
13632if (
I == SDEI.end())
13635// Use of operator[] on the DenseMap may cause an insertion, which invalidates 13636// the iterator, hence the need to make a copy to prevent a use-after-free. 13637 NodeExtraInfo NEI =
I->second;
13639// No deep copy required for the types of extra info set. 13641// FIXME: Investigate if other types of extra info also need deep copy. This 13642// depends on the types of nodes they can be attached to: if some extra info 13643// is only ever attached to nodes where a replacement To node is always the 13644// node where later use and propagation of the extra info has the intended 13645// semantics, no deep copy is required. 13646 SDEI[To] = std::move(NEI);
13650// We need to copy NodeExtraInfo to all _new_ nodes that are being introduced 13651// through the replacement of From with To. Otherwise, replacements of a node 13652// (From) with more complex nodes (To and its operands) may result in lost 13653// extra info where the root node (To) is insignificant in further propagating 13654// and using extra info when further lowering to MIR. 13656// In the first step pre-populate the visited set with the nodes reachable 13657// from the old From node. This avoids copying NodeExtraInfo to parts of the 13658// DAG that is not new and should be left untouched. 13661auto VisitFrom = [&](
auto &&Self,
constSDNode *
N,
int MaxDepth) {
13662if (MaxDepth == 0) {
13663// Remember this node in case we need to increase MaxDepth and continue 13664// populating FromReach from this node. 13665 Leafs.emplace_back(
N);
13668if (!FromReach.
insert(
N).second)
13671 Self(Self,
Op.getNode(), MaxDepth - 1);
13674// Copy extra info to To and all its transitive operands (that are new). 13676auto DeepCopyTo = [&](
auto &&Self,
constSDNode *
N) {
13679if (!Visited.
insert(
N).second)
13684if (!Self(Self,
Op.getNode()))
13687// Copy only if entry node was not reached. 13692// We first try with a lower MaxDepth, assuming that the path to common 13693// operands between From and To is relatively short. This significantly 13694// improves performance in the common case. The initial MaxDepth is big 13695// enough to avoid retry in the common case; the last MaxDepth is large 13696// enough to avoid having to use the fallback below (and protects from 13697// potential stack exhaustion from recursion). 13698for (
int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
13699 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.
clear()) {
13700// StartFrom is the previous (or initial) set of leafs reachable at the 13701// previous maximum depth. 13704for (
constSDNode *
N : StartFrom)
13705 VisitFrom(VisitFrom,
N, MaxDepth - PrevDepth);
13708// This should happen very rarely (reached the entry node). 13709LLVM_DEBUG(
dbgs() << __func__ <<
": MaxDepth=" << MaxDepth <<
" too low\n");
13713// This should not happen - but if it did, that means the subgraph reachable 13714// from From has depth greater or equal to maximum MaxDepth, and VisitFrom() 13715// could not visit all reachable common operands. Consequently, we were able 13716// to reach the entry node. 13717errs() <<
"warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
13718assert(
false &&
"From subgraph too complex - increase max. MaxDepth?");
13719// Best-effort fallback if assertions disabled. 13720 SDEI[To] = std::move(NEI);
13728// If this node has already been checked, don't check it again. 13732// If a node has already been visited on this depth-first walk, reject it as 13734if (!Visited.
insert(
N).second) {
13735errs() <<
"Detected cycle in SelectionDAG\n";
13736dbgs() <<
"Offending node:\n";
13737N->dumprFull(DAG);
dbgs() <<
"\n";
13754#ifdef EXPENSIVE_CHECKS 13756#endif// EXPENSIVE_CHECKS 13758assert(
N &&
"Checking nonexistent SDNode");
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define __asan_unpoison_memory_region(p, size)
#define LLVM_LIKELY(EXPR)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is MaybeLiveUses might be modified but its content should be ignored(since it might not be complete). DeadArgumentEliminationPass
Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
Given that RA is a live value
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines a hash set that can be used to remove duplication of nodes in a graph.
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
mir Rename Register Operands
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
Remove Loads Into Fake Uses
Contains matchers for matching SelectionDAG nodes and values.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
This file contains some templates that are useful if you are working with the STL at all.
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, BatchAAResults *BatchAA)
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
static std::optional< APInt > FoldValueWithUndef(unsigned Opcode, const APInt &C1, bool IsUndef1, const APInt &C2, bool IsUndef2)
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, SelectionDAG &DAG)
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, SDVTList VTList, ArrayRef< SDValue > OpList)
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, const TargetLowering &TLI, const ConstantDataArraySlice &Slice)
getMemsetStringVal - Similar to getMemsetValue.
static cl::opt< bool > EnableMemCpyDAGOpt("enable-memcpy-dag-opt", cl::Hidden, cl::init(true), cl::desc("Gang up loads and stores generated by inlining of memcpy"))
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B)
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList)
AddNodeIDValueTypes - Value type lists are intern'd so we can represent them solely with their pointe...
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef< int > M)
Swaps the values of N1 and N2.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice)
Returns true if memcpy source is constant data.
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo)
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)
AddNodeIDOpcode - Add the node opcode to the NodeID data.
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike)
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
static cl::opt< int > MaxLdStGlue("ldstmemcpy-glue-max", cl::desc("Number limit for gluing ld/st of memcpy."), cl::Hidden, cl::init(0))
static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef< SDValue > Ops)
AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode, SDNode *FPNode)
Given a store node StoreNode, return true if it is safe to fold that node into FPNode,...
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
Try to simplify vector concatenation to an input value, undef, or build vector.
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, SelectionDAG &DAG, SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N)
If this is an SDNode with special info, add this info to the NodeID data.
static bool gluePropagatesDivergence(const SDNode *Node)
Return true if a glue output should propagate divergence information.
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G)
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members.
static void VerifySDNode(SDNode *N, const TargetLowering *TLI)
VerifySDNode - Check the given SDNode. Aborts if it is invalid.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SmallVector< SDValue, 32 > &OutChains, unsigned From, unsigned To, SmallVector< SDValue, 16 > &OutLoadChains, SmallVector< SDValue, 16 > &OutStoreChains)
static int isSignedOp(ISD::CondCode Opcode)
For an integer comparison, return 1 if the comparison is a signed operation and 2 if the result is an...
static std::optional< APInt > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, unsigned AS)
static cl::opt< unsigned > MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(8192), cl::desc("DAG combiner limit number of steps when searching DAG " "for predecessor nodes"))
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR)
Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
static unsigned getSize(unsigned Kind)
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
opStatus divide(const APFloat &RHS, roundingMode RM)
void copySign(const APFloat &RHS)
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
opStatus subtract(const APFloat &RHS, roundingMode RM)
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
opStatus add(const APFloat &RHS, roundingMode RM)
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus multiply(const APFloat &RHS, roundingMode RM)
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
opStatus mod(const APFloat &RHS)
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
APInt umul_ov(const APInt &RHS, bool &Overflow) const
APInt usub_sat(const APInt &RHS) const
APInt udiv(const APInt &RHS) const
Unsigned division operation.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
unsigned popcount() const
Count the number of bits set.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt abs() const
Get the absolute value.
APInt sadd_sat(const APInt &RHS) const
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
void clearAllBits()
Set every bit to 0.
APInt rotr(unsigned rotateAmt) const
Rotate right by rotateAmt.
APInt reverseBits() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
bool sle(const APInt &RHS) const
Signed less or equal comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
APInt sshl_sat(const APInt &RHS) const
APInt ushl_sat(const APInt &RHS) const
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
APInt rotl(unsigned rotateAmt) const
Rotate left by rotateAmt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
unsigned logBase2() const
APInt uadd_sat(const APInt &RHS) const
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
void setAllBits()
Set every bit to 1.
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
APInt sext(unsigned width) const
Sign extend to a new width.
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
APInt ssub_sat(const APInt &RHS) const
An arbitrary precision integer that knows its signedness.
unsigned getSrcAddressSpace() const
unsigned getDestAddressSpace() const
Recycle small arrays allocated from a BumpPtrAllocator.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
void clear()
clear - Removes all bits from the bitvector.
bool none() const
none - Returns true if none of the bits are set.
size_type size() const
size - Returns the number of bits in this bitvector.
int64_t getOffset() const
unsigned getTargetFlags() const
const BlockAddress * getBlockAddress() const
The address of a basic block.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &RawBitElements, BitVector &UndefElements) const
Extract the raw bit data from a build vector of Undef, Constant or ConstantFP node elements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &DstBitElements, ArrayRef< APInt > SrcBitElements, BitVector &DstUndefElements, const BitVector &SrcUndefElements)
Recast bit data SrcBitElements to DstEltSizeInBits wide elements.
bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
ConstantFPSDNode * getConstantFPSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant FP or null if this is not a constant FP splat.
std::optional< std::pair< APInt, APInt > > isConstantSequence() const
If this BuildVector is constant and represents the numerical series "<a, a+n, a+2n,...
SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
void Reset()
Deallocate all but the current slab and reset the current pointer to the beginning of it,...
This class represents a function call, abstracting a target machine's calling convention.
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
ConstantFP - Floating Point Values [float, double].
const APFloat & getValue() const
This is the shared class of boolean and integer constants.
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
const APInt & getValue() const
Return the constant as an APInt value reference.
bool isMachineConstantPoolEntry() const
This class represents a range of values.
ConstantRange multiply(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
KnownBits toKnownBits() const
Return known bits for values in this range.
ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const
Return whether unsigned mul of the two ranges always/never overflows.
ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
OverflowResult
Represents whether an operation on the given constant range is known to always or never overflow.
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
static ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed)
Returns the ops for a zero- or sign-extension in a DIExpression.
static void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static const DIExpression * convertToVariadicExpression(const DIExpression *Expr)
If Expr is a non-variadic expression (i.e.
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
Base class for variables.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Implements a dense probed hash-table based set.
const char * getSymbol() const
unsigned getTargetFlags() const
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
MachineBasicBlock * MBB
MBB - The current block.
Data structure describing the variable locations in a function.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
int64_t getOffset() const
unsigned getAddressSpace() const
unsigned getTargetFlags() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
This class is used to form a handle around another node that is persistent and is updated across invo...
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
constexpr bool isValid() const
This is an important class for using LLVM in a threaded context.
This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate the offet and size that ar...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getIntegerVT(unsigned BitWidth)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Abstract base class for all machine specific constantpool value subclasses.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
bool isNonTemporal() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
bool isDereferenceable() const
This class contains meta information specific to a module.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *MMO)
MachineMemOperand * MMO
Memory reference information.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
unsigned getRawSubclassData() const
Return the SubclassData value, without HasDebugValue.
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
A Module instance is used to store all the information related to an LLVM module.
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
The optimization diagnostic interface.
Pass interface - Implemented by all 'passes'.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
unsigned getAddressSpace() const
Return the address space of the Pointer type.
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
Analysis providing profile information.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
Wrapper class representing virtual and physical registers.
Keeps track of dbg_value information through SDISel.
BumpPtrAllocator & getAlloc()
void add(SDDbgValue *V, bool isParameter)
void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
ArrayRef< SDDbgValue * > getSDDbgValues(const SDNode *Node) const
Holds the information from a dbg_label node through SDISel.
Holds the information for a single machine location through SDISel; either an SDNode,...
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
@ SDNODE
Value is the result of an expression.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
unsigned getIROrder() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
static constexpr size_t getMaxNumOperands()
Return the maximum number of operands that a SDNode can hold.
iterator_range< use_iterator > uses()
MemSDNodeBitfields MemSDNodeBits
void Profile(FoldingSetNodeID &ID) const
Gather unique data for the node.
bool getHasDebugValue() const
SDNodeFlags getFlags() const
void setNodeId(int Id)
Set unique node id.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
void DropOperands()
Release the operands and set this node to have zero operands.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
SDNode * getUser()
This returns the SDNode that contains this Use.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
virtual bool isTargetMemoryOpcode(unsigned Opcode) const
Returns true if a node with the given target-specific opcode has a memory operand.
virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const
Emit target-specific code that performs a memset.
virtual SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memmove.
virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memcpy.
SDNodeFlags getFlags() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const
Determine if the result of the unsigned sub of 2 nodes can overflow.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
void updateDivergence(SDNode *N)
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
SelectionDAG(const TargetMachine &TM, CodeGenOptLevel)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
static unsigned getHasPredecessorMaxSteps()
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
bool cannotBeOrderedNegativeFP(SDValue Op) const
Test whether the given float value is known to be positive.
SDValue getRegister(Register Reg, EVT VT)
bool calculateDivergence(SDNode *N)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
std::optional< uint64_t > getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
bool isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
const DataLayout & getDataLayout() const
SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
OverflowKind computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const
Determine if the result of the unsigned mul of 2 nodes can overflow.
void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, const SDLoc &DLoc)
Helper used to make a call to a library function that has one argument of pointer type.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
OverflowKind computeOverflowForSignedMul(SDValue N0, SDValue N1) const
Determine if the result of the signed mul of 2 nodes can overflow.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
SDValue getPartialReduceAdd(SDLoc DL, EVT ReducedTy, SDValue Op1, SDValue Op2)
Create the DAG equivalent of vector_partial_reduce where Op1 and Op2 are its operands and ReducedTY i...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, SDValue EVL, EVT VT)
Create a vector-predicated logical NOT operation as (VP_XOR Val, BooleanOne, Mask,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
std::optional< uint64_t > getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT SVT, MachineMemOperand *MMO, bool IsCompressing=false)
void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
bool isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth=0) const
Test if the given fp value is known to be an integer power-of-2, either positive or negative.
OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const
Determine if the result of the signed sub of 2 nodes can overflow.
bool expandMultipleResultFPLibCall(RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={})
Expands a node with multiple results to an FP or vector libcall.
std::optional< uint64_t > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
LLVMContext * getContext() const
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
Fold floating-point operations when all operands are constants and/or undefined.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
std::optional< ConstantRange > getValidShiftAmountRange(SDValue V, const APInt &DemandedElts, unsigned Depth) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
std::optional< bool > isBoolConstant(SDValue N, bool AllowTruncation=false) const
Check if a value \op N is a constant using the target's BooleanContent for its type.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
bool isADDLike(SDValue Op, bool NoWrap=false) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
ilist< SDNode >::iterator allnodes_iterator
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Completely target-dependent object reference.
int64_t getOffset() const
unsigned getTargetFlags() const
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
const TargetMachine & getTargetMachine() const
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool shallExtractConstSplatVectorElementToStore(Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const
Return true if the target shall perform extract vector element and store given that the vector is kno...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
virtual unsigned getMaxGluedStoresPerMemcpy() const
Get maximum # of store operations to be glued together.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
virtual bool hasVectorBlend() const
Return true if the target has a vector blend instruction.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
virtual void verifyTargetSDNode(const SDNode *N) const
Check the given SDNode. Aborts if it is invalid.
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const
virtual bool isSDNodeAlwaysUniform(const SDNode *N) const
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
Provides info so a possible vectorization of a function can be computed.
StringRef getVectorFnName() const
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt mulhu(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on zero-extended operands.
const APInt abdu(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be unsigned.
APInt avgCeilU(const APInt &C1, const APInt &C2)
Compute the ceil of the unsigned average of C1 and C2.
APInt avgFloorU(const APInt &C1, const APInt &C2)
Compute the floor of the unsigned average of C1 and C2.
const APInt abds(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be signed.
APInt mulhs(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on sign-extended operands.
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
APInt avgFloorS(const APInt &C1, const APInt &C2)
Compute the floor of the signed average of C1 and C2.
APInt avgCeilS(const APInt &C1, const APInt &C2)
Compute the ceil of the signed average of C1 and C2.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SRCVALUE
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ TargetIndex
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EXPERIMENTAL_VECTOR_HISTOGRAM
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ SET_FPENV_MEM
Sets the current floating point environment.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool matchUnaryFpPredicate(SDValue Op, std::function< bool(ConstantFPSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantFPSDNode predicate.
bool isExtOpcode(unsigned Opcode)
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed)
Returns true if the specified node is a vector where all elements can be truncated to the specified e...
bool matchUnaryPredicateImpl(SDValue Op, std::function< bool(ConstNodeType *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getBaseOpcodeForVP(unsigned Opcode, bool hasFPExcept)
Translate this VP Opcode to its corresponding non-VP Opcode.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
bool isFreezeUndef(const SDNode *N)
Return true if the specified node is FREEZE(UNDEF).
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
std::optional< unsigned > getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
NodeType getInverseMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns ISD::(U|S)MAX and ISD::(U|S)MIN,...
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
initializer< Ty > init(const Ty &Val)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
This is an optimization pass for GlobalISel generic memory operations.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
bool operator<(int64_t V1, const APSInt &V2)
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
SDValue peekThroughExtractSubvectors(SDValue V)
Return the non-extracted vector source operand of V if it exists.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
MaybeAlign getAlign(const Function &F, unsigned Index)
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 maximumNumber semantics.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void checkForCycles(const SelectionDAG *DAG, bool force=false)
void sort(IteratorTy Start, IteratorTy End)
LLVM_READONLY APFloat minimumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimumNumber semantics.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
CodeGenOptLevel
Code generation optimization level.
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 minimumNumber semantics.
@ Mul
Product of integers.
bool isNullConstantOrUndef(SDValue V)
Returns true if V is a constant integer zero or an UNDEF node.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
OutputIt copy(R &&Range, OutputIt Out)
constexpr unsigned BitWidth
bool funcReturnsFirstArgOfCall(const CallInst &CI)
Returns true if the parent of CI returns CI's first argument after calling CI.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known)
Compute known bits from the range metadata.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
LLVM_READONLY APFloat maximumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximumNumber semantics.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
MDNode * TBAA
The tag for type-based alias analysis.
static const fltSemantics & IEEEsingle() LLVM_READNONE
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
static constexpr roundingMode rmTowardNegative
static constexpr roundingMode rmNearestTiesToEven
static constexpr roundingMode rmTowardZero
static const fltSemantics & IEEEquad() LLVM_READNONE
static const fltSemantics & IEEEdouble() LLVM_READNONE
static const fltSemantics & IEEEhalf() LLVM_READNONE
static constexpr roundingMode rmTowardPositive
static const fltSemantics & BFloat() LLVM_READNONE
opStatus
IEEE-754R 7: Default exception handling.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
void move(uint64_t Delta)
Moves the Offset and adjusts Length accordingly.
const ConstantDataArray * Array
ConstantDataArray pointer.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
intptr_t getRawBits() const
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
KnownBits sextInReg(unsigned SrcBitWidth) const
Return known bits for a in-register sign extension of the value we're tracking.
static KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from zero-extended multiply-hi.
unsigned countMinSignBits() const
Returns the number of times the sign bit is replicated into the other bits.
static KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
bool isNonNegative() const
Returns true if this value is known to be non-negative.
bool isZero() const
Returns true if value is all zero.
void makeNonNegative()
Make this value non-negative.
static KnownBits usub_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.usub.sat(LHS, RHS)
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
static KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
void makeNegative()
Make this value negative.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
KnownBits byteSwap() const
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
void setAllZero()
Make all bits known to be zero and discard any previous information.
KnownBits reverseBits() const
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
unsigned getBitWidth() const
Get the bit width of this value.
static KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits unionWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for either this or RHS or both.
static KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
bool isNonZero() const
Returns true if this value is known to be non-zero.
static KnownBits abdu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for abdu(LHS, RHS).
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
static KnownBits avgFloorU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorU.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS, const KnownBits &Borrow)
Compute known bits results from subtracting RHS from LHS with 1-bit Borrow.
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
static KnownBits abds(KnownBits LHS, KnownBits RHS)
Compute known bits for abds(LHS, RHS).
static KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from sign-extended multiply-hi.
static KnownBits srem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for srem(LHS, RHS).
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
static KnownBits sdiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for sdiv(LHS, RHS).
static KnownBits avgFloorS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorS.
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS)
Return true if LHS and RHS have no common bits set.
bool isNegative() const
Returns true if this value is known to be negative.
static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry)
Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
void insertBits(const KnownBits &SubBits, unsigned BitPosition)
Insert the bits from a smaller known bits starting at bitPosition.
static KnownBits avgCeilU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilU.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
KnownBits abs(bool IntMinIsPoison=false) const
Compute known bits for the absolute value.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
static KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
static KnownBits avgCeilS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilS.
This class contains a discriminated union of information about pointers in memory operands,...
bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
DAGUpdateListener *const Next
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)
