1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7//===----------------------------------------------------------------------===// 9// This implements routines for translating from LLVM IR into SelectionDAG IR. 11//===----------------------------------------------------------------------===// 78#include "llvm/IR/IntrinsicsAArch64.h" 79#include "llvm/IR/IntrinsicsAMDGPU.h" 80#include "llvm/IR/IntrinsicsWebAssembly.h" 111using namespacePatternMatch;
112using namespaceSwitchCG;
114#define DEBUG_TYPE "isel" 116/// LimitFloatPrecision - Generate low-precision inline sequences for 117/// some float libcalls (6, 8 or 12 bits). 122cl::desc(
"Insert the experimental `assertalign` node."),
127cl::desc(
"Generate low-precision inline sequences " 128"for some float libcalls"),
134cl::desc(
"Set the case probability threshold for peeling the case from a " 135"switch statement. A value greater than 100 will void this " 138// Limit the width of DAG chains. This is important in general to prevent 139// DAG-based analysis from blowing up. For example, alias analysis and 140// load clustering may not complete in reasonable time. It is difficult to 141// recognize and avoid this situation within each individual analysis, and 142// future analyses are likely to have the same behavior. Limiting DAG width is 143// the safe approach and will be especially important with global DAGs. 145// MaxParallelChains default is arbitrarily high to avoid affecting 146// optimization, but could be lowered to improve compile time. Any ld-ld-st-st 147// sequence over this should have been converted to llvm.memcpy by the 148// frontend. It is easy to induce this behavior with .ll code such as: 149// %buffer = alloca [4096 x i8] 150// %data = load [4096 x i8]* %argPtr 151// store [4096 x i8] %data, [4096 x i8]* %buffer 155constSDValue *Parts,
unsigned NumParts,
158 std::optional<CallingConv::ID>
CC);
160/// getCopyFromParts - Create a value that contains the specified legal parts 161/// combined into the value they represent. If the parts combine to a type 162/// larger than ValueVT then AssertOp can be used to specify whether the extra 163/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT 164/// (ISD::AssertSext). 167unsigned NumParts,
MVT PartVT,
EVT ValueVT,
constValue *V,
169 std::optional<CallingConv::ID>
CC = std::nullopt,
170 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
171// Let the target assemble the parts if it wants to 181assert(NumParts > 0 &&
"No parts to assemble!");
185// Assemble the value from multiple parts. 190// Assemble the power of 2 part. 192unsigned RoundBits = PartBits * RoundParts;
193EVT RoundVT = RoundBits == ValueBits ?
203 PartVT, HalfVT, V, InChain);
214if (RoundParts < NumParts) {
215// Assemble the trailing non-power-of-2 part. 216unsigned OddParts = NumParts - RoundParts;
219 OddVT, V, InChain,
CC);
221// Combine the round and odd parts. 235// FP split into multiple FP parts (for ppcf128) 236assert(ValueVT ==
EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
245// FP split into integer parts (soft fp) 247 !PartVT.
isVector() &&
"Unexpected split");
254// There is now one part, held in Val. Correct it to match ValueVT. 255// PartEVT is the type of the register class that holds the value. 256// ValueVT is the type of the inline asm operation. 259if (PartEVT == ValueVT)
263 ValueVT.
bitsLT(PartEVT)) {
264// For an FP value in an integer part, we need to truncate to the right 270// Handle types that have the same size. 274// Handle types with different sizes. 276if (ValueVT.
bitsLT(PartEVT)) {
277// For a truncate, see if we have any information to 278// indicate whether the truncated bits will always be 279// zero or sign-extension. 281 Val = DAG.
getNode(*AssertOp,
DL, PartEVT, Val,
289// FP_ROUND's are always exact here. 296 llvm::Attribute::StrictFP)) {
298 DAG.
getVTList(ValueVT, MVT::Other), InChain, Val,
308// Handle MMX to a narrower integer type by bitcasting MMX to integer and 310if (PartEVT == MVT::x86mmx && ValueVT.
isInteger() &&
311 ValueVT.
bitsLT(PartEVT)) {
325if (
constCallInst *CI = dyn_cast<CallInst>(
I))
326if (CI->isInlineAsm()) {
328 *CI, ErrMsg +
", possible invalid constraint for vector type"));
334/// getCopyFromPartsVector - Create a value that contains the specified legal 335/// parts combined into the value they represent. If the parts combine to a 336/// type larger than ValueVT then AssertOp can be used to specify whether the 337/// extra bits are known to be zero (ISD::AssertZext) or sign extended from 338/// ValueVT (ISD::AssertSext). 340constSDValue *Parts,
unsigned NumParts,
343 std::optional<CallingConv::ID> CallConv) {
345assert(NumParts > 0 &&
"No parts to assemble!");
346constbool IsABIRegCopy = CallConv.has_value();
351// Handle a multi-element vector. 355unsigned NumIntermediates;
360 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT,
361 NumIntermediates, RegisterVT);
365 NumIntermediates, RegisterVT);
368assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
369 NumParts = NumRegs;
// Silence a compiler warning. 370assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
373"Part type sizes don't match!");
375// Assemble the parts into intermediate operands. 377if (NumIntermediates == NumParts) {
378// If the register was not expanded, truncate or copy the value, 380for (
unsigned i = 0; i != NumParts; ++i)
382 V, InChain, CallConv);
383 }
elseif (NumParts > 0) {
384// If the intermediate type was expanded, build the intermediate 385// operands from the parts. 386assert(NumParts % NumIntermediates == 0 &&
387"Must expand into a divisible number of parts!");
388unsigned Factor = NumParts / NumIntermediates;
389for (
unsigned i = 0; i != NumIntermediates; ++i)
391 IntermediateVT, V, InChain, CallConv);
394// Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the 395// intermediate operands. 406DL, BuiltVectorTy, Ops);
409// There is now one part, held in Val. Correct it to match ValueVT. 412if (PartEVT == ValueVT)
416// Vector/Vector bitcast. 420// If the parts vector has more elements than the value vector, then we 421// have a vector widening case (e.g. <2 x float> -> <4 x float>). 422// Extract the elements we want. 428"Cannot narrow, it would be a lossy transformation");
434if (PartEVT == ValueVT)
439// Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>). 444// Promoted vector extract 448// Trivial bitcast if the types are the same size and the destination 449// vector type is legal. 455// Certain ABIs require that vectors are passed as integers. For vectors 456// are the same size, this is an obvious bitcast. 459 }
elseif (ValueVT.
bitsLT(PartEVT)) {
462// Drop the extra bits. 468 *DAG.
getContext(), V,
"non-trivial scalar-to-vector conversion");
472// Handle cases such as i8 -> <1 x i1> 479// It's possible a scalar floating point type gets softened to integer and 480// then promoted to a larger integer. If PartEVT is the larger integer 481// we need to truncate it and then bitcast to the FP type. 499 std::optional<CallingConv::ID> CallConv);
501/// getCopyToParts - Create a series of nodes that contain the specified value 502/// split into legal parts. If the parts contain more bits than Val, then, for 503/// integers, ExtendKind can be used to specify how to generate the extra bits. 506unsigned NumParts,
MVT PartVT,
constValue *V,
507 std::optional<CallingConv::ID> CallConv = std::nullopt,
509// Let the target split the parts if it wants to 516// Handle the vector case separately. 521unsigned OrigNumParts = NumParts;
523"Copying to an illegal type!");
530if (PartEVT == ValueVT) {
531assert(NumParts == 1 &&
"No-op copy with multiple parts!");
538// If the parts cover more bits than the value has, promote the value. 540assert(NumParts == 1 &&
"Do not know what to promote to!");
544// FP values need to be bitcast, then extended if they are being put 545// into a larger container. 553 Val = DAG.
getNode(ExtendKind,
DL, ValueVT, Val);
554if (PartVT == MVT::x86mmx)
558// Different types of the same size. 559assert(NumParts == 1 && PartEVT != ValueVT);
562// If the parts cover less bits than value has, truncate the value. 568if (PartVT == MVT::x86mmx)
572// The value may have changed - recompute ValueVT. 575"Failed to tile the value with PartVT!");
578if (PartEVT != ValueVT) {
580"scalar-to-vector conversion failed");
588// Expand the value into multiple parts. 589if (NumParts & (NumParts - 1)) {
590// The number of parts is not a power of 2. Split off and copy the tail. 592"Do not know what to expand to!");
594unsigned RoundBits = RoundParts * PartBits;
595unsigned OddParts = NumParts - RoundParts;
603// The odd parts were reversed by getCopyToParts - unreverse them. 604 std::reverse(Parts + RoundParts, Parts + NumParts);
606 NumParts = RoundParts;
611// The number of parts is a power of 2. Repeatedly bisect the value using 618for (
unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
619for (
unsigned i = 0; i < NumParts; i += StepSize) {
620unsigned ThisBits = StepSize * PartBits / 2;
623SDValue &Part1 = Parts[i+StepSize/2];
630if (ThisBits == PartBits && ThisVT != PartVT) {
638 std::reverse(Parts, Parts + OrigNumParts);
652// We only support widening vectors with equivalent element types and 653// fixed/scalable properties. If a target needs to widen a fixed-length type 654// to a scalable one, it should be possible to use INSERT_SUBVECTOR below. 655if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
659// Have a try for bf16 because some targets share its ABI with fp16. 660if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
662"Cannot widen to illegal type");
665 }
elseif (PartEVT != ValueEVT) {
669// Widening a scalable vector to another scalable vector is done by inserting 670// the vector into a larger undef one. 675// Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in 680 Ops.
append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
682// FIXME: Use CONCAT for 2x -> 4x. 686/// getCopyToPartsVector - Create a series of nodes that contain the specified 687/// value split into legal parts. 691 std::optional<CallingConv::ID> CallConv) {
695constbool IsABIRegCopy = CallConv.has_value();
699if (PartEVT == ValueVT) {
702// Bitconvert vector->vector case. 712// Promoted vector extract 718 TargetLowering::TypeWidenVector) {
719// Combination of widening and promotion. 726// Don't extract an integer from a float vector. This can happen if the 727// FP type gets softened to integer and then promoted. The promotion 728// prevents it from being picked up by the earlier bitcast case. 731// If we reach this condition and PartVT is FP, this means that 732// ValueVT is also FP and both have a different size, otherwise we 733// would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here 734// would be invalid since that would mean the smaller FP type has to 735// be extended to the larger one. 745"lossy conversion of vector to scalar type");
757// Handle a multi-element vector. 760unsigned NumIntermediates;
764 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
769 NumIntermediates, RegisterVT);
772assert(NumRegs == NumParts &&
"Part count doesn't match vector breakdown!");
773 NumParts = NumRegs;
// Silence a compiler warning. 774assert(RegisterVT == PartVT &&
"Part type doesn't match vector breakdown!");
777"Mixing scalable and fixed vectors when copying in parts");
779 std::optional<ElementCount> DestEltCnt;
789if (ValueVT == BuiltVectorTy) {
792// Bitconvert vector->vector case. 811// Split the vector into intermediate operands. 813for (
unsigned i = 0; i != NumIntermediates; ++i) {
815// This does something sensible for scalable vectors - see the 816// definition of EXTRACT_SUBVECTOR for further details. 827// Split the intermediate operands into legal parts. 828if (NumParts == NumIntermediates) {
829// If the register was not expanded, promote or copy the value, 831for (
unsigned i = 0; i != NumParts; ++i)
833 }
elseif (NumParts > 0) {
834// If the intermediate type was expanded, split each the value into 836assert(NumIntermediates != 0 &&
"division by zero");
837assert(NumParts % NumIntermediates == 0 &&
838"Must expand into a divisible number of parts!");
839unsigned Factor = NumParts / NumIntermediates;
840for (
unsigned i = 0; i != NumIntermediates; ++i)
847EVT valuevt, std::optional<CallingConv::ID>
CC)
848 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
849 RegCount(1, regs.
size()), CallConv(
CC) {}
853 std::optional<CallingConv::ID>
CC) {
867for (
unsigned i = 0; i != NumRegs; ++i)
868Regs.push_back(Reg + i);
869RegVTs.push_back(RegisterVT);
871 Reg = Reg.id() + NumRegs;
879// A Value with type {} or [0 x %t] needs no registers. 885// Assemble the legal parts into the final values. 889// Copy the legal parts from the registers. 898for (
unsigned i = 0; i != NumRegs; ++i) {
904 *Glue =
P.getValue(2);
907 Chain =
P.getValue(1);
910// If the source register was virtual and if we know something about it, 911// add an assert node. 926// The current value is a zero. 927// Explicitly express that as it would be easier for 928// optimizations to kick in. 933// FIXME: We capture more information than the dag can represent. For 934// now, just use the tightest assertzext/assertsext possible. 936EVT FromVT(MVT::Other);
940 }
elseif (NumSignBits > 1) {
947// Add an assertion node. 948assert(FromVT != MVT::Other);
954 RegisterVT, ValueVT, V, Chain,
CallConv);
969// Get the list of the values's legal parts. 970unsigned NumRegs =
Regs.size();
984 NumParts, RegisterVT, V,
CallConv, ExtendKind);
988// Copy the parts into the registers. 990for (
unsigned i = 0; i != NumRegs; ++i) {
1002if (NumRegs == 1 || Glue)
1003// If NumRegs > 1 && Glue is used then the use of the last CopyToReg is 1004// flagged to it. That is the CopyToReg nodes and the user are considered 1005// a single scheduling unit. If we create a TokenFactor and return it as 1006// chain, then the TokenFactor is both a predecessor (operand) of the 1007// user as well as a successor (the TF operands are flagged to the user). 1008// c1, f1 = CopyToReg 1009// c2, f2 = CopyToReg 1010// c3 = TokenFactor c1, c2 1013 Chain = Chains[NumRegs-1];
1019unsigned MatchingIdx,
constSDLoc &dl,
1021 std::vector<SDValue> &Ops)
const{
1026 Flag.setMatchingOp(MatchingIdx);
1028// Put the register class of the virtual registers in the flag word. That 1029// way, later passes can recompute register class constraints for inline 1030// assembly as well as normal instructions. 1031// Don't do this for tied operands that can use the regclass information 1035 Flag.setRegClass(RC->
getID());
1042// Clobbers should always have a 1:1 mapping with registers, and may 1043// reference registers that have illegal (e.g. vector) types. Hence, we 1044// shouldn't try to apply any sort of splitting logic to them. 1046"No 1:1 mapping from clobbers to regs?");
1049for (
unsignedI = 0, E =
ValueVTs.size();
I != E; ++
I) {
1054"If we clobbered the stack pointer, MFI should know about it.");
1063for (
unsigned i = 0; i != NumRegs; ++i) {
1064assert(Reg <
Regs.size() &&
"Mismatch in # registers expected");
1065unsigned TheReg =
Regs[Reg++];
1066 Ops.push_back(DAG.
getRegister(TheReg, RegisterVT));
1076unsignedRegCount = std::get<0>(CountAndVT);
1077MVT RegisterVT = std::get<1>(CountAndVT);
1101 UnusedArgNodeMap.clear();
1103 PendingExports.clear();
1104 PendingConstrainedFP.clear();
1105 PendingConstrainedFPStrict.clear();
1113 DanglingDebugInfoMap.clear();
1116// Update DAG root to include dependencies on Pending chains. 1123// Add current root to PendingChains, unless we already indirectly 1126unsigned i = 0, e = Pending.
size();
1127for (; i != e; ++i) {
1129if (Pending[i].
getNode()->getOperand(0) == Root)
1130break;
// Don't add the root if we already indirectly depend on it. 1137if (Pending.
size() == 1)
1152// Chain up all pending constrained intrinsics together with all 1153// pending loads, by simply appending them to PendingLoads and 1154// then calling getMemoryRoot(). 1156 PendingConstrainedFP.size() +
1157 PendingConstrainedFPStrict.size());
1159 PendingConstrainedFP.end());
1161 PendingConstrainedFPStrict.end());
1162 PendingConstrainedFP.clear();
1163 PendingConstrainedFPStrict.clear();
1168// We need to emit pending fpexcept.strict constrained intrinsics, 1169// so append them to the PendingExports list. 1170 PendingExports.append(PendingConstrainedFPStrict.begin(),
1171 PendingConstrainedFPStrict.end());
1172 PendingConstrainedFPStrict.clear();
1173return updateRoot(PendingExports);
1180assert(Variable &&
"Missing variable");
1182// Check if address has undef value. 1187 <<
"dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1194if (!
N.getNode() && isa<Argument>(
Address))
1195// Check unused arguments map. 1201// Parameters are handled specially. 1202auto *FINode = dyn_cast<FrameIndexSDNode>(
N.getNode());
1203if (IsParameter && FINode) {
1204// Byval parameter. We have a frame index at this point. 1206/*IsIndirect*/true,
DL, SDNodeOrder);
1207 }
elseif (isa<Argument>(
Address)) {
1208// Address is an argument, so try to emit its dbg value using 1209// virtual register info from the FuncInfo.ValueMap. 1211 FuncArgumentDbgValueKind::Declare,
N);
1215true,
DL, SDNodeOrder);
1219// If Address is an argument then try to emit its dbg value using 1220// virtual register info from the FuncInfo.ValueMap. 1222 FuncArgumentDbgValueKind::Declare,
N)) {
1224 <<
" (could not emit func-arg dbg_value)\n");
1230// Add SDDbgValue nodes for any var locs here. Do so before updating 1231// SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}. 1233// Add SDDbgValue nodes for any var locs here. Do so before updating 1234// SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}. 1235for (
auto It = FnVarLocs->locs_begin(&
I),
End = FnVarLocs->locs_end(&
I);
1237auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1239if (It->Values.isKillLocation(It->Expr)) {
1245 It->Values.hasArgList())) {
1248 FnVarLocs->getDILocalVariable(It->VariableID),
1249 It->Expr, Vals.
size() > 1, It->DL, SDNodeOrder);
1254// We must skip DbgVariableRecords if they've already been processed above as 1255// we have just emitted the debug values resulting from assignment tracking 1256// analysis, making any existing DbgVariableRecords redundant (and probably 1257// less correct). We still need to process DbgLabelRecords. This does sink 1258// DbgLabelRecords to the bottom of the group of debug records. That sholdn't 1259// be important as it does so deterministcally and ordering between 1260// DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR 1263// Is there is any debug-info attached to this instruction, in the form of 1264// DbgRecord non-instruction debug-info records. 1267assert(DLR->getLabel() &&
"Missing label");
1269DAG.
getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1274if (SkipDbgVariableRecords)
1291// A DbgVariableRecord with no locations is a kill location. 1293if (Values.
empty()) {
1299// A DbgVariableRecord with an undef or absent location is also a kill 1302 [](
Value *V) {
return !V || isa<UndefValue>(V); })) {
1310 SDNodeOrder, IsVariadic)) {
1320// Set up outgoing PHI node register values before emitting the terminator. 1321if (
I.isTerminator()) {
1322 HandlePHINodesInSuccessorBlocks(
I.getParent());
1325// Increase the SDNodeOrder if dealing with a non-debug instruction. 1326if (!isa<DbgInfoIntrinsic>(
I))
1331// Set inserted listener only if required. 1332bool NodeInserted =
false;
1333 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1334MDNode *PCSectionsMD =
I.getMetadata(LLVMContext::MD_pcsections);
1335MDNode *MMRA =
I.getMetadata(LLVMContext::MD_mmra);
1336if (PCSectionsMD || MMRA) {
1337 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1338DAG, [&](
SDNode *) { NodeInserted =
true; });
1344 !isa<GCStatepointInst>(
I))
// statepoints handle their exports internally 1348if (PCSectionsMD || MMRA) {
1349auto It = NodeMap.find(&
I);
1350if (It != NodeMap.end()) {
1355 }
elseif (NodeInserted) {
1356// This should not happen; if it does, don't let it go unnoticed so we can 1357// fix it. Relevant visit*() function is probably missing a setValue(). 1358errs() <<
"warning: loosing !pcsections and/or !mmra metadata [" 1359 <<
I.getModule()->getName() <<
"]\n";
1368void SelectionDAGBuilder::visitPHI(
constPHINode &) {
1373// Note: this doesn't use InstVisitor, because it has to work with 1374// ConstantExpr's in addition to instructions. 1377// Build the switch statement using the Instruction.def file. 1378#define HANDLE_INST(NUM, OPCODE, CLASS) \ 1379 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; 1380#include "llvm/IR/Instruction.def" 1389// For variadic dbg_values we will now insert an undef. 1390// FIXME: We can potentially recover these! 1392for (
constValue *V : Values) {
1397/*IsIndirect=*/false,
DL, Order,
1398/*IsVariadic=*/true);
1412// TODO: Dangling debug info will eventually either be resolved or produce 1413// an Undef DBG_VALUE. However in the resolution case, a gap may appear 1414// between the original dbg.value location and its resolved DBG_VALUE, 1415// which we should ideally fill with an extra Undef DBG_VALUE. 1417 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr,
DL, Order);
1422auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1423DIVariable *DanglingVariable = DDI.getVariable();
1427 << printDDI(
nullptr, DDI) <<
"\n");
1433for (
auto &DDIMI : DanglingDebugInfoMap) {
1434 DanglingDebugInfoVector &DDIV = DDIMI.second;
1436// If debug info is to be dropped, run it through final checks to see 1437// whether it can be salvaged. 1438for (
auto &DDI : DDIV)
1439if (isMatchingDbgValue(DDI))
1446// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, 1447// generate the debug data structures now that we've seen its definition. 1450auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1451if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1454 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1455for (
auto &DDI : DDIV) {
1458unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1462"Expected inlined-at fields to agree");
1465// FIXME: I doubt that it is correct to resolve a dangling DbgValue as a 1466// FuncArgumentDbgValue (it would be hoisted to the function entry, and if 1467// we couldn't resolve it directly when examining the DbgValue intrinsic 1468// in the first place we should not be more successful here). Unless we 1469// have some test case that prove this to be correct we should avoid 1470// calling EmitFuncArgumentDbgValue here. 1471if (!EmitFuncArgumentDbgValue(V, Variable, Expr,
DL,
1472 FuncArgumentDbgValueKind::Value, Val)) {
1474 << printDDI(V, DDI) <<
"\n");
1476// Increase the SDNodeOrder for the DbgValue here to make sure it is 1477// inserted after the definition of Val when emitting the instructions 1478// after ISel. An alternative could be to teach 1479// ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly. 1481 <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " 1482 << ValSDNodeOrder <<
"\n");
1483 SDV = getDbgValue(Val, Variable, Expr,
DL,
1484 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1489 <<
" in EmitFuncArgumentDbgValue\n");
1491LLVM_DEBUG(
dbgs() <<
"Dropping debug info for " << printDDI(V, DDI)
1503 DanglingDebugInfo &DDI) {
1504// TODO: For the variadic implementation, instead of only checking the fail 1505// state of `handleDebugValue`, we need know specifically which values were 1506// invalid, so that we attempt to salvage only those values when processing 1508constValue *OrigV = V;
1512unsigned SDOrder = DDI.getSDNodeOrder();
1514// Currently we consider only dbg.value intrinsics -- we tell the salvager 1515// that DW_OP_stack_value is desired. 1516bool StackValue =
true;
1518// Can this Value can be encoded without any further work? 1522// Attempt to salvage back through as many instructions as possible. Bail if 1523// a non-instruction is seen, such as a constant expression or global 1524// variable. FIXME: Further work could recover those too. 1525while (isa<Instruction>(V)) {
1526constInstruction &VAsInst = *cast<const Instruction>(V);
1527// Temporary "0", awaiting real implementation. 1533// If we cannot salvage any further, and haven't yet found a suitable debug 1534// expression, bail out. 1538// TODO: If AdditionalValues isn't empty, then the salvage can only be 1539// represented with a DBG_VALUE_LIST, so we give up. When we have support 1540// here for variadic dbg_values, remove that condition. 1541if (!AdditionalValues.
empty())
1544// New value and expr now represent this debuginfo. 1547// Some kind of simplification occurred: check whether the operand of the 1548// salvaged debug expression can be encoded in this DAG. 1551dbgs() <<
"Salvaged debug location info for:\n " << *Var <<
"\n" 1552 << *OrigV <<
"\nBy stripping back to:\n " << *V <<
"\n");
1557// This was the final opportunity to salvage this debug information, and it 1558// couldn't be done. Place an undef DBG_VALUE at this location to terminate 1559// any earlier variable location. 1560assert(OrigV &&
"V shouldn't be null");
1565 << printDDI(OrigV, DDI) <<
"\n");
1576/*IsVariadic*/false);
1582unsigned Order,
bool IsVariadic) {
1586// Filter EntryValue locations out early. 1587if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1592for (
constValue *V : Values) {
1594if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1595 isa<ConstantPointerNull>(V)) {
1600// Look through IntToPtr constants. 1601if (
auto *CE = dyn_cast<ConstantExpr>(V))
1602if (CE->getOpcode() == Instruction::IntToPtr) {
1607// If the Value is a frame index, we can create a FrameIndex debug value 1608// without relying on the DAG at all. 1609if (
constAllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1617// Do not use getValue() in here; we don't want to generate code at 1618// this point if it hasn't been done yet. 1620if (!
N.getNode() && isa<Argument>(V))
// Check unused arguments map. 1621N = UnusedArgNodeMap[V];
1624// Only emit func arg dbg value for non-variadic dbg.values for now. 1626 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1627 FuncArgumentDbgValueKind::Value,
N))
1629if (
auto *FISDN = dyn_cast<FrameIndexSDNode>(
N.getNode())) {
1630// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can 1631// describe stack slot locations. 1633// Consider "int x = 0; int *px = &x;". There are two kinds of 1634// interesting debug values here after optimization: 1636// dbg.value(i32* %px, !"int *px", !DIExpression()), and 1637// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) 1639// Both describe the direct values of their associated variables. 1650// Special rules apply for the first dbg.values of parameter variables in a 1651// function. Identify them by the fact they reference Argument Values, that 1652// they're parameters, and they are parameters of the current function. We 1653// need to let them dangle until they get an SDNode. 1659// The value is not used in this block yet (or it would have an SDNode). 1660// We still want the value to appear for the user if possible -- if it has 1661// an associated VReg, we can refer to that instead. 1664unsigned Reg = VMI->second;
1665// If this is a PHI node, it may be split up into several MI PHI nodes 1666// (in FunctionLoweringInfo::set). 1668 V->getType(), std::nullopt);
1670// FIXME: We could potentially support variadic dbg_values here. 1674unsigned BitsToDescribe = 0;
1676 BitsToDescribe = *VarSize;
1678 BitsToDescribe = Fragment->SizeInBits;
1680// Bail out if all bits are described already. 1681if (
Offset >= BitsToDescribe)
1683// TODO: handle scalable vectors. 1684unsigned RegisterSize = RegAndSize.second;
1685unsigned FragmentSize = (
Offset + RegisterSize > BitsToDescribe)
1689 Expr,
Offset, FragmentSize);
1693 Var, *FragmentExpr, RegAndSize.first,
false, DbgLoc, Order);
1699// We can use simple vreg locations for variadic dbg_values as well. 1703// We failed to create a SDDbgOperand for V. 1707// We have created a SDDbgOperand for each Value in Values. 1711/*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1717// Try to fixup any remaining dangling debug info -- and drop it if we can't. 1718for (
auto &Pair : DanglingDebugInfoMap)
1719for (
auto &DDI : Pair.second)
1724/// getCopyFromRegs - If there was virtual register allocated for the value V 1725/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. 1735 std::nullopt);
// This is not an ABI copy. 1745/// getValue - Return an SDValue for the given Value. 1747// If we already have an SDValue for this value, use it. It's important 1748// to do this first, so that we don't create a CopyFromReg if we already 1749// have a regular SDValue. 1751if (
N.getNode())
returnN;
1753// If there's a virtual register allocated and initialized for this 1758// Otherwise create a new SDValue and remember it. 1765/// getNonRegisterValue - Return an SDValue for the given Value, but 1766/// don't look in FuncInfo.ValueMap for a virtual register. 1768// If we already have an SDValue for this value, use it. 1772// Remove the debug location from the node as the node is about to be used 1773// in a location which may differ from the original debug location. This 1774// is relevant to Constant and ConstantFP nodes because they can appear 1775// as constant expressions inside PHI nodes. 1781// Otherwise create a new SDValue and remember it. 1788/// getValueImpl - Helper function for getValue and getNonRegisterValue. 1789/// Create an SDValue for the given value. 1793if (
constConstant *
C = dyn_cast<Constant>(V)) {
1805getValue(CPA->getAddrDiscriminator()),
1809if (isa<ConstantPointerNull>(
C)) {
1810unsigned AS = V->getType()->getPointerAddressSpace();
1818if (
constConstantFP *CFP = dyn_cast<ConstantFP>(
C))
1821if (isa<UndefValue>(
C) && !V->getType()->isAggregateType())
1825visit(CE->getOpcode(), *CE);
1831if (isa<ConstantStruct>(
C) || isa<ConstantArray>(
C)) {
1833for (
constUse &U :
C->operands()) {
1835// If the operand is an empty aggregate, there are no values. 1837// Add each leaf value from the operand to the Constants list 1838// to form a flattened list of all the values. 1839for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
1840 Constants.push_back(
SDValue(Val, i));
1847 dyn_cast<ConstantDataSequential>(
C)) {
1849for (
unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1851// Add each leaf value from the operand to the Constants list 1852// to form a flattened list of all the values. 1853for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
1857if (isa<ArrayType>(CDS->getType()))
1862if (
C->getType()->isStructTy() ||
C->getType()->isArrayTy()) {
1863assert((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(
C)) &&
1864"Unknown struct or array constant!");
1868unsigned NumElts = ValueVTs.
size();
1870returnSDValue();
// empty struct 1872for (
unsigned i = 0; i != NumElts; ++i) {
1873EVT EltVT = ValueVTs[i];
1874if (isa<UndefValue>(
C))
1888if (
constauto *Equiv = dyn_cast<DSOLocalEquivalent>(
C))
1889returngetValue(Equiv->getGlobalValue());
1891if (
constauto *
NC = dyn_cast<NoCFIValue>(
C))
1894if (VT == MVT::aarch64svcount) {
1895assert(
C->isNullValue() &&
"Can only zero this target type!");
1901assert(
C->isNullValue() &&
"Can only zero this target type!");
1912VectorType *VecTy = cast<VectorType>(V->getType());
1914// Now that we know the number and type of the elements, get that number of 1915// elements into the Ops array based on what kind of constant it is. 1918unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1919for (
unsigned i = 0; i != NumElements; ++i)
1925if (isa<ConstantAggregateZero>(
C)) {
1941// If this is a static alloca, generate it as the frameindex instead of 1943if (
constAllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1951// If this is an instruction which fast-isel has deferred, select it now. 1952if (
constInstruction *Inst = dyn_cast<Instruction>(V)) {
1956 Inst->getType(), std::nullopt);
1964if (
constauto *BB = dyn_cast<BasicBlock>(V))
1970void SelectionDAGBuilder::visitCatchPad(
constCatchPadInst &
I) {
1978// In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues. 1979if (IsMSVCCXX || IsCoreCLR)
1984// Update machine-CFG edge. 1993// If this is not a fall-through branch or optimizations are switched off, 2002// Figure out the funclet membership for the catchret's successor. 2003// This will be used by the FuncletLayout pass to determine how to order the 2005// A 'catchret' returns to the outer scope's color. 2006Value *ParentPad =
I.getCatchSwitchParentPad();
2008if (isa<ConstantTokenNone>(ParentPad))
2011 SuccessorColor = cast<Instruction>(ParentPad)->
getParent();
2012assert(SuccessorColor &&
"No parent funclet for catchret!");
2014assert(SuccessorColorMBB &&
"No MBB for SuccessorColor!");
2016// Create the terminator node. 2023void SelectionDAGBuilder::visitCleanupPad(
constCleanupPadInst &CPI) {
2024// Don't emit any special code for the cleanuppad instruction. It just marks 2025// the start of an EH scope/funclet. 2034// In wasm EH, even though a catchpad may not catch an exception if a tag does 2035// not match, it is OK to add only the first unwind destination catchpad to the 2036// successors, because there will be at least one invoke instruction within the 2037// catch scope that points to the next unwind destination, if one exists, so 2038// CFGSort cannot mess up with BB sorting order. 2039// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic 2040// call within them, and catchpads only consisting of 'catch (...)' have a 2041// '__cxa_end_catch' call within them, both of which generate invokes in case 2042// the next unwind destination exists, i.e., the next unwind destination is not 2045// Having at most one EH pad successor is also simpler and helps later 2050// invoke void @foo to ... unwind label %catch.dispatch 2052// %0 = catchswitch within ... [label %catch.start] unwind label %next 2055// ... in this BB or some other child BB dominated by this BB there will be an 2056// invoke that points to 'next' BB as an unwind destination 2058// next: ; We don't need to add this to 'current' BB's successor 2067if (isa<CleanupPadInst>(Pad)) {
2068// Stop on cleanup pads. 2069 UnwindDests.emplace_back(FuncInfo.
getMBB(EHPadBB), Prob);
2070 UnwindDests.back().first->setIsEHScopeEntry();
2072 }
elseif (
constauto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2073// Add the catchpad handlers to the possible destinations. We don't 2074// continue to the unwind destination of the catchswitch for wasm. 2075for (
constBasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2076 UnwindDests.emplace_back(FuncInfo.
getMBB(CatchPadBB), Prob);
2077 UnwindDests.back().first->setIsEHScopeEntry();
2086/// When an invoke or a cleanupret unwinds to the next EH pad, there are 2087/// many places it could ultimately go. In the IR, we have a single unwind 2088/// destination, but in the machine CFG, we enumerate all the possible blocks. 2089/// This function skips over imaginary basic blocks that hold catchswitch 2090/// instructions, and finds all the "real" machine 2091/// basic block destinations. As those destinations may not be successors of 2092/// EHPadBB, here we also calculate the edge probability to those destinations. 2093/// The passed-in Prob is the edge probability to EHPadBB. 2108assert(UnwindDests.size() <= 1 &&
2109"There should be at most one unwind destination for wasm");
2116if (isa<LandingPadInst>(Pad)) {
2117// Stop on landingpads. They are not funclets. 2118 UnwindDests.emplace_back(FuncInfo.
getMBB(EHPadBB), Prob);
2120 }
elseif (isa<CleanupPadInst>(Pad)) {
2121// Stop on cleanup pads. Cleanups are always funclet entries for all known 2123 UnwindDests.emplace_back(FuncInfo.
getMBB(EHPadBB), Prob);
2124 UnwindDests.
back().first->setIsEHScopeEntry();
2125 UnwindDests.back().first->setIsEHFuncletEntry();
2127 }
elseif (
constauto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2128// Add the catchpad handlers to the possible destinations. 2129for (
constBasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2130 UnwindDests.emplace_back(FuncInfo.
getMBB(CatchPadBB), Prob);
2131// For MSVC++ and the CLR, catchblocks are funclets and need prologues. 2132if (IsMSVCCXX || IsCoreCLR)
2133 UnwindDests.back().first->setIsEHFuncletEntry();
2135 UnwindDests.back().first->setIsEHScopeEntry();
2137 NewEHPadBB = CatchSwitch->getUnwindDest();
2143if (BPI && NewEHPadBB)
2145 EHPadBB = NewEHPadBB;
2150// Update successor info. 2152auto UnwindDest =
I.getUnwindDest();
2159for (
auto &UnwindDest : UnwindDests) {
2160 UnwindDest.first->setIsEHPad();
2161 addSuccessorWithProb(
FuncInfo.
MBB, UnwindDest.first, UnwindDest.second);
2165// Create the terminator node. 2173void SelectionDAGBuilder::visitCatchSwitch(
constCatchSwitchInst &CSI) {
2177void SelectionDAGBuilder::visitRet(
constReturnInst &
I) {
2184// Calls to @llvm.experimental.deoptimize don't generate a return value, so 2187// %val = call <ty> @llvm.experimental.deoptimize() 2191if (
I.getParent()->getTerminatingDeoptimizeCall()) {
2199// Emit a store of the return value through the virtual register. 2200// Leave Outs empty so that LowerReturn won't try to load return 2201// registers the usual way. 2211unsigned NumValues = ValueVTs.
size();
2214Align BaseAlign =
DL.getPrefTypeAlign(
I.getOperand(0)->getType());
2215for (
unsigned i = 0; i != NumValues; ++i) {
2216// An aggregate return value cannot wrap around the address space, so 2217// offsets to its parts don't wrap either. 2222if (MemVTs[i] != ValueVTs[i])
2226// FIXME: better loc info would be nice. 2232 MVT::Other, Chains);
2233 }
elseif (
I.getNumOperands() != 0) {
2236unsigned NumValues = ValueVTs.
size();
2243I.getOperand(0)->getType(),
F->getCallingConv(),
2244/*IsVarArg*/false,
DL);
2247if (
F->getAttributes().hasRetAttr(Attribute::SExt))
2249elseif (
F->getAttributes().hasRetAttr(Attribute::ZExt))
2253bool RetInReg =
F->getAttributes().hasRetAttr(Attribute::InReg);
2255for (
unsigned j = 0;
j != NumValues; ++
j) {
2256EVT VT = ValueVTs[
j];
2268 &Parts[0], NumParts, PartVT, &
I,
CC, ExtendKind);
2270// 'inreg' on function refers to return value 2275if (
I.getOperand(0)->getType()->isPointerTy()) {
2277Flags.setPointerAddrSpace(
2278 cast<PointerType>(
I.getOperand(0)->getType())->getAddressSpace());
2282Flags.setInConsecutiveRegs();
2283if (j == NumValues - 1)
2284Flags.setInConsecutiveRegsLast();
2287// Propagate extension type if any 2292elseif (
F->getAttributes().hasRetAttr(Attribute::NoExt))
2295for (
unsigned i = 0; i < NumParts; ++i) {
2298 VT,
/*isfixed=*/true, 0, 0));
2305// Push in swifterror virtual register as the last element of Outs. This makes 2306// sure swifterror virtual register will be returned in the swifterror 2307// physical register. 2310F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2313Flags.setSwiftError();
2316/*isfixed=*/true,
/*origidx=*/1,
/*partOffs=*/0));
2317// Create SDNode for the swifterror virtual register. 2330// Verify that the target's LowerReturn behaved as expected. 2332"LowerReturn didn't return a valid chain!");
2334// Update the DAG with the new chain value resulting from return lowering. 2338/// CopyToExportRegsIfNeeded - If the given value has virtual registers 2339/// created for it, emit nodes to copy the value into the virtual 2343if (V->getType()->isEmptyTy())
2348assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2349"Unused value assigned virtual registers!");
2354/// ExportFromCurrentBlock - If this condition isn't known to be exported from 2355/// the current basic block, add it to ValueMap now so that we'll get a 2358// No need to export constants. 2359if (!isa<Instruction>(V) && !isa<Argument>(V))
return;
2370// The operands of the setcc have to be in this block. We don't know 2371// how to export them from some other block. 2372if (
constInstruction *VI = dyn_cast<Instruction>(V)) {
2373// Can export from current BB. 2374if (VI->getParent() == FromBB)
2377// Is already exported, noop. 2381// If this is an argument, we can export it if the BB is the entry block or 2382// if it is already exported. 2383if (isa<Argument>(V)) {
2387// Otherwise, can only export this if it is already exported. 2391// Otherwise, constants can always be exported. 2395/// Return branch probability calculated by BranchProbabilityInfo for IR blocks. 2400constBasicBlock *SrcBB = Src->getBasicBlock();
2401constBasicBlock *DstBB = Dst->getBasicBlock();
2403// If BPI is not available, set the default probability as 1 / N, where N is 2404// the number of successors. 2405auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
2415 Src->addSuccessorWithoutProb(Dst);
2418 Prob = getEdgeProbability(Src, Dst);
2419 Src->addSuccessor(Dst, Prob);
2425returnI->getParent() == BB;
2429/// EmitBranchForMergedCondition - Helper method for FindMergedConditions. 2430/// This function emits a branch and is used at the leaves of an OR or an 2431/// AND operator tree. 2443// If the leaf of the tree is a comparison, merge the condition into 2446// The operands of the cmp have to be in this block. We don't know 2447// how to export them from some other block. If this is the first block 2448// of the sequence, no exporting is needed. 2449if (CurBB == SwitchBB ||
2455 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2460 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2466CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1),
nullptr,
2468SL->SwitchCases.push_back(CB);
2473// Create a CaseBlock record representing this branch. 2477SL->SwitchCases.push_back(CB);
2480// Collect dependencies on V recursively. This is used for the cost analysis in 2481// `shouldKeepJumpConditionsTogether`. 2486// Return false if we have an incomplete count. 2490auto *
I = dyn_cast<Instruction>(V);
2494if (Necessary !=
nullptr) {
2495// This instruction is necessary for the other side of the condition so 2497if (Necessary->contains(
I))
2501// Already added this dep. 2505for (
unsigned OpIdx = 0, E =
I->getNumOperands(); OpIdx < E; ++OpIdx)
2516if (
I.getNumSuccessors() != 2)
2519if (!
I.isConditional())
2532// See if we are either likely to get an early out or compute both lhs/rhs 2537 std::optional<bool> Likely;
2540elseif (BPI->
isEdgeHot(
I.getParent(), IfFalse))
2544if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2545// Its likely we will have to compute both lhs and rhs of condition 2550// Its likely we will get an early out. 2559// Collect "all" instructions that lhs condition is dependent on. 2560// Use map for stable iteration (to avoid non-determanism of iteration of 2561// SmallPtrSet). The `bool` value is just a dummy. 2564// Collect "all" instructions that rhs condition is dependent on AND are 2565// dependencies of lhs. This gives us an estimate on which instructions we 2566// stand to save by splitting the condition. 2569// Add the compare instruction itself unless its a dependency on the LHS. 2570if (
constauto *RhsI = dyn_cast<Instruction>(Rhs))
2579// See if this instruction will need to computed independently of whether RHS 2581Value *BrCond =
I.getCondition();
2582auto ShouldCountInsn = [&RhsDeps, &BrCond](
constInstruction *Ins) {
2583for (
constauto *U : Ins->users()) {
2584// If user is independent of RHS calculation we don't need to count it. 2585if (
auto *UIns = dyn_cast<Instruction>(U))
2586if (UIns != BrCond && !RhsDeps.
contains(UIns))
2592// Prune instructions from RHS Deps that are dependencies of unrelated 2593// instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly 2594// arbitrary and just meant to cap the how much time we spend in the pruning 2595// loop. Its highly unlikely to come into affect. 2597// Stop after a certain point. No incorrectness from including too many 2599for (
unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2601for (
constauto &InsPair : RhsDeps) {
2602if (!ShouldCountInsn(InsPair.first)) {
2603 ToDrop = InsPair.first;
2607if (ToDrop ==
nullptr)
2609 RhsDeps.erase(ToDrop);
2612for (
constauto &InsPair : RhsDeps) {
2613// Finally accumulate latency that we can only attribute to computing the 2614// RHS condition. Use latency because we are essentially trying to calculate 2615// the cost of the dependency chain. 2616// Possible TODO: We could try to estimate ILP and make this more precise. 2620if (CostOfIncluding > CostThresh)
2635// Skip over not part of the tree and remember to invert op and operands at 2646constValue *BOpOp0, *BOpOp1;
2647// Compute the effective opcode for Cond, taking into account whether it needs 2648// to be inverted, e.g. 2649// and (not (or A, B)), C 2651// and (and (not A, not B), C) 2660if (BOpc == Instruction::And)
2661 BOpc = Instruction::Or;
2662elseif (BOpc == Instruction::Or)
2663 BOpc = Instruction::And;
2667// If this node is not part of the or/and tree, emit it as a branch. 2668// Note that all nodes in the tree should have same opcode. 2669bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->
hasOneUse();
2674 TProb, FProb, InvertCond);
2678// Create TmpBB after CurBB. 2684if (Opc == Instruction::Or) {
2694// We have flexibility in setting Prob for BB1 and Prob for TmpBB. 2695// The requirement is that 2696// TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) 2697// = TrueProb for original BB. 2698// Assuming the original probabilities are A and B, one choice is to set 2699// BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to 2700// A/(1+B) and 2B/(1+B). This choice assumes that 2701// TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. 2702// Another choice is to assume TrueProb for BB1 equals to TrueProb for 2703// TmpBB, but the math is more complicated. 2705auto NewTrueProb = TProb / 2;
2706auto NewFalseProb = TProb / 2 + FProb;
2707// Emit the LHS condition. 2709 NewFalseProb, InvertCond);
2711// Normalize A/2 and B to get A/(1+B) and 2B/(1+B). 2714// Emit the RHS condition into TmpBB. 2716 Probs[1], InvertCond);
2718assert(Opc == Instruction::And &&
"Unknown merge op!");
2727// This requires creation of TmpBB after CurBB. 2729// We have flexibility in setting Prob for BB1 and Prob for TmpBB. 2730// The requirement is that 2731// FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) 2732// = FalseProb for original BB. 2733// Assuming the original probabilities are A and B, one choice is to set 2734// BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to 2735// 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 == 2736// TrueProb for BB1 * FalseProb for TmpBB. 2738auto NewTrueProb = TProb + FProb / 2;
2739auto NewFalseProb = FProb / 2;
2740// Emit the LHS condition. 2742 NewFalseProb, InvertCond);
2744// Normalize A and B/2 to get 2A/(1+A) and B/(1+A). 2747// Emit the RHS condition into TmpBB. 2749 Probs[1], InvertCond);
2753/// If the set of cases should be emitted as a series of branches, return true. 2754/// If we should emit this as a bunch of and/or'd together conditions, return 2758if (Cases.size() != 2)
returntrue;
2760// If this is two comparisons of the same values or'd or and'd together, they 2761// will get folded into a single comparison, so don't emit two blocks. 2762if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2763 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2764 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2765 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2769// Handle: (X != null) | (Y != null) --> (X|Y) != 0 2770// Handle: (X == null) & (Y == null) --> (X|Y) == 0 2771if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2772 Cases[0].
CC == Cases[1].
CC &&
2773 isa<Constant>(Cases[0].CmpRHS) &&
2774 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2775if (Cases[0].
CC ==
ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2777if (Cases[0].
CC ==
ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2784void SelectionDAGBuilder::visitBr(
constBranchInst &
I) {
2787// Update machine-CFG edges. 2790if (
I.isUnconditional()) {
2791// Update machine-CFG edges. 2794// If this is not a fall-through branch or optimizations are switched off, 2796if (Succ0MBB != NextBlock(BrMBB) ||
2807// If this condition is one of the special cases we handle, do special stuff 2809constValue *CondVal =
I.getCondition();
2812// If this is a series of conditions that are or'd or and'd together, emit 2813// this as a sequence of branches instead of setcc's with and/or operations. 2814// As long as jumps are not expensive (exceptions for multi-use logic ops, 2815// unpredictable branches, and vector extracts because those jumps are likely 2816// expensive for any target), this should improve performance. 2817// For example, instead of something like: 2829bool IsUnpredictable =
I.hasMetadata(LLVMContext::MD_unpredictable);
2830constInstruction *BOp = dyn_cast<Instruction>(CondVal);
2834constValue *BOp0, *BOp1;
2837 Opcode = Instruction::And;
2839 Opcode = Instruction::Or;
2847 Opcode, BOp0, BOp1))) {
2849 getEdgeProbability(BrMBB, Succ0MBB),
2850 getEdgeProbability(BrMBB, Succ1MBB),
2851/*InvertCond=*/false);
2852// If the compares in later blocks need to use values not currently 2853// exported from this block, export them now. This block should always 2854// be the first entry. 2855assert(
SL->SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!");
2857// Allow some cases to be rejected. 2859for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i) {
2864// Emit the branch for this block. 2866SL->SwitchCases.erase(
SL->SwitchCases.begin());
2870// Okay, we decided not to do this, remove any inserted MBB's and clear 2872for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i)
2875SL->SwitchCases.clear();
2879// Create a CaseBlock record representing this branch. 2885// Use visitSwitchCase to actually insert the fast branch sequence for this 2890/// visitSwitchCase - Emits the necessary code to represent a single node in 2891/// the binary search tree resulting from lowering a switch instruction. 2899// Branch or fall through to TrueBB. 2902if (CB.
TrueBB != NextBlock(SwitchBB)) {
2912// Build the setcc now. 2914// Fold "(X == true)" to X and "(X == false)" to !X to 2915// handle common cases produced by branch lowering. 2926// If a pointer's DAG type is larger than its memory type then the DAG 2927// values are zero-extended. This breaks signed comparisons so truncate 2928// back to the underlying type before doing the compare. 2944if (cast<ConstantInt>(CB.
CmpLHS)->isMinValue(
true)) {
2955// Update successor info 2957// TrueBB and FalseBB are always different unless the incoming IR is 2958// degenerate. This only happens when running llc on weird IR. 2963// If the lhs block is the next block, invert the condition so that we can 2964// fall through to the lhs instead of the rhs block. 2965if (CB.
TrueBB == NextBlock(SwitchBB)) {
2978// Insert the false branch. Do this even if it's a fall through branch, 2979// this makes it easier to do DAG optimizations which require inverting 2980// the branch condition. 2987/// visitJumpTable - Emit JumpTable node in the current MBB 2989// Emit the code for the jump table 2990assert(JT.SL &&
"Should set SDLoc for SelectionDAG!");
2991assert(JT.Reg &&
"Should lower JT Header first!");
2996 Index.getValue(1), Table, Index);
3000/// visitJumpTableHeader - This function emits necessary code to produce index 3001/// in the JumpTable from switch case. 3005assert(JT.SL &&
"Should set SDLoc for SelectionDAG!");
3006constSDLoc &dl = *JT.SL;
3008// Subtract the lowest switch case value from the value being switched on. 3014// The SDNode we just created, which holds the value being switched on minus 3015// the smallest case value, needs to be copied to a virtual register so it 3016// can be used as an index into the jump table in a subsequent basic block. 3017// This value may be smaller or larger than the target's pointer type, and 3018// therefore require extension or truncating. 3027 JT.Reg = JumpTableReg;
3030// Emit the range check for the jump table, and branch to the default block 3031// for the switch statement if the value being switched on exceeds the 3032// largest case in the switch. 3039 MVT::Other, CopyTo, CMP,
3042// Avoid emitting unnecessary branches to the next block. 3043if (JT.MBB != NextBlock(SwitchBB))
3049// Avoid emitting unnecessary branches to the next block. 3050if (JT.MBB != NextBlock(SwitchBB))
3058/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global 3059/// variable if there exists one. 3078if (PtrTy != PtrMemTy)
3083/// Codegen a new tail for a stack protector check ParentMBB which has had its 3084/// tail spliced into a stack protector check success bb. 3086/// For a high level explanation of how this fits into the stack protector 3087/// generation see the comment on the declaration of class 3088/// StackProtectorDescriptor. 3092// First create the loads to the guard/stack slot for the comparison. 3107// Generate code to load the content of the guard slot. 3116// Retrieve guard check function, nullptr if instrumentation is inlined. 3118// The target provides a guard check function to validate the guard value. 3119// Generate a call to that function with the content of the guard slot as 3126 Entry.Node = GuardVal;
3128if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3129 Entry.IsInReg =
true;
3130 Args.push_back(Entry);
3136getValue(GuardCheckFn), std::move(Args));
3138 std::pair<SDValue, SDValue> Result = TLI.
LowerCallTo(CLI);
3143// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD. 3144// Otherwise, emit a volatile load to retrieve the stack guard value. 3152 Guard =
DAG.
getLoad(PtrMemTy, dl, Chain, GuardPtr,
3157// Perform the comparison via a getsetcc. 3163// If the guard/stackslot do not equal, branch to failure MBB. 3167// Otherwise branch to success MBB. 3175/// Codegen the failure basic block for a stack protector check. 3177/// A failure stack protector machine basic block consists simply of a call to 3178/// __stack_chk_fail(). 3180/// For a high level explanation of how this fits into the stack protector 3181/// generation see the comment on the declaration of class 3182/// StackProtectorDescriptor. 3192// Emit a trap instruction if we are required to do so. 3200/// visitBitTestHeader - This function emits necessary code to produce value 3201/// suitable for "bit tests" 3206// Subtract the minimum value. 3212// Determine the type of the test operands. 3214bool UsePtrType =
false;
3218for (
unsigned i = 0, e =
B.Cases.size(); i != e; ++i)
3220// Switch table case range are encoded into series of masks. 3221// Just use pointer type, it's guaranteed to fit. 3238if (!
B.FallthroughUnreachable)
3239 addSuccessorWithProb(SwitchBB,
B.Default,
B.DefaultProb);
3240 addSuccessorWithProb(SwitchBB,
MBB,
B.Prob);
3244if (!
B.FallthroughUnreachable) {
3245// Conditional branch to the default block. 3256// Avoid emitting unnecessary branches to the next block. 3257if (
MBB != NextBlock(SwitchBB))
3263/// visitBitTestCase - this function produces one "bit test" 3276// Testing for a single bit; just compare the shift count with what it 3277// would need to be to shift a 1 bit in that position. 3282 }
elseif (PopCount == BB.
Range) {
3283// There is only one zero bit in the range, test for it directly. 3288// Make desired shift 3292// Emit bit tests and jumps 3300// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb. 3301 addSuccessorWithProb(SwitchBB,
B.TargetBB,
B.ExtraProb);
3302// The branch probability from SwitchBB to NextMBB is BranchProbToNext. 3303 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3304// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is 3305// one as they are relative probabilities (and thus work more like weights), 3306// and hence we need to normalize them to let the sum of them become one. 3313// Avoid emitting unnecessary branches to the next block. 3314if (NextMBB != NextBlock(SwitchBB))
3321void SelectionDAGBuilder::visitInvoke(
constInvokeInst &
I) {
3324// Retrieve successors. Look through artificial IR level blocks like 3325// catchswitch for successors. 3330// Deopt and ptrauth bundles are lowered in helper functions, and we don't 3331// have to do anything here to lower funclet bundles. 3332assert(!
I.hasOperandBundlesOtherThan(
3333 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3334 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3335 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3336 LLVMContext::OB_clang_arc_attachedcall}) &&
3337"Cannot lower invokes with arbitrary operand bundles yet!");
3339constValue *Callee(
I.getCalledOperand());
3340constFunction *Fn = dyn_cast<Function>(Callee);
3341if (isa<InlineAsm>(Callee))
3342 visitInlineAsm(
I, EHPadBB);
3347case Intrinsic::donothing:
3348// Ignore invokes to @llvm.donothing: jump directly to the next BB. 3349case Intrinsic::seh_try_begin:
3350case Intrinsic::seh_scope_begin:
3351case Intrinsic::seh_try_end:
3352case Intrinsic::seh_scope_end:
3354// a block referenced by EH table 3355// so dtor-funclet not removed by opts 3358case Intrinsic::experimental_patchpoint_void:
3359case Intrinsic::experimental_patchpoint:
3360 visitPatchpoint(
I, EHPadBB);
3362case Intrinsic::experimental_gc_statepoint:
3365case Intrinsic::wasm_rethrow: {
3366// This is usually done in visitTargetIntrinsic, but this intrinsic is 3367// special because it can be invoked, so we manually lower it to a DAG 3380 }
elseif (
I.hasDeoptState()) {
3381// Currently we do not lower any intrinsic calls with deopt operand bundles. 3382// Eventually we will support lowering the @llvm.experimental.deoptimize 3383// intrinsic, and right now there are no plans to support other intrinsics 3392// If the value of the invoke is used outside of its defining block, make it 3393// available as a virtual register. 3394// We already took care of the exported value for the statepoint instruction 3395// during call to the LowerStatepoint. 3396if (!isa<GCStatepointInst>(
I)) {
3407// Update successor info. 3408 addSuccessorWithProb(InvokeMBB, Return);
3409for (
auto &UnwindDest : UnwindDests) {
3410 UnwindDest.first->setIsEHPad();
3411 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3415// Drop into normal successor. 3420void SelectionDAGBuilder::visitCallBr(
constCallBrInst &
I) {
3423// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't 3424// have to do anything here to lower funclet bundles. 3425assert(!
I.hasOperandBundlesOtherThan(
3426 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3427"Cannot lower callbrs with arbitrary operand bundles yet!");
3429assert(
I.isInlineAsm() &&
"Only know how to handle inlineasm callbr");
3433// Retrieve successors. 3435 Dests.
insert(
I.getDefaultDest());
3438// Update successor info. 3440for (
unsigned i = 0, e =
I.getNumIndirectDests(); i < e; ++i) {
3443Target->setIsInlineAsmBrIndirectTarget();
3444Target->setMachineBlockAddressTaken();
3445Target->setLabelMustBeEmitted();
3446// Don't add duplicate machine successors. 3447if (Dests.
insert(Dest).second)
3452// Drop into default successor. 3458void SelectionDAGBuilder::visitResume(
constResumeInst &RI) {
3459llvm_unreachable(
"SelectionDAGBuilder shouldn't visit resume instructions!");
3462void SelectionDAGBuilder::visitLandingPad(
constLandingPadInst &LP) {
3464"Call to landingpad not in landing pad!");
3466// If there aren't registers to copy the values into (e.g., during SjLj 3467// exceptions), then don't bother to create these DAG nodes. 3474// If landingpad's return type is token type, we don't create DAG nodes 3475// for its exception pointer and selector value. The extraction of exception 3476// pointer or selector value from token type landingpads is not currently 3484assert(ValueVTs.
size() == 2 &&
"Only two-valued landingpads are supported");
3486// Get the two live-in registers as SDValues. The physregs have already been 3487// copied into virtual registers. 3514if (JTB.first.HeaderBB ==
First)
3515 JTB.first.HeaderBB =
Last;
3517// Update BitTestCases. 3526// Update machine-CFG edges with unique successors. 3528for (
unsigned i = 0, e =
I.getNumSuccessors(); i != e; ++i) {
3530bool Inserted =
Done.insert(BB).second;
3535 addSuccessorWithProb(IndirectBrMBB, Succ);
3548// We may be able to ignore unreachable behind a noreturn call. 3549if (
constCallInst *Call = dyn_cast_or_null<CallInst>(
I.getPrevNode());
3550 Call &&
Call->doesNotReturn()) {
3553// Do not emit an additional trap instruction. 3554if (
Call->isNonContinuableTrap())
3561void SelectionDAGBuilder::visitUnary(
constUser &
I,
unsigned Opcode) {
3563if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3564Flags.copyFMF(*FPOp);
3572void SelectionDAGBuilder::visitBinary(
constUser &
I,
unsigned Opcode) {
3574if (
auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&
I)) {
3575Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3576Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3578if (
auto *ExactOp = dyn_cast<PossiblyExactOperator>(&
I))
3579Flags.setExact(ExactOp->isExact());
3580if (
auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&
I))
3581Flags.setDisjoint(DisjointOp->isDisjoint());
3582if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3583Flags.copyFMF(*FPOp);
3592void SelectionDAGBuilder::visitShift(
constUser &
I,
unsigned Opcode) {
3599// Coerce the shift amount to the right type if we can. This exposes the 3600// truncate or zext to optimization early. 3601if (!
I.getType()->isVectorTy() && Op2.
getValueType() != ShiftTy) {
3603"Unexpected shift type");
3614 dyn_cast<const OverflowingBinaryOperator>(&
I)) {
3615 nuw = OFBinOp->hasNoUnsignedWrap();
3616 nsw = OFBinOp->hasNoSignedWrap();
3619 dyn_cast<const PossiblyExactOperator>(&
I))
3620 exact = ExactOp->isExact();
3623Flags.setExact(exact);
3624Flags.setNoSignedWrap(nsw);
3625Flags.setNoUnsignedWrap(nuw);
3631void SelectionDAGBuilder::visitSDiv(
constUser &
I) {
3636Flags.setExact(isa<PossiblyExactOperator>(&
I) &&
3637 cast<PossiblyExactOperator>(&
I)->isExact());
3642void SelectionDAGBuilder::visitICmp(
constICmpInst &
I) {
3652// If a pointer's DAG type is larger than its memory type then the DAG values 3653// are zero-extended. This breaks signed comparisons so truncate back to the 3654// underlying type before doing the compare. 3661Flags.setSameSign(
I.hasSameSign());
3669void SelectionDAGBuilder::visitFCmp(
constFCmpInst &
I) {
3675auto *FPMO = cast<FPMathOperator>(&
I);
3680Flags.copyFMF(*FPMO);
3688// Check if the condition of the select has one use or two users that are both 3689// selects with the same condition. 3692 return isa<SelectInst>(V);
3696void SelectionDAGBuilder::visitSelect(
constUser &
I) {
3700unsigned NumValues = ValueVTs.
size();
3701if (NumValues == 0)
return;
3711bool IsUnaryAbs =
false;
3715if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
3716Flags.copyFMF(*FPOp);
3718Flags.setUnpredictable(
3719 cast<SelectInst>(
I).getMetadata(LLVMContext::MD_unpredictable));
3721// Min/max matching is only viable if all output VTs are the same. 3723EVT VT = ValueVTs[0];
3727// We care about the legality of the operation after it has been type 3732// If the vselect is legal, assume we want to leave this as a vector setcc + 3733// vselect. Otherwise, if this is going to be scalarized, we want to see if 3734// min/max is legal on the scalar type. 3735bool UseScalarMinMax = VT.
isVector() &&
3738// ValueTracking's select pattern matching does not account for -0.0, 3739// so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that 3740// -0.0 is less than +0.0. 3744switch (SPR.Flavor) {
3750switch (SPR.NaNBehavior) {
3763switch (SPR.NaNBehavior) {
3789// If the underlying comparison instruction is used by any other 3790// instruction, the consumed instructions won't be destroyed, so it is 3791// not profitable to convert to a min/max. 3807for (
unsigned i = 0; i != NumValues; ++i) {
3816for (
unsigned i = 0; i != NumValues; ++i) {
3830void SelectionDAGBuilder::visitTrunc(
constUser &
I) {
3831// TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). 3836if (
auto *Trunc = dyn_cast<TruncInst>(&
I)) {
3837Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
3838Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
3844void SelectionDAGBuilder::visitZExt(
constUser &
I) {
3845// ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 3846// ZExt also can't be a cast to bool for same reason. So, nothing much to do 3852if (
auto *PNI = dyn_cast<PossiblyNonNegInst>(&
I))
3853Flags.setNonNeg(PNI->hasNonNeg());
3855// Eagerly use nonneg information to canonicalize towards sign_extend if 3856// that is the target's preference. 3857// TODO: Let the target do this later. 3858if (
Flags.hasNonNeg() &&
3867void SelectionDAGBuilder::visitSExt(
constUser &
I) {
3868// SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 3869// SExt also can't be a cast to bool for same reason. So, nothing much to do 3876void SelectionDAGBuilder::visitFPTrunc(
constUser &
I) {
3877// FPTrunc is never a no-op cast, no need to check 3887void SelectionDAGBuilder::visitFPExt(
constUser &
I) {
3888// FPExt is never a no-op cast, no need to check 3895void SelectionDAGBuilder::visitFPToUI(
constUser &
I) {
3896// FPToUI is never a no-op cast, no need to check 3903void SelectionDAGBuilder::visitFPToSI(
constUser &
I) {
3904// FPToSI is never a no-op cast, no need to check 3911void SelectionDAGBuilder::visitUIToFP(
constUser &
I) {
3912// UIToFP is never a no-op cast, no need to check 3917if (
auto *PNI = dyn_cast<PossiblyNonNegInst>(&
I))
3918Flags.setNonNeg(PNI->hasNonNeg());
3923void SelectionDAGBuilder::visitSIToFP(
constUser &
I) {
3924// SIToFP is never a no-op cast, no need to check 3931void SelectionDAGBuilder::visitPtrToInt(
constUser &
I) {
3932// What to do depends on the size of the integer and the size of the pointer. 3933// We can either truncate, zero extend, or no-op, accordingly. 3945void SelectionDAGBuilder::visitIntToPtr(
constUser &
I) {
3946// What to do depends on the size of the integer and the size of the pointer. 3947// We can either truncate, zero extend, or no-op, accordingly. 3957void SelectionDAGBuilder::visitBitCast(
constUser &
I) {
3963// BitCast assures us that source and destination are the same size so this is 3964// either a BITCAST or a no-op. 3965if (DestVT !=
N.getValueType())
3967 DestVT,
N));
// convert types. 3968// Check if the original LLVM IR Operand was a ConstantInt, because getValue() 3969// might fold any kind of constant expression to an integer constant and that 3970// is not what we are looking for. Only recognize a bitcast of a genuine 3971// constant integer as an opaque constant. 3972elseif(
ConstantInt *
C = dyn_cast<ConstantInt>(
I.getOperand(0)))
3979void SelectionDAGBuilder::visitAddrSpaceCast(
constUser &
I) {
3981constValue *SV =
I.getOperand(0);
3986unsigned DestAS =
I.getType()->getPointerAddressSpace();
3994void SelectionDAGBuilder::visitInsertElement(
constUser &
I) {
4002 InVec, InVal, InIdx));
4005void SelectionDAGBuilder::visitExtractElement(
constUser &
I) {
4015void SelectionDAGBuilder::visitShuffleVector(
constUser &
I) {
4019if (
auto *SVI = dyn_cast<ShuffleVectorInst>(&
I))
4020Mask = SVI->getShuffleMask();
4022Mask = cast<ConstantExpr>(
I).getShuffleMask();
4028if (
all_of(Mask, [](
int Elem) {
return Elem == 0; }) &&
4030// Canonical splat form of first element of first input vector. 4038// For now, we only handle splats for scalable vectors. 4039// The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation 4040// for targets that support a SPLAT_VECTOR for non-scalable vector types. 4044unsigned MaskNumElts =
Mask.size();
4046if (SrcNumElts == MaskNumElts) {
4051// Normalize the shuffle vector since mask and vector length don't match. 4052if (SrcNumElts < MaskNumElts) {
4053// Mask is longer than the source vectors. We can use concatenate vector to 4054// make the mask and vectors lengths match. 4056if (MaskNumElts % SrcNumElts == 0) {
4057// Mask length is a multiple of the source vector length. 4058// Check if the shuffle is some kind of concatenation of the input 4060unsigned NumConcat = MaskNumElts / SrcNumElts;
4063for (
unsigned i = 0; i != MaskNumElts; ++i) {
4067// Ensure the indices in each SrcVT sized piece are sequential and that 4068// the same source is used for the whole piece. 4069if ((
Idx % SrcNumElts != (i % SrcNumElts)) ||
4070 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4071 ConcatSrcs[i / SrcNumElts] != (
int)(
Idx / SrcNumElts))) {
4075// Remember which source this index came from. 4076 ConcatSrcs[i / SrcNumElts] =
Idx / SrcNumElts;
4079// The shuffle is concatenating multiple vectors together. Just emit 4080// a CONCAT_VECTORS operation. 4083for (
auto Src : ConcatSrcs) {
4096unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
4097unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4101// Pad both vectors with undefs to make them the same length as the mask. 4112// Readjust mask for new input vector length. 4114for (
unsigned i = 0; i != MaskNumElts; ++i) {
4116if (
Idx >= (
int)SrcNumElts)
4117Idx -= SrcNumElts - PaddedMaskNumElts;
4123// If the concatenated vector was padded, extract a subvector with the 4124// correct number of elements. 4125if (MaskNumElts != PaddedMaskNumElts)
4133assert(SrcNumElts > MaskNumElts);
4135// Analyze the access pattern of the vector to see if we can extract 4136// two subvectors and do the shuffle. 4137int StartIdx[2] = {-1, -1};
// StartIdx to extract from 4138bool CanExtract =
true;
4139for (
intIdx : Mask) {
4144if (
Idx >= (
int)SrcNumElts) {
4149// If all the indices come from the same MaskNumElts sized portion of 4150// the sources we can use extract. Also make sure the extract wouldn't 4151// extract past the end of the source. 4153if (NewStartIdx + MaskNumElts > SrcNumElts ||
4154 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4156// Make sure we always update StartIdx as we use it to track if all 4157// elements are undef. 4158 StartIdx[Input] = NewStartIdx;
4161if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4166// Extract appropriate subvector and generate a vector shuffle 4167for (
unsigned Input = 0; Input < 2; ++Input) {
4168SDValue &Src = Input == 0 ? Src1 : Src2;
4169if (StartIdx[Input] < 0)
4177// Calculate new mask. 4179for (
int &
Idx : MappedOps) {
4180if (
Idx >= (
int)SrcNumElts)
4181Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4190// We can't use either concat vectors or extract subvectors so fall back to 4191// replacing the shuffle with extract and build vector. 4192// to insert and build vector. 4195for (
intIdx : Mask) {
4201SDValue &Src =
Idx < (int)SrcNumElts ? Src1 : Src2;
4202if (
Idx >= (
int)SrcNumElts)
Idx -= SrcNumElts;
4216constValue *Op0 =
I.getOperand(0);
4217constValue *Op1 =
I.getOperand(1);
4218Type *AggTy =
I.getType();
4220bool IntoUndef = isa<UndefValue>(Op0);
4221bool FromUndef = isa<UndefValue>(Op1);
4231unsigned NumAggValues = AggValueVTs.
size();
4232unsigned NumValValues = ValValueVTs.
size();
4235// Ignore an insertvalue that produces an empty object 4243// Copy the beginning value(s) from the original aggregate. 4244for (; i != LinearIndex; ++i)
4245 Values[i] = IntoUndef ?
DAG.
getUNDEF(AggValueVTs[i]) :
4247// Copy values from the inserted value(s). 4250for (; i != LinearIndex + NumValValues; ++i)
4251 Values[i] = FromUndef ?
DAG.
getUNDEF(AggValueVTs[i]) :
4254// Copy remaining value(s) from the original aggregate. 4255for (; i != NumAggValues; ++i)
4256 Values[i] = IntoUndef ?
DAG.
getUNDEF(AggValueVTs[i]) :
4265constValue *Op0 =
I.getOperand(0);
4267Type *ValTy =
I.getType();
4268bool OutOfUndef = isa<UndefValue>(Op0);
4276unsigned NumValValues = ValValueVTs.
size();
4278// Ignore a extractvalue that produces an empty object 4287// Copy out the selected value(s). 4288for (
unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4289 Values[i - LinearIndex] =
4298void SelectionDAGBuilder::visitGetElementPtr(
constUser &
I) {
4299Value *Op0 =
I.getOperand(0);
4300// Note that the pointer operand may be a vector of pointers. Take the scalar 4301// element which holds a pointer. 4308// Normalize Vector GEP - all scalar operands should be converted to the 4310bool IsVectorGEP =
I.getType()->isVectorTy();
4312 IsVectorGEP ? cast<VectorType>(
I.getType())->getElementCount()
4315if (IsVectorGEP && !
N.getValueType().isVector()) {
4324if (
StructType *StTy = GTI.getStructTypeOrNull()) {
4325unsignedField = cast<Constant>(
Idx)->getUniqueInteger().getZExtValue();
4331// In an inbounds GEP with an offset that is nonnegative even when 4332// interpreted as signed, assume there is no unsigned overflow. 4342// IdxSize is the width of the arithmetic according to IR semantics. 4343// In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth 4344// (and fix up the result later). 4349// We intentionally mask away the high bits here; ElementSize may not 4352/*isSigned=*/false,
/*implicitTrunc=*/true);
4353bool ElementScalable = ElementSize.
isScalable();
4355// If this is a scalar constant or a splat vector of constants, 4356// handle it quickly. 4357constauto *
C = dyn_cast<Constant>(
Idx);
4358if (
C && isa<VectorType>(
C->getType()))
4359C =
C->getSplatValue();
4361constauto *CI = dyn_cast_or_null<ConstantInt>(
C);
4362if (CI && CI->isZero())
4364if (CI && !ElementScalable) {
4374// In an inbounds GEP with an offset that is nonnegative even when 4375// interpreted as signed, assume there is no unsigned overflow. 4379Flags.setNoUnsignedWrap(
true);
4387// N = N + Idx * ElementMul; 4392 VectorElementCount);
4396// If the index is smaller or larger than intptr_t, truncate or extend 4401// The multiplication of an index by the type size does not wrap the 4402// pointer index type in a signed sense (mul nsw). 4405// The multiplication of an index by the type size does not wrap the 4406// pointer index type in an unsigned sense (mul nuw). 4409if (ElementScalable) {
4410EVT VScaleTy =
N.getValueType().getScalarType();
4419// If this is a multiply by a power of two, turn it into a shl 4420// immediately. This is a very common case. 4421if (ElementMul != 1) {
4422if (ElementMul.isPowerOf2()) {
4423unsigned Amt = ElementMul.logBase2();
4436// The successive addition of the current address, truncated to the 4437// pointer index type and interpreted as an unsigned number, and each 4438// offset, also interpreted as an unsigned number, does not wrap the 4439// pointer index type (add nuw). 4454if (PtrMemTy != PtrTy && !cast<GEPOperator>(
I).isInBounds())
4460void SelectionDAGBuilder::visitAlloca(
constAllocaInst &
I) {
4461// If this is a fixed sized alloca in the entry block of the function, 4462// allocate it statically on the stack. 4464return;
// getValue will auto-populate this. 4467Type *Ty =
I.getAllocatedType();
4471MaybeAlign Alignment = std::max(
DL.getPrefTypeAlign(Ty),
I.getAlign());
4491// Handle alignment. If the requested alignment is less than or equal to 4492// the stack alignment, ignore it. If the size is greater than or equal to 4493// the stack alignment, we note this in the DYNAMIC_STACKALLOC node. 4495if (*Alignment <= StackAlign)
4496 Alignment = std::nullopt;
4499// Round the size of the allocation up to the stack alignment size 4500// by add SA-1 to the size. This doesn't overflow because we're computing 4501// an address inside an alloca. 4506// Mask out the low bits for alignment purposes. 4522// If !noundef is not present, then !range violation results in a poison 4523// value rather than immediate undefined behavior. In theory, transferring 4524// these annotations to SDAG is fine, but in practice there are key SDAG 4525// transforms that are known not to be poison-safe, such as folding logical 4526// and/or to bitwise and/or. For now, only transfer !range if !noundef is 4528if (!
I.hasMetadata(LLVMContext::MD_noundef))
4530returnI.getMetadata(LLVMContext::MD_range);
4534if (
constauto *CB = dyn_cast<CallBase>(&
I)) {
4535// see comment in getRangeMetadata about this check 4536if (CB->hasRetAttr(Attribute::NoUndef))
4537return CB->getRange();
4544void SelectionDAGBuilder::visitLoad(
constLoadInst &
I) {
4546return visitAtomicLoad(
I);
4549constValue *SV =
I.getOperand(0);
4551// Swifterror values can come from either a function parameter with 4552// swifterror attribute or an alloca with swifterror attribute. 4553if (
constArgument *Arg = dyn_cast<Argument>(SV)) {
4554if (Arg->hasSwiftErrorAttr())
4555return visitLoadFromSwiftError(
I);
4558if (
constAllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4559if (Alloca->isSwiftError())
4560return visitLoadFromSwiftError(
I);
4566Type *Ty =
I.getType();
4570unsigned NumValues = ValueVTs.
size();
4574Align Alignment =
I.getAlign();
4577bool isVolatile =
I.isVolatile();
4582bool ConstantMemory =
false;
4584// Serialize volatile loads with other side effects. 4593// Do not serialize (non-volatile) loads of constant memory with anything. 4595 ConstantMemory =
true;
4598// Do not serialize non-volatile loads against each other. 4611for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4612// Serializing loads here may result in excessive register pressure, and 4613// TokenFactor places arbitrary choke points on the scheduler. SD scheduling 4614// could recover a bit by hoisting nodes upward in the chain by recognizing 4615// they are side-effect free or do not alias. The optimizer should really 4616// avoid this case by converting large object/array copies to llvm.memcpy 4617// (MaxParallelChains should always remain as failsafe). 4626// TODO: MachinePointerInfo only supports a fixed length offset. 4634 MMOFlags, AAInfo, Ranges);
4635 Chains[ChainI] =
L.getValue(1);
4637if (MemVTs[i] != ValueVTs[i])
4643if (!ConstantMemory) {
4656void SelectionDAGBuilder::visitStoreToSwiftError(
constStoreInst &
I) {
4658"call visitStoreToSwiftError when backend supports swifterror");
4662constValue *SrcV =
I.getOperand(0);
4664 SrcV->
getType(), ValueVTs, &Offsets, 0);
4665assert(ValueVTs.
size() == 1 && Offsets[0] == 0 &&
4666"expect a single EVT for swifterror");
4669// Create a virtual register, then update the virtual register. 4672// Chain, DL, Reg, N or Chain, DL, Reg, N, Glue 4673// Chain can be getRoot or getControlRoot. 4675SDValue(Src.getNode(), Src.getResNo()));
4679void SelectionDAGBuilder::visitLoadFromSwiftError(
constLoadInst &
I) {
4681"call visitLoadFromSwiftError when backend supports swifterror");
4684 !
I.hasMetadata(LLVMContext::MD_nontemporal) &&
4685 !
I.hasMetadata(LLVMContext::MD_invariant_load) &&
4686"Support volatile, non temporal, invariant for load_from_swift_error");
4688constValue *SV =
I.getOperand(0);
4689Type *Ty =
I.getType();
4694I.getAAMetadata()))) &&
4695"load_from_swift_error should not be constant memory");
4700 ValueVTs, &Offsets, 0);
4701assert(ValueVTs.
size() == 1 && Offsets[0] == 0 &&
4702"expect a single EVT for swifterror");
4704// Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT 4712void SelectionDAGBuilder::visitStore(
constStoreInst &
I) {
4714return visitAtomicStore(
I);
4716constValue *SrcV =
I.getOperand(0);
4717constValue *PtrV =
I.getOperand(1);
4721// Swifterror values can come from either a function parameter with 4722// swifterror attribute or an alloca with swifterror attribute. 4723if (
constArgument *Arg = dyn_cast<Argument>(PtrV)) {
4724if (Arg->hasSwiftErrorAttr())
4725return visitStoreToSwiftError(
I);
4728if (
constAllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4729if (Alloca->isSwiftError())
4730return visitStoreToSwiftError(
I);
4737 SrcV->
getType(), ValueVTs, &MemVTs, &Offsets);
4738unsigned NumValues = ValueVTs.
size();
4742// Get the lowered operands. Note that we do this after 4743// checking if NumResults is zero, because with zero results 4744// the operands won't have values in the map. 4751Align Alignment =
I.getAlign();
4757for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4758// See visitLoad comments. 4766// TODO: MachinePointerInfo only supports a fixed length offset. 4774if (MemVTs[i] != ValueVTs[i])
4777DAG.
getStore(Root, dl, Val,
Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4778 Chains[ChainI] = St;
4787void SelectionDAGBuilder::visitMaskedStore(
constCallInst &
I,
4788bool IsCompressing) {
4793// llvm.masked.store.*(Src0, Ptr, alignment, Mask) 4794 Src0 =
I.getArgOperand(0);
4795Ptr =
I.getArgOperand(1);
4796 Alignment = cast<ConstantInt>(
I.getArgOperand(2))->getAlignValue();
4797Mask =
I.getArgOperand(3);
4801// llvm.masked.compressstore.*(Src0, Ptr, Mask) 4802 Src0 =
I.getArgOperand(0);
4803Ptr =
I.getArgOperand(1);
4804Mask =
I.getArgOperand(2);
4805 Alignment =
I.getParamAlign(1).valueOrOne();
4808Value *PtrOperand, *MaskOperand, *Src0Operand;
4811 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4813 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4823if (
I.hasMetadata(LLVMContext::MD_nontemporal))
4845// Get a uniform base for the Gather/Scatter intrinsic. 4846// The first argument of the Gather/Scatter intrinsic is a vector of pointers. 4847// We try to represent it as a base pointer + vector of indices. 4848// Usually, the vector of pointers comes from a 'getelementptr' instruction. 4849// The first operand of the GEP may be a single pointer or a vector of pointers 4851// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind 4853// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind 4854// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, .. 4856// When the first GEP operand is a single pointer - it is the uniform base we 4857// are looking for. If first operand of the GEP is a splat vector - we 4858// extract the splat value and use it as a uniform base. 4859// In all other cases the function returns 'false'. 4868assert(
Ptr->getType()->isVectorTy() &&
"Unexpected pointer type");
4870// Handle splat constant pointer. 4871if (
auto *
C = dyn_cast<Constant>(
Ptr)) {
4872C =
C->getSplatValue();
4878ElementCount NumElts = cast<VectorType>(
Ptr->getType())->getElementCount();
4887if (!
GEP ||
GEP->getParent() != CurBB)
4890if (
GEP->getNumOperands() != 2)
4893constValue *BasePtr =
GEP->getPointerOperand();
4894constValue *IndexVal =
GEP->getOperand(
GEP->getNumOperands() - 1);
4896// Make sure the base is scalar and the index is a vector. 4900TypeSize ScaleVal =
DL.getTypeAllocSize(
GEP->getResultElementType());
4904// Target may not support the required addressing mode. 4918void SelectionDAGBuilder::visitMaskedScatter(
constCallInst &
I) {
4921// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask) 4926Align Alignment = cast<ConstantInt>(
I.getArgOperand(2))
4927 ->getMaybeAlignValue()
4938unsigned AS =
Ptr->getType()->getScalarType()->getPointerAddressSpace();
4958 Ops, MMO, IndexType,
false);
4963void SelectionDAGBuilder::visitMaskedLoad(
constCallInst &
I,
bool IsExpanding) {
4968// @llvm.masked.load.*(Ptr, alignment, Mask, Src0) 4969Ptr =
I.getArgOperand(0);
4970 Alignment = cast<ConstantInt>(
I.getArgOperand(1))->getAlignValue();
4971Mask =
I.getArgOperand(2);
4972 Src0 =
I.getArgOperand(3);
4976// @llvm.masked.expandload.*(Ptr, Mask, Src0) 4977Ptr =
I.getArgOperand(0);
4978 Alignment =
I.getParamAlign(0).valueOrOne();
4979Mask =
I.getArgOperand(1);
4980 Src0 =
I.getArgOperand(2);
4983Value *PtrOperand, *MaskOperand, *Src0Operand;
4986 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4988 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4999// Do not serialize masked loads of constant memory with anything. 5006if (
I.hasMetadata(LLVMContext::MD_nontemporal))
5016// The Load/Res may point to different values and both of them are output 5032void SelectionDAGBuilder::visitMaskedGather(
constCallInst &
I) {
5035// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) 5042Align Alignment = cast<ConstantInt>(
I.getArgOperand(1))
5043 ->getMaybeAlignValue()
5055unsigned AS =
Ptr->getType()->getScalarType()->getPointerAddressSpace();
5101AAMDNodes(),
nullptr, SSID, SuccessOrdering, FailureOrdering);
5104 dl, MemVT, VTs, InChain,
5118switch (
I.getOperation()) {
5174void SelectionDAGBuilder::visitFence(
constFenceInst &
I) {
5188void SelectionDAGBuilder::visitAtomicLoad(
constLoadInst &
I) {
5208nullptr, SSID, Order);
5224void SelectionDAGBuilder::visitAtomicStore(
constStoreInst &
I) {
5246nullptr, SSID, Ordering);
5260/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC 5262void SelectionDAGBuilder::visitTargetIntrinsic(
constCallInst &
I,
5263unsigned Intrinsic) {
5264// Ignore the callsite's attributes. A specific call site may be marked with 5265// readnone, but the lowering code will expect the chain based on the 5268bool HasChain = !
F->doesNotAccessMemory();
5270 HasChain &&
F->onlyReadsMemory() &&
F->willReturn() &&
F->doesNotThrow();
5272// Build the operand list. 5274if (HasChain) {
// If this intrinsic has side-effects, chainify it. 5276// We don't need to serialize loads against other loads. 5283// Info is set by getTgtMemIntrinsic 5290// Add the intrinsic ID as an integer operand if it's not a target intrinsic. 5296// Add all operands of the call to the operand list. 5297for (
unsigned i = 0, e =
I.arg_size(); i != e; ++i) {
5298constValue *Arg =
I.getArgOperand(i);
5299if (!
I.paramHasAttr(i, Attribute::ImmArg)) {
5304// Use TargetConstant instead of a regular constant for immarg. 5306if (
constConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5307assert(CI->getBitWidth() <= 64 &&
5308"large intrinsic immediates not handled");
5324// Propagate fast-math-flags from IR to node(s). 5326if (
auto *FPMO = dyn_cast<FPMathOperator>(&
I))
5327Flags.copyFMF(*FPMO);
5334auto *Token = Bundle->Inputs[0].get();
5336assert(Ops.
back().getValueType() != MVT::Glue &&
5337"Did not expected another glue node here.");
5343// In some cases, custom collection of operands from CallInst I may be needed. 5345if (IsTgtIntrinsic) {
5346// This is target intrinsic that touches memory 5348// TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic 5349// didn't yield anything useful. 5353elseif (
Info.fallbackAddressSpace)
5357Info.size,
I.getAAMetadata());
5358 }
elseif (!HasChain) {
5360 }
elseif (!
I.getType()->isVoidTy()) {
5374if (!
I.getType()->isVoidTy()) {
5375if (!isa<VectorType>(
I.getType()))
5380// Insert `assertalign` node if there's an alignment. 5390/// GetSignificand - Get the significand and build it into a floating-point 5391/// number with exponent of 1: 5393/// Op = (Op & 0x007fffff) | 0x3f800000; 5395/// where Op is the hexadecimal representation of floating point value. 5404/// GetExponent - Get the exponent: 5406/// (float)(int)(((Op & 0x7f800000) >> 23) - 127); 5408/// where Op is the hexadecimal representation of floating point value. 5422/// getF32Constant - Get 32-bit floating point constant. 5431// TODO: What fast-math-flags should be set on the floating-point nodes? 5433// IntegerPartOfX = ((int32_t)(t0); 5436// FractionalPartOfX = t0 - (float)IntegerPartOfX; 5440// IntegerPartOfX <<= 23; 5447SDValue TwoToFractionalPartOfX;
5449// For floating-point precision of 6: 5451// TwoToFractionalPartOfX = 5453// (0.735607626f + 0.252464424f * x) * x; 5455// error 0.0144103317, which is 6 bits 5464// For floating-point precision of 12: 5466// TwoToFractionalPartOfX = 5469// (0.224338339f + 0.792043434e-1f * x) * x) * x; 5471// error 0.000107046256, which is 13 to 14 bits 5482 }
else {
// LimitFloatPrecision <= 18 5483// For floating-point precision of 18: 5485// TwoToFractionalPartOfX = 5489// (0.554906021e-1f + 5490// (0.961591928e-2f + 5491// (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 5492// error 2.47208000*10^(-7), which is better than 18 bits 5514// Add the exponent into the result in integer domain. 5520/// expandExp - Lower an exp intrinsic. Handles the special sequences for 5521/// limited-precision mode. 5524if (
Op.getValueType() == MVT::f32 &&
5527// Put the exponent in the right bit position for later addition to the 5532// TODO: What fast-math-flags should be set here? 5538// No special expansion. 5542/// expandLog - Lower a log intrinsic. Handles the special sequences for 5543/// limited-precision mode. 5546// TODO: What fast-math-flags should be set on the floating-point nodes? 5548if (
Op.getValueType() == MVT::f32 &&
5552// Scale the exponent by log(2). 5558// Get the significand and build it into a floating-point number with 5564// For floating-point precision of 6: 5568// (1.4034025f - 0.23903021f * x) * x; 5570// error 0.0034276066, which is better than 8 bits 5579// For floating-point precision of 12: 5585// (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; 5587// error 0.000061011436, which is 14 bits 5601 }
else {
// LimitFloatPrecision <= 18 5602// For floating-point precision of 18: 5610// (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; 5612// error 0.0000023660568, which is better than 18 bits 5637// No special expansion. 5641/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for 5642/// limited-precision mode. 5645// TODO: What fast-math-flags should be set on the floating-point nodes? 5647if (
Op.getValueType() == MVT::f32 &&
5654// Get the significand and build it into a floating-point number with 5658// Different possible minimax approximations of significand in 5659// floating-point for various degrees of accuracy over [1,2]. 5662// For floating-point precision of 6: 5664// Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; 5666// error 0.0049451742, which is more than 7 bits 5675// For floating-point precision of 12: 5681// (.645142248f - 0.816157886e-1f * x) * x) * x) * x; 5683// error 0.0000876136000, which is better than 13 bits 5697 }
else {
// LimitFloatPrecision <= 18 5698// For floating-point precision of 18: 5707// 0.25691327e-1f * x) * x) * x) * x) * x) * x; 5709// error 0.0000018516, which is better than 18 bits 5734// No special expansion. 5738/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for 5739/// limited-precision mode. 5742// TODO: What fast-math-flags should be set on the floating-point nodes? 5744if (
Op.getValueType() == MVT::f32 &&
5748// Scale the exponent by log10(2) [0.30102999f]. 5753// Get the significand and build it into a floating-point number with 5759// For floating-point precision of 6: 5763// (0.60948995f - 0.10380950f * x) * x; 5765// error 0.0014886165, which is 6 bits 5774// For floating-point precision of 12: 5779// (-0.31664806f + 0.47637168e-1f * x) * x) * x; 5781// error 0.00019228036, which is better than 12 bits 5792 }
else {
// LimitFloatPrecision <= 18 5793// For floating-point precision of 18: 5800// (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; 5802// error 0.0000037995730, which is better than 18 bits 5821return DAG.
getNode(
ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5824// No special expansion. 5828/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for 5829/// limited-precision mode. 5832if (
Op.getValueType() == MVT::f32 &&
5836// No special expansion. 5840/// visitPow - Lower a pow intrinsic. Handles the special sequences for 5841/// limited-precision mode with x == 10.0f. 5846if (
LHS.getValueType() == MVT::f32 &&
RHS.getValueType() == MVT::f32 &&
5850 IsExp10 = LHSC->isExactlyValue(Ten);
5854// TODO: What fast-math-flags should be set on the FMUL node? 5856// Put the exponent in the right bit position for later addition to the 5859// #define LOG2OF10 3.3219281f 5860// t0 = Op * LOG2OF10; 5866// No special expansion. 5870/// ExpandPowI - Expand a llvm.powi intrinsic. 5873// If RHS is a constant, we can expand this out to a multiplication tree if 5874// it's beneficial on the target, otherwise we end up lowering to a call to 5875// __powidf2 (for example). 5877unsigned Val = RHSC->getSExtValue();
5885// Get the exponent as a positive value. 5888// We use the simple binary decomposition method to generate the multiply 5889// sequence. There are more optimal ways to do this (for example, 5890// powi(x,15) generates one more multiply than it should), but this has 5891// the benefit of being both really simple and much better than a libcall. 5892SDValue Res;
// Logically starts equal to 1.0 5894// TODO: Intrinsics should have fast-math-flags that propagate to these 5902 Res = CurSquare;
// 1.0*CurSquare. 5906 CurSquare, CurSquare);
5910// If the original was negative, invert the result, producing 1/(x*x*x). 5911if (RHSC->getSExtValue() < 0)
5918// Otherwise, expand to a libcall. 5925EVT VT =
LHS.getValueType();
5930// If the type is legal but the operation isn't, this node might survive all 5931// the way to operation legalization. If we end up there and we do not have 5932// the ability to widen the type (if VT*2 is not legal), we cannot expand the 5935// Coax the legalizer into expanding the node during type legalization instead 5936// by bumping the size by one bit. This will force it to Promote, enabling the 5937// early expansion and avoiding the need to expand later. 5939// We don't have to do this if Scale is 0; that can always be expanded, unless 5940// it's a saturating signed operation. Those can experience true integer 5941// division overflow, a case which we must avoid. 5943// FIXME: We wouldn't have to do this (or any of the early 5944// expansion/promotion) if it was possible to expand a libcall of an 5945// illegal type during operation legalization. But it's not, so things 5948if ((ScaleInt > 0 || (Saturating &&
Signed)) &&
5952 Opcode, VT, ScaleInt);
5966// For saturating operations, we need to shift up the LHS to get the 5967// proper saturation width, and then shift down again afterwards. 5982// getUnderlyingArgRegs - Find underlying registers used for a truncated, 5983// bitcasted, or split argument. Returns a list of <Register, size in bits> 5987switch (
N.getOpcode()) {
5990 Regs.emplace_back(cast<RegisterSDNode>(
Op)->
getReg(),
5991Op.getValueType().getSizeInBits());
6011/// If the DbgValueInst is a dbg_value of a function argument, create the 6012/// corresponding DBG_VALUE machine instruction for it now. At the end of 6013/// instruction selection, they will be inserted to the entry BB. 6014/// We don't currently support this for variadic dbg_values, as they shouldn't 6015/// appear for function arguments or in the prologue. 6016bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6019constArgument *Arg = dyn_cast<Argument>(V);
6026// Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind 6027// we've been asked to pursue. 6031// For VRegs, in instruction referencing mode, create a DBG_INSTR_REF 6032// pointing at the VReg, which will be patched up later. 6033auto &Inst =
TII->get(TargetOpcode::DBG_INSTR_REF);
6035/* Reg */ Reg,
/* isDef */false,
/* isImp */false,
6036/* isKill */false,
/* isDead */false,
6037/* isUndef */false,
/* isEarlyClobber */false,
6038/* SubReg */ 0,
/* isDebug */true)});
6040auto *NewDIExpr = FragExpr;
6041// We don't have an "Indirect" field in DBG_INSTR_REF, fold that into 6047returnBuildMI(MF,
DL, Inst,
false, MOs, Variable, NewDIExpr);
6049// Create a completely standard DBG_VALUE. 6050auto &Inst =
TII->get(TargetOpcode::DBG_VALUE);
6051returnBuildMI(MF,
DL, Inst, Indirect, Reg, Variable, FragExpr);
6055if (Kind == FuncArgumentDbgValueKind::Value) {
6056// ArgDbgValues are hoisted to the beginning of the entry block. So we 6057// should only emit as ArgDbgValue if the dbg.value intrinsic is found in 6063// ArgDbgValues are hoisted to the beginning of the entry block. So we 6064// should only emit as ArgDbgValue if the dbg.value intrinsic describes a 6065// variable that also is a param. 6067// Although, if we are at the top of the entry block already, we can still 6068// emit using ArgDbgValue. This might catch some situations when the 6069// dbg.value refers to an argument that isn't used in the entry block, so 6070// any CopyToReg node would be optimized out and the only way to express 6071// this DBG_VALUE is by using the physical reg (or FI) as done in this 6072// method. ArgDbgValues are hoisted to the beginning of the entry block. So 6073// we should only emit as ArgDbgValue if the Variable is an argument to the 6074// current function, and the dbg.value intrinsic is found in the entry 6076bool VariableIsFunctionInputArg = Variable->
isParameter() &&
6077 !
DL->getInlinedAt();
6079if (!IsInPrologue && !VariableIsFunctionInputArg)
6082// Here we assume that a function argument on IR level only can be used to 6083// describe one input parameter on source level. If we for example have 6084// source code like this 6086// struct A { long x, y; }; 6087// void foo(struct A a, long b) { 6095// define void @foo(i32 %a1, i32 %a2, i32 %b) { 6097// call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment 6098// call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment 6099// call void @llvm.dbg.value(metadata i32 %b, "b", 6101// call void @llvm.dbg.value(metadata i32 %a1, "b" 6104// then the last dbg.value is describing a parameter "b" using a value that 6105// is an argument. But since we already has used %a1 to describe a parameter 6106// we should not handle that last dbg.value here (that would result in an 6107// incorrect hoisting of the DBG_VALUE to the function entry). 6108// Notice that we allow one dbg.value per IR level argument, to accommodate 6109// for the situation with fragments above. 6110// If there is no node for the value being handled, we return true to skip 6111// the normal generation of debug info, as it would kill existing debug 6112// info for the parameter in case of duplicates. 6113if (VariableIsFunctionInputArg) {
6118return !NodeMap[
V].getNode();
6123bool IsIndirect =
false;
6124 std::optional<MachineOperand>
Op;
6125// Some arguments' frame index is recorded during argument lowering. 6127if (FI != std::numeric_limits<int>::max())
6131if (!
Op &&
N.getNode()) {
6134if (ArgRegsAndSizes.
size() == 1)
6135Reg = ArgRegsAndSizes.
front().first;
6137if (Reg &&
Reg.isVirtual()) {
6145 IsIndirect =
Kind != FuncArgumentDbgValueKind::Value;
6149if (!
Op &&
N.getNode()) {
6150// Check if frame index is available. 6154 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6159// Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg 6163for (
constauto &RegAndSize : SplitRegs) {
6164// If the expression is already a fragment, the current register 6165// offset+size might extend beyond the fragment. In this case, only 6166// the register bits that are inside the fragment are relevant. 6167int RegFragmentSizeInBits = RegAndSize.second;
6169uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6170// The register is entirely outside the expression fragment, 6171// so is irrelevant for debug info. 6172if (
Offset >= ExprFragmentSizeInBits)
6174// The register is partially outside the expression fragment, only 6175// the low bits within the fragment are relevant for debug info. 6176if (
Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6177 RegFragmentSizeInBits = ExprFragmentSizeInBits -
Offset;
6182 Expr,
Offset, RegFragmentSizeInBits);
6183Offset += RegAndSize.second;
6184// If a valid fragment expression cannot be created, the variable's 6185// correct value cannot be determined and so it is set as Undef. 6193 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6194 Kind != FuncArgumentDbgValueKind::Value);
6199// Check if ValueMap has reg number. 6205V->getType(), std::nullopt);
6206if (RFV.occupiesMultipleRegs()) {
6207 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6212 IsIndirect =
Kind != FuncArgumentDbgValueKind::Value;
6213 }
elseif (ArgRegsAndSizes.
size() > 1) {
6214// This was split due to the calling convention, and no virtual register 6215// mapping exists for the value. 6216 splitMultiRegDbgValue(ArgRegsAndSizes);
6225"Expected inlined-at fields to agree");
6229 NewMI = MakeVRegDbgValue(
Op->getReg(), Expr, IsIndirect);
6231 NewMI =
BuildMI(MF,
DL,
TII->get(TargetOpcode::DBG_VALUE),
true, *
Op,
6234// Otherwise, use ArgDbgValues. 6239/// Return the appropriate SDDbgValue based on N. 6244unsigned DbgSDNodeOrder) {
6245if (
auto *FISDN = dyn_cast<FrameIndexSDNode>(
N.getNode())) {
6246// Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe 6247// stack slot locations. 6249// Consider "int x = 0; int *px = &x;". There are two kinds of interesting 6250// debug values here after optimization: 6252// dbg.value(i32* %px, !"int *px", !DIExpression()), and 6253// dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) 6255// Both describe the direct values of their associated variables. 6257/*IsIndirect*/false, dl, DbgSDNodeOrder);
6260/*IsIndirect*/false, dl, DbgSDNodeOrder);
6265case Intrinsic::smul_fix:
6267case Intrinsic::umul_fix:
6269case Intrinsic::smul_fix_sat:
6271case Intrinsic::umul_fix_sat:
6273case Intrinsic::sdiv_fix:
6275case Intrinsic::udiv_fix:
6277case Intrinsic::sdiv_fix_sat:
6279case Intrinsic::udiv_fix_sat:
6286void SelectionDAGBuilder::lowerCallToExternalSymbol(
constCallInst &
I,
6287constchar *FunctionName) {
6288assert(FunctionName &&
"FunctionName must not be nullptr");
6295/// Given a @llvm.call.preallocated.setup, return the corresponding 6296/// preallocated call. 6298assert(cast<CallBase>(PreallocatedSetup)
6301"expected call_preallocated_setup Value");
6302for (
constauto *U : PreallocatedSetup->
users()) {
6303auto *UseCall = cast<CallBase>(U);
6304constFunction *Fn = UseCall->getCalledFunction();
6305if (!Fn || Fn->
getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6312/// If DI is a debug value with an EntryValue expression, lower it using the 6313/// corresponding physical register of the associated Argument value 6314/// (guaranteed to exist by the verifier). 6315bool SelectionDAGBuilder::visitEntryValueDbgValue(
6321// These properties are guaranteed by the verifier. 6322constArgument *Arg = cast<Argument>(Values[0]);
6328dbgs() <<
"Dropping dbg.value: expression is entry_value but " 6329"couldn't find an associated register for the Argument\n");
6332Register ArgVReg = ArgIt->getSecond();
6335if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6337 Variable, Expr, PhysReg,
false/*IsIndidrect*/, DbgLoc, SDNodeOrder);
6338DAG.
AddDbgValue(SDV,
false/*treat as dbg.declare byval parameter*/);
6341LLVM_DEBUG(
dbgs() <<
"Dropping dbg.value: expression is entry_value but " 6342"couldn't find a physical register\n");
6346/// Lower the call to the specified intrinsic function. 6347void SelectionDAGBuilder::visitConvergenceControl(
constCallInst &
I,
6348unsigned Intrinsic) {
6351case Intrinsic::experimental_convergence_anchor:
6354case Intrinsic::experimental_convergence_entry:
6357case Intrinsic::experimental_convergence_loop: {
6359auto *Token = Bundle->Inputs[0].get();
6367void SelectionDAGBuilder::visitVectorHistogram(
constCallInst &
I,
6368unsigned IntrinsicID) {
6369// For now, we're only lowering an 'add' histogram. 6370// We can add others later, e.g. saturating adds, min/max. 6371assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6372"Tried to lower unsupported histogram type");
6393unsigned AS =
Ptr->getType()->getScalarType()->getPointerAddressSpace();
6419 Ops, MMO, IndexType);
6425void SelectionDAGBuilder::visitVectorExtractLastActive(
constCallInst &
I,
6426unsigned Intrinsic) {
6427assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6428"Tried lowering invalid vector extract last");
6444EVT BoolVT =
Mask.getValueType().getScalarType();
6452/// Lower the call to the specified intrinsic function. 6453void SelectionDAGBuilder::visitIntrinsicCall(
constCallInst &
I,
6454unsigned Intrinsic) {
6461if (
auto *FPOp = dyn_cast<FPMathOperator>(&
I))
6462Flags.copyFMF(*FPOp);
6466// By default, turn this into a target intrinsic node. 6467 visitTargetIntrinsic(
I, Intrinsic);
6469case Intrinsic::vscale: {
6474case Intrinsic::vastart: visitVAStart(
I);
return;
6475case Intrinsic::vaend: visitVAEnd(
I);
return;
6476case Intrinsic::vacopy: visitVACopy(
I);
return;
6477case Intrinsic::returnaddress:
6482case Intrinsic::addressofreturnaddress:
6487case Intrinsic::sponentry:
6492case Intrinsic::frameaddress:
6497case Intrinsic::read_volatile_register:
6498case Intrinsic::read_register: {
6502DAG.
getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6510case Intrinsic::write_register: {
6512Value *RegValue =
I.getArgOperand(1);
6515DAG.
getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6520case Intrinsic::memcpy: {
6521constauto &MCI = cast<MemCpyInst>(
I);
6525// @llvm.memcpy defines 0 and 1 to both mean no alignment. 6526Align DstAlign = MCI.getDestAlign().valueOrOne();
6527Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6528Align Alignment = std::min(DstAlign, SrcAlign);
6529bool isVol = MCI.isVolatile();
6530// FIXME: Support passing different dest/src alignments to the memcpy DAG 6534/* AlwaysInline */false, &
I, std::nullopt,
6538 updateDAGForMaybeTailCall(MC);
6541case Intrinsic::memcpy_inline: {
6542constauto &MCI = cast<MemCpyInlineInst>(
I);
6546assert(isa<ConstantSDNode>(
Size) &&
"memcpy_inline needs constant size");
6547// @llvm.memcpy.inline defines 0 and 1 to both mean no alignment. 6548Align DstAlign = MCI.getDestAlign().valueOrOne();
6549Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6550Align Alignment = std::min(DstAlign, SrcAlign);
6551bool isVol = MCI.isVolatile();
6552// FIXME: Support passing different dest/src alignments to the memcpy DAG 6555/* AlwaysInline */true, &
I, std::nullopt,
6559 updateDAGForMaybeTailCall(MC);
6562case Intrinsic::memset: {
6563constauto &MSI = cast<MemSetInst>(
I);
6567// @llvm.memset defines 0 and 1 to both mean no alignment. 6568Align Alignment = MSI.getDestAlign().valueOrOne();
6569bool isVol = MSI.isVolatile();
6572 Root, sdl, Op1, Op2, Op3, Alignment, isVol,
/* AlwaysInline */false,
6574 updateDAGForMaybeTailCall(MS);
6577case Intrinsic::memset_inline: {
6578constauto &MSII = cast<MemSetInlineInst>(
I);
6582assert(isa<ConstantSDNode>(
Size) &&
"memset_inline needs constant size");
6583// @llvm.memset defines 0 and 1 to both mean no alignment. 6584Align DstAlign = MSII.getDestAlign().valueOrOne();
6585bool isVol = MSII.isVolatile();
6588/* AlwaysInline */true, &
I,
6591 updateDAGForMaybeTailCall(MC);
6594case Intrinsic::memmove: {
6595constauto &MMI = cast<MemMoveInst>(
I);
6599// @llvm.memmove defines 0 and 1 to both mean no alignment. 6600Align DstAlign = MMI.getDestAlign().valueOrOne();
6601Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6602Align Alignment = std::min(DstAlign, SrcAlign);
6603bool isVol = MMI.isVolatile();
6604// FIXME: Support passing different dest/src alignments to the memmove DAG 6608/* OverrideTailCall */ std::nullopt,
6612 updateDAGForMaybeTailCall(MM);
6615case Intrinsic::memcpy_element_unordered_atomic: {
6621Type *LengthTy =
MI.getLength()->getType();
6622unsigned ElemSz =
MI.getElementSizeInBytes();
6628 updateDAGForMaybeTailCall(MC);
6631case Intrinsic::memmove_element_unordered_atomic: {
6632auto &
MI = cast<AtomicMemMoveInst>(
I);
6637Type *LengthTy =
MI.getLength()->getType();
6638unsigned ElemSz =
MI.getElementSizeInBytes();
6644 updateDAGForMaybeTailCall(MC);
6647case Intrinsic::memset_element_unordered_atomic: {
6648auto &
MI = cast<AtomicMemSetInst>(
I);
6653Type *LengthTy =
MI.getLength()->getType();
6654unsigned ElemSz =
MI.getElementSizeInBytes();
6659 updateDAGForMaybeTailCall(MC);
6662case Intrinsic::call_preallocated_setup: {
6671case Intrinsic::call_preallocated_arg: {
6678 MVT::i32);
// arg index 6686case Intrinsic::dbg_declare: {
6687constauto &DI = cast<DbgDeclareInst>(
I);
6688// Debug intrinsics are handled separately in assignment tracking mode. 6689// Some intrinsics are handled right after Argument lowering. 6690if (AssignmentTrackingEnabled ||
6693LLVM_DEBUG(
dbgs() <<
"SelectionDAG visiting dbg_declare: " << DI <<
"\n");
6697// Assume dbg.declare can not currently use DIArgList, i.e. 6698// it is non-variadic. 6699assert(!DI.hasArgList() &&
"Only dbg.value should currently use DIArgList");
6704case Intrinsic::dbg_label: {
6707assert(Label &&
"Missing label");
6714case Intrinsic::dbg_assign: {
6715// Debug intrinsics are handled separately in assignment tracking mode. 6716if (AssignmentTrackingEnabled)
6718// If assignment tracking hasn't been enabled then fall through and treat 6719// the dbg.assign as a dbg.value. 6722case Intrinsic::dbg_value: {
6723// Debug intrinsics are handled separately in assignment tracking mode. 6724if (AssignmentTrackingEnabled)
6744 SDNodeOrder, IsVariadic))
6750case Intrinsic::eh_typeid_for: {
6751// Find the type id for the given typeinfo. 6759case Intrinsic::eh_return_i32:
6760case Intrinsic::eh_return_i64:
6768case Intrinsic::eh_unwind_init:
6771case Intrinsic::eh_dwarf_cfa:
6776case Intrinsic::eh_sjlj_callsite: {
6777ConstantInt *CI = cast<ConstantInt>(
I.getArgOperand(0));
6783case Intrinsic::eh_sjlj_functioncontext: {
6784// Get and store the index of the function context. 6787 cast<AllocaInst>(
I.getArgOperand(0)->stripPointerCasts());
6792case Intrinsic::eh_sjlj_setjmp: {
6802case Intrinsic::eh_sjlj_longjmp:
6806case Intrinsic::eh_sjlj_setup_dispatch:
6810case Intrinsic::masked_gather:
6811 visitMaskedGather(
I);
6813case Intrinsic::masked_load:
6816case Intrinsic::masked_scatter:
6817 visitMaskedScatter(
I);
6819case Intrinsic::masked_store:
6820 visitMaskedStore(
I);
6822case Intrinsic::masked_expandload:
6823 visitMaskedLoad(
I,
true/* IsExpanding */);
6825case Intrinsic::masked_compressstore:
6826 visitMaskedStore(
I,
true/* IsCompressing */);
6828case Intrinsic::powi:
6835case Intrinsic::log2:
6839case Intrinsic::log10:
6846case Intrinsic::exp2:
6854case Intrinsic::sqrt:
6855case Intrinsic::fabs:
6859case Intrinsic::asin:
6860case Intrinsic::acos:
6861case Intrinsic::atan:
6862case Intrinsic::sinh:
6863case Intrinsic::cosh:
6864case Intrinsic::tanh:
6865case Intrinsic::exp10:
6866case Intrinsic::floor:
6867case Intrinsic::ceil:
6868case Intrinsic::trunc:
6869case Intrinsic::rint:
6870case Intrinsic::nearbyint:
6871case Intrinsic::round:
6872case Intrinsic::roundeven:
6873case Intrinsic::canonicalize: {
6878case Intrinsic::sqrt: Opcode =
ISD::FSQRT;
break;
6879case Intrinsic::fabs: Opcode =
ISD::FABS;
break;
6880case Intrinsic::sin: Opcode =
ISD::FSIN;
break;
6881case Intrinsic::cos: Opcode =
ISD::FCOS;
break;
6882case Intrinsic::tan: Opcode =
ISD::FTAN;
break;
6883case Intrinsic::asin: Opcode =
ISD::FASIN;
break;
6884case Intrinsic::acos: Opcode =
ISD::FACOS;
break;
6885case Intrinsic::atan: Opcode =
ISD::FATAN;
break;
6886case Intrinsic::sinh: Opcode =
ISD::FSINH;
break;
6887case Intrinsic::cosh: Opcode =
ISD::FCOSH;
break;
6888case Intrinsic::tanh: Opcode =
ISD::FTANH;
break;
6891case Intrinsic::ceil: Opcode =
ISD::FCEIL;
break;
6893case Intrinsic::rint: Opcode =
ISD::FRINT;
break;
6906case Intrinsic::atan2:
6912case Intrinsic::lround:
6913case Intrinsic::llround:
6914case Intrinsic::lrint:
6915case Intrinsic::llrint: {
6922case Intrinsic::lrint: Opcode =
ISD::LRINT;
break;
6932case Intrinsic::minnum:
6938case Intrinsic::maxnum:
6944case Intrinsic::minimum:
6950case Intrinsic::maximum:
6956case Intrinsic::minimumnum:
6962case Intrinsic::maximumnum:
6968case Intrinsic::copysign:
6974case Intrinsic::ldexp:
6980case Intrinsic::sincos:
6981case Intrinsic::frexp: {
6986case Intrinsic::sincos:
6989case Intrinsic::frexp:
7000case Intrinsic::arithmetic_fence: {
7012#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ 7013 case Intrinsic::INTRINSIC: 7014#include "llvm/IR/ConstrainedOps.def" 7015 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(
I));
7017#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID: 7018#include "llvm/IR/VPIntrinsics.def" 7019 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(
I));
7021case Intrinsic::fptrunc_round: {
7022// Get the last argument, the metadata and convert it to an integer in the 7024Metadata *MD = cast<MetadataAsValue>(
I.getArgOperand(1))->getMetadata();
7025 std::optional<RoundingMode> RoundMode =
7030// Propagate fast-math-flags from IR to node(s). 7032Flags.copyFMF(*cast<FPMathOperator>(&
I));
7043case Intrinsic::fmuladd: {
7053// TODO: Intrinsic calls should have fast-math-flags. 7064case Intrinsic::convert_to_fp16:
7071case Intrinsic::convert_from_fp16:
7077case Intrinsic::fptosi_sat: {
7084case Intrinsic::fptoui_sat: {
7091case Intrinsic::set_rounding:
7097case Intrinsic::is_fpclass: {
7102 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
7108 !
F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7109// If ISD::IS_FPCLASS should be expanded, do it right now, because the 7110// expansion can use illegal types. Making expansion early allows 7111// legalizing these types prior to selection. 7124case Intrinsic::get_fpenv: {
7129// Use GET_FPENV if it is legal or custom. Otherwise use memory-based node 7130// and temporary storage in stack. 7139int SPFI = cast<FrameIndexSDNode>(Temp.
getNode())->getIndex();
7146 Res =
DAG.
getLoad(EnvVT, sdl, Chain, Temp, MPI);
7152case Intrinsic::set_fpenv: {
7158// If SET_FPENV is custom or legal, use it. Otherwise use loading 7159// environment from memory. 7163// Allocate space in stack, copy environment bits into it and use this 7164// memory in SET_FPENV_MEM. 7166int SPFI = cast<FrameIndexSDNode>(Temp.
getNode())->getIndex();
7169 Chain =
DAG.
getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7179case Intrinsic::reset_fpenv:
7182case Intrinsic::get_fpmode:
7191case Intrinsic::set_fpmode:
7196case Intrinsic::reset_fpmode: {
7201case Intrinsic::pcmarker: {
7206case Intrinsic::readcyclecounter: {
7214case Intrinsic::readsteadycounter: {
7222case Intrinsic::bitreverse:
7227case Intrinsic::bswap:
7232case Intrinsic::cttz: {
7234ConstantInt *CI = cast<ConstantInt>(
I.getArgOperand(1));
7240case Intrinsic::ctlz: {
7242ConstantInt *CI = cast<ConstantInt>(
I.getArgOperand(1));
7248case Intrinsic::ctpop: {
7254case Intrinsic::fshl:
7255case Intrinsic::fshr: {
7256bool IsFSHL =
Intrinsic == Intrinsic::fshl;
7260EVT VT =
X.getValueType();
7271case Intrinsic::sadd_sat: {
7277case Intrinsic::uadd_sat: {
7283case Intrinsic::ssub_sat: {
7289case Intrinsic::usub_sat: {
7295case Intrinsic::sshl_sat: {
7301case Intrinsic::ushl_sat: {
7307case Intrinsic::smul_fix:
7308case Intrinsic::umul_fix:
7309case Intrinsic::smul_fix_sat:
7310case Intrinsic::umul_fix_sat: {
7318case Intrinsic::sdiv_fix:
7319case Intrinsic::udiv_fix:
7320case Intrinsic::sdiv_fix_sat:
7321case Intrinsic::udiv_fix_sat: {
7326 Op1, Op2, Op3,
DAG, TLI));
7329case Intrinsic::smax: {
7335case Intrinsic::smin: {
7341case Intrinsic::umax: {
7347case Intrinsic::umin: {
7353case Intrinsic::abs: {
7354// TODO: Preserve "int min is poison" arg in SDAG? 7359case Intrinsic::scmp: {
7366case Intrinsic::ucmp: {
7373case Intrinsic::stacksave: {
7381case Intrinsic::stackrestore:
7385case Intrinsic::get_dynamic_area_offset: {
7389// Result type for @llvm.get.dynamic.area.offset should match PtrTy for 7400case Intrinsic::stackguard: {
7421case Intrinsic::stackprotector: {
7422// Emit code into the DAG to store the stack guard onto the stack. 7431 Src =
getValue(
I.getArgOperand(0));
// The guard's value. 7441// Store the stack protector onto the stack. 7443 Chain, sdl, Src, FIN,
7450case Intrinsic::objectsize:
7453case Intrinsic::is_constant:
7456case Intrinsic::annotation:
7457case Intrinsic::ptr_annotation:
7458case Intrinsic::launder_invariant_group:
7459case Intrinsic::strip_invariant_group:
7460// Drop the intrinsic, but forward the value 7464case Intrinsic::assume:
7465case Intrinsic::experimental_noalias_scope_decl:
7466case Intrinsic::var_annotation:
7467case Intrinsic::sideeffect:
7468// Discard annotate attributes, noalias scope declarations, assumptions, and 7469// artificial side-effects. 7472case Intrinsic::codeview_annotation: {
7473// Emit a label associated with this metadata. 7476Metadata *MD = cast<MetadataAsValue>(
I.getArgOperand(0))->getMetadata();
7483case Intrinsic::init_trampoline: {
7484constFunction *
F = cast<Function>(
I.getArgOperand(1)->stripPointerCasts());
7499case Intrinsic::adjust_trampoline:
7504case Intrinsic::gcroot: {
7506"only valid in functions with gc specified, enforced by Verifier");
7508constValue *Alloca =
I.getArgOperand(0)->stripPointerCasts();
7509constConstant *TypeMap = cast<Constant>(
I.getArgOperand(1));
7515case Intrinsic::gcread:
7516case Intrinsic::gcwrite:
7518case Intrinsic::get_rounding:
7524case Intrinsic::expect:
7525case Intrinsic::expect_with_probability:
7526// Just replace __builtin_expect(exp, c) and 7527// __builtin_expect_with_probability(exp, c, p) with EXP. 7531case Intrinsic::ubsantrap:
7532case Intrinsic::debugtrap:
7533case Intrinsic::trap: {
7535I.getAttributes().getFnAttr(
"trap-func-name").getValueAsString();
7536if (TrapFuncName.
empty()) {
7538case Intrinsic::trap:
7541case Intrinsic::debugtrap:
7544case Intrinsic::ubsantrap:
7548 cast<ConstantInt>(
I.getArgOperand(0))->getZExtValue(), sdl,
7554I.hasFnAttr(Attribute::NoMerge));
7558if (Intrinsic == Intrinsic::ubsantrap) {
7560Args[0].Val =
I.getArgOperand(0);
7566 CLI.setDebugLoc(sdl).setChain(
getRoot()).setLibCallee(
7571 CLI.NoMerge =
I.hasFnAttr(Attribute::NoMerge);
7577case Intrinsic::allow_runtime_check:
7578case Intrinsic::allow_ubsan_check:
7582case Intrinsic::uadd_with_overflow:
7583case Intrinsic::sadd_with_overflow:
7584case Intrinsic::usub_with_overflow:
7585case Intrinsic::ssub_with_overflow:
7586case Intrinsic::umul_with_overflow:
7587case Intrinsic::smul_with_overflow: {
7591case Intrinsic::uadd_with_overflow:
Op =
ISD::UADDO;
break;
7592case Intrinsic::sadd_with_overflow:
Op =
ISD::SADDO;
break;
7593case Intrinsic::usub_with_overflow:
Op =
ISD::USUBO;
break;
7594case Intrinsic::ssub_with_overflow:
Op =
ISD::SSUBO;
break;
7595case Intrinsic::umul_with_overflow:
Op =
ISD::UMULO;
break;
7596case Intrinsic::smul_with_overflow:
Op =
ISD::SMULO;
break;
7602EVT OverflowVT = MVT::i1;
7611case Intrinsic::prefetch: {
7613unsigned rw = cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue();
7626/* align */ std::nullopt, Flags);
7628// Chain the prefetch in parallel with any pending loads, to stay out of 7629// the way of later optimizations. 7635case Intrinsic::lifetime_start:
7636case Intrinsic::lifetime_end: {
7637bool IsStart = (
Intrinsic == Intrinsic::lifetime_start);
7638// Stack coloring is not enabled in O0, discard region information. 7642const int64_t ObjectSize =
7643 cast<ConstantInt>(
I.getArgOperand(0))->getSExtValue();
7648for (
constValue *Alloca : Allocas) {
7649constAllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
7651// Could not find an Alloca. 7655// First check that the Alloca is static, otherwise it won't have a 7656// valid frame index. 7665Offset = -1;
// Cannot determine offset from alloca to lifetime object. 7672case Intrinsic::pseudoprobe: {
7673autoGuid = cast<ConstantInt>(
I.getArgOperand(0))->getZExtValue();
7674autoIndex = cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue();
7675auto Attr = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
7680case Intrinsic::invariant_start:
7681// Discard region information. 7685case Intrinsic::invariant_end:
7686// Discard region information. 7688case Intrinsic::clear_cache: {
7693 {InputChain, StartVal, EndVal});
7698case Intrinsic::donothing:
7699case Intrinsic::seh_try_begin:
7700case Intrinsic::seh_scope_begin:
7701case Intrinsic::seh_try_end:
7702case Intrinsic::seh_scope_end:
7705case Intrinsic::experimental_stackmap:
7708case Intrinsic::experimental_patchpoint_void:
7709case Intrinsic::experimental_patchpoint:
7712case Intrinsic::experimental_gc_statepoint:
7715case Intrinsic::experimental_gc_result:
7716 visitGCResult(cast<GCResultInst>(
I));
7718case Intrinsic::experimental_gc_relocate:
7719 visitGCRelocate(cast<GCRelocateInst>(
I));
7721case Intrinsic::instrprof_cover:
7723case Intrinsic::instrprof_increment:
7725case Intrinsic::instrprof_timestamp:
7727case Intrinsic::instrprof_value_profile:
7729case Intrinsic::instrprof_mcdc_parameters:
7731case Intrinsic::instrprof_mcdc_tvbitmap_update:
7733case Intrinsic::localescape: {
7737// Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission 7738// is the same on all targets. 7739for (
unsignedIdx = 0, E =
I.arg_size();
Idx < E; ++
Idx) {
7740Value *Arg =
I.getArgOperand(
Idx)->stripPointerCasts();
7741if (isa<ConstantPointerNull>(Arg))
7742continue;
// Skip null pointers. They represent a hole in index space. 7745"can only escape static allocas");
7750TII->get(TargetOpcode::LOCAL_ESCAPE))
7758case Intrinsic::localrecover: {
7759// i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx) 7762// Get the symbol that defines the frame offset. 7763auto *Fn = cast<Function>(
I.getArgOperand(0)->stripPointerCasts());
7764auto *
Idx = cast<ConstantInt>(
I.getArgOperand(2));
7766unsigned(
Idx->getLimitedValue(std::numeric_limits<int>::max()));
7774// Create a MCSymbol for the label to avoid any target lowering 7775// that would make this PC relative. 7780// Add the offset to the FP. 7787case Intrinsic::fake_use: {
7790// For Values not declared or previously used in this basic block, the 7791// NodeMap will not have an entry, and `getValue` will assert if V has no 7792// valid register value. 7793auto FakeUseValue = [&]() ->
SDValue {
7798// If there's a virtual register allocated and initialized for this 7802// FIXME: Do we want to preserve constants? It seems pointless. 7803if (isa<Constant>(V))
7807if (!FakeUseValue || FakeUseValue.isUndef())
7810 Ops[1] = FakeUseValue;
7811// Also, do not translate a fake use with an undef operand, or any other 7813if (!Ops[1] || Ops[1].
isUndef())
7819case Intrinsic::eh_exceptionpointer:
7820case Intrinsic::eh_exceptioncode: {
7821// Get the exception pointer vreg, copy from it, and resize it to fit. 7822constauto *CPI = cast<CatchPadInst>(
I.getArgOperand(0));
7827if (Intrinsic == Intrinsic::eh_exceptioncode)
7832case Intrinsic::xray_customevent: {
7833// Here we want to make sure that the intrinsic behaves as if it has a 7834// specific calling convention. 7841// We want to say that we always want the arguments in registers. 7850// We need to enforce the calling convention for the callsite, so that 7851// argument ordering is enforced correctly, and that register allocation can 7852// see that some registers may be assumed clobbered and have to preserve 7853// them across calls to the intrinsic. 7861case Intrinsic::xray_typedevent: {
7862// Here we want to make sure that the intrinsic behaves as if it has a 7863// specific calling convention. 7870// We want to say that we always want the arguments in registers. 7871// It's unclear to me how manipulating the selection DAG here forces callers 7872// to provide arguments in registers instead of on the stack. 7883// We need to enforce the calling convention for the callsite, so that 7884// argument ordering is enforced correctly, and that register allocation can 7885// see that some registers may be assumed clobbered and have to preserve 7886// them across calls to the intrinsic. 7888 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7894case Intrinsic::experimental_deoptimize:
7897case Intrinsic::stepvector:
7900case Intrinsic::vector_reduce_fadd:
7901case Intrinsic::vector_reduce_fmul:
7902case Intrinsic::vector_reduce_add:
7903case Intrinsic::vector_reduce_mul:
7904case Intrinsic::vector_reduce_and:
7905case Intrinsic::vector_reduce_or:
7906case Intrinsic::vector_reduce_xor:
7907case Intrinsic::vector_reduce_smax:
7908case Intrinsic::vector_reduce_smin:
7909case Intrinsic::vector_reduce_umax:
7910case Intrinsic::vector_reduce_umin:
7911case Intrinsic::vector_reduce_fmax:
7912case Intrinsic::vector_reduce_fmin:
7913case Intrinsic::vector_reduce_fmaximum:
7914case Intrinsic::vector_reduce_fminimum:
7915 visitVectorReduce(
I, Intrinsic);
7918case Intrinsic::icall_branch_funnel: {
7927"llvm.icall.branch.funnel operand must be a GlobalValue");
7930structBranchFunnelTarget {
7936for (
unsignedOp = 1,
N =
I.arg_size();
Op !=
N;
Op += 2) {
7941"to the same GlobalValue");
7944auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7947"llvm.icall.branch.funnel operand must be a GlobalValue");
7953 [](
const BranchFunnelTarget &T1,
const BranchFunnelTarget &T2) {
7954returnT1.Offset < T2.Offset;
7957for (
auto &
T : Targets) {
7972case Intrinsic::wasm_landingpad_index:
7973// Information this intrinsic contained has been transferred to 7974// MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely 7978case Intrinsic::aarch64_settag:
7979case Intrinsic::aarch64_settag_zero: {
7981bool ZeroMemory =
Intrinsic == Intrinsic::aarch64_settag_zero;
7990case Intrinsic::amdgcn_cs_chain: {
7991assert(
I.arg_size() == 5 &&
"Additional args not supported yet");
7992assert(cast<ConstantInt>(
I.getOperand(4))->isZero() &&
7993"Non-zero flags not supported yet");
7995// At this point we don't care if it's amdgpu_cs_chain or 7996// amdgpu_cs_chain_preserve. 8004// We only have 2 actual args: one for the SGPRs and one for the VGPRs. 8005// We'll also tack the value of the EXEC mask at the end. 8009for (
unsignedIdx : {2, 3, 1}) {
8012 Arg.
Ty =
I.getOperand(
Idx)->getType();
8017assert(Args[0].IsInReg &&
"SGPR args should be marked inreg");
8018assert(!Args[1].IsInReg &&
"VGPR args should not be marked inreg");
8019Args[2].IsInReg =
true;
// EXEC should be inreg 8024 .setCallee(
CC,
RetTy, Callee, std::move(Args))
8027 .setConvergent(
I.isConvergent());
8029 std::pair<SDValue, SDValue>
Result =
8033"Should've lowered as tail call");
8038case Intrinsic::ptrmask: {
8042// On arm64_32, pointers are 32 bits when stored in memory, but 8043// zero-extended to 64 bits when in registers. Thus the mask is 32 bits to 8044// match the index type, but the pointer is 64 bits, so the the mask must be 8045// zero-extended up to 64 bits to match the pointer. 8058case Intrinsic::threadlocal_address: {
8062case Intrinsic::get_active_lane_mask: {
8065EVT ElementVT =
Index.getValueType();
8068 visitTargetIntrinsic(
I, Intrinsic);
8086case Intrinsic::experimental_get_vector_length: {
8087assert(cast<ConstantInt>(
I.getOperand(1))->getSExtValue() > 0 &&
8088"Expected positive VF");
8089unsigned VF = cast<ConstantInt>(
I.getOperand(1))->getZExtValue();
8090bool IsScalable = cast<ConstantInt>(
I.getOperand(2))->isOne();
8096 visitTargetIntrinsic(
I, Intrinsic);
8100// Expand to a umin between the trip count and the maximum elements the type 8104// Extend the trip count to at least the result VT. 8114// Clip to the result type if needed. 8120case Intrinsic::experimental_vector_partial_reduce_add: {
8123 visitTargetIntrinsic(
I, Intrinsic);
8132case Intrinsic::experimental_cttz_elts: {
8135EVT OpVT =
Op.getValueType();
8138 visitTargetIntrinsic(
I, Intrinsic);
8143// Compare the input vector elements to zero & use to count trailing zeros 8150// If the zero-is-poison flag is set, we can assume the upper limit 8151// of the result is VF-1. 8153 !cast<ConstantSDNode>(
getValue(
I.getOperand(1)))->isZero();
8155if (isa<ScalableVectorType>(
I.getOperand(0)->getType()))
8162// Create the new vector type & get the vector length 8183case Intrinsic::vector_insert: {
8188// The intrinsic's index type is i64, but the SDNode requires an index type 8189// suitable for the target. Convert the index as required. 8191if (
Index.getValueType() != VectorIdxTy)
8199case Intrinsic::vector_extract: {
8204// The intrinsic's index type is i64, but the SDNode requires an index type 8205// suitable for the target. Convert the index as required. 8207if (
Index.getValueType() != VectorIdxTy)
8214case Intrinsic::experimental_vector_match: {
8220EVT ResVT =
Mask.getValueType();
8223// If the target has native support for this vector match operation, lower 8224// the intrinsic untouched; otherwise, expand it below. 8226 visitTargetIntrinsic(
I, Intrinsic);
8232for (
unsigned i = 0; i < SearchSize; ++i) {
8244case Intrinsic::vector_reverse:
8245 visitVectorReverse(
I);
8247case Intrinsic::vector_splice:
8248 visitVectorSplice(
I);
8250case Intrinsic::callbr_landingpad:
8251 visitCallBrLandingPad(
I);
8253case Intrinsic::vector_interleave2:
8254 visitVectorInterleave(
I);
8256case Intrinsic::vector_deinterleave2:
8257 visitVectorDeinterleave(
I);
8259case Intrinsic::experimental_vector_compress:
8266case Intrinsic::experimental_convergence_anchor:
8267case Intrinsic::experimental_convergence_entry:
8268case Intrinsic::experimental_convergence_loop:
8269 visitConvergenceControl(
I, Intrinsic);
8271case Intrinsic::experimental_vector_histogram_add: {
8272 visitVectorHistogram(
I, Intrinsic);
8275case Intrinsic::experimental_vector_extract_last_active: {
8276 visitVectorExtractLastActive(
I, Intrinsic);
8282void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8286// We do not need to serialize constrained FP intrinsics against 8287// each other or against (nonvolatile) loads, so they can be 8288// chained like loads. 8298// Push node to the appropriate list so that future instructions can be 8299// chained up correctly. 8303// The only reason why ebIgnore nodes still need to be chained is that 8304// they might depend on the current rounding mode, and therefore must 8305// not be moved across instruction that may change that mode. 8308// These must not be moved across calls or instructions that may change 8309// floating-point exception masks. 8310 PendingConstrainedFP.push_back(OutChain);
8313// These must not be moved across calls or instructions that may change 8314// floating-point exception masks or read floating-point exception flags. 8315// In addition, they cannot be optimized out even if unused. 8316 PendingConstrainedFPStrict.push_back(OutChain);
8328Flags.setNoFPExcept(
true);
8330if (
auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8331Flags.copyFMF(*FPOp);
8336#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 8337 case Intrinsic::INTRINSIC: \ 8338 Opcode = ISD::STRICT_##DAGN; \ 8340#include "llvm/IR/ConstrainedOps.def" 8341case Intrinsic::experimental_constrained_fmuladd: {
8343// Break fmuladd into fmul and fadd. 8348 pushOutChain(
Mul, EB);
8359// A few strict DAG nodes carry additional operands that are not 8360// set up by the default code above. 8369auto *
FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8379 pushOutChain(Result, EB);
8386 std::optional<unsigned> ResOPC;
8388case Intrinsic::vp_ctlz: {
8389bool IsZeroUndef = cast<ConstantInt>(VPIntrin.
getArgOperand(1))->isOne();
8390 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8393case Intrinsic::vp_cttz: {
8394bool IsZeroUndef = cast<ConstantInt>(VPIntrin.
getArgOperand(1))->isOne();
8395 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8398case Intrinsic::vp_cttz_elts: {
8399bool IsZeroPoison = cast<ConstantInt>(VPIntrin.
getArgOperand(1))->isOne();
8400 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;
8403#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \ 8404 case Intrinsic::VPID: \ 8405 ResOPC = ISD::VPSD; \ 8407#include "llvm/IR/VPIntrinsics.def" 8412"Inconsistency: no SDNode available for this VPIntrinsic!");
8414if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8415 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8417return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8418 : ISD::VP_REDUCE_FMUL;
8424void SelectionDAGBuilder::visitVPLoad(
8433// Do not serialize variable-length loads of constant memory with 8444 MMO,
false/*IsExpanding */);
8450void SelectionDAGBuilder::visitVPGather(
8486 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8492void SelectionDAGBuilder::visitVPStore(
8496EVT VT = OpValues[0].getValueType();
8509/* IsTruncating */false,
/*IsCompressing*/false);
8514void SelectionDAGBuilder::visitVPScatter(
8519EVT VT = OpValues[0].getValueType();
8549 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8550 OpValues[2], OpValues[3]},
8556void SelectionDAGBuilder::visitVPStridedLoad(
8575 OpValues[2], OpValues[3], MMO,
8576false/*IsExpanding*/);
8583void SelectionDAGBuilder::visitVPStridedStore(
8587EVT VT = OpValues[0].getValueType();
8599DAG.
getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8601/*IsCompressing*/false);
8607void SelectionDAGBuilder::visitVPCmp(
constVPCmpIntrinsic &VPIntrin) {
8615// FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan) 8616// flags, but calls that don't return floating-point types can't be 8617// FPMathOperators, like vp.fcmp. This affects constrained fcmp too. 8627// #2 is the condition code 8632"Unexpected target EVL type");
8641void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8648if (
constauto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8649return visitVPCmp(*CmpI);
8660"Unexpected target EVL type");
8666if (
I == EVLParamPos)
8674if (
auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8681 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8684 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8686case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8687 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8690 visitVPStore(VPIntrin, OpValues);
8692case ISD::VP_SCATTER:
8693 visitVPScatter(VPIntrin, OpValues);
8695case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8696 visitVPStridedStore(VPIntrin, OpValues);
8698case ISD::VP_FMULADD: {
8699assert(OpValues.
size() == 5 &&
"Unexpected number of operands");
8701if (
auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8708 ISD::VP_FMUL,
DL, VTs,
8709 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8712 {
Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8717case ISD::VP_IS_FPCLASS: {
8720autoConstant = OpValues[1]->getAsZExtVal();
8723 {OpValues[0],
Check, OpValues[2], OpValues[3]});
8727case ISD::VP_INTTOPTR: {
8738case ISD::VP_PTRTOINT: {
8753case ISD::VP_CTLZ_ZERO_UNDEF:
8755case ISD::VP_CTTZ_ZERO_UNDEF:
8756case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
8757case ISD::VP_CTTZ_ELTS: {
8759DAG.
getNode(Opcode,
DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8771// Insert a label before the invoke call to mark the try range. This can be 8772// used to detect deletion of the invoke via the MachineModuleInfo. 8775// For SjLj, keep track of which landing pads go with which invokes 8776// so as to maintain the ordering of pads in the LSDA. 8782// Now that the call site is handled, stop tracking it. 8792assert(BeginLabel &&
"BeginLabel should've been set");
8796// Insert a label at the end of the invoke call to mark the try range. This 8797// can be used to detect deletion of the invoke via the MachineModuleInfo. 8801// Inform MachineModuleInfo of range. 8803// There is a platform (e.g. wasm) that uses funclet style IR but does not 8804// actually use outlined funclets and their LSDA info style. 8806assert(
II &&
"II should've been set");
8817std::pair<SDValue, SDValue>
8823// Both PendingLoads and PendingExports must be flushed here; 8824// this call might not return. 8831 std::pair<SDValue, SDValue> Result = TLI.
LowerCallTo(CLI);
8834"Non-null chain expected with non-tail call!");
8835assert((Result.second.getNode() || !Result.first.getNode()) &&
8836"Null value expected with tail call!");
8838if (!Result.second.getNode()) {
8839// As a special case, a null chain means that a tail call has been emitted 8840// and the DAG root is already updated. 8843// Since there's no actual continuation from this block, nothing can be 8844// relying on us setting vregs for them. 8845 PendingExports.clear();
8860bool isTailCall,
bool isMustTailCall,
8870constValue *SwiftErrorVal =
nullptr;
8874// Avoid emitting tail calls in functions with the disable-tail-calls 8876auto *Caller = CB.
getParent()->getParent();
8877if (Caller->getFnAttribute(
"disable-tail-calls").getValueAsString() ==
8878"true" && !isMustTailCall)
8881// We can't tail call inside a function with a swifterror argument. Lowering 8882// does not support this yet. It would have to move into the swifterror 8883// register before the call. 8885 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8894if (V->getType()->isEmptyTy())
8898 Entry.Node = ArgNode; Entry.Ty = V->getType();
8902// Use swifterror virtual register as input to the call. 8905// We find the virtual register for the actual swifterror argument. 8906// Instead of using the Value, we use the virtual register instead. 8912 Args.push_back(Entry);
8914// If we have an explicit sret argument that is an Instruction, (i.e., it 8915// might point to function-local memory), we can't meaningfully tail-call. 8916if (Entry.IsSRet && isa<Instruction>(V))
8920// If call site has a cfguardtarget operand bundle, create and add an 8921// additional ArgListEntry. 8924Value *V = Bundle->Inputs[0];
8926 Entry.Node = ArgNode;
8927 Entry.Ty = V->getType();
8928 Entry.IsCFGuardTarget =
true;
8929 Args.push_back(Entry);
8932// Check if target-independent constraints permit a tail call here. 8933// Target-dependent constraints are checked within TLI->LowerCallTo. 8937// Disable tail calls if there is an swifterror argument. Targets have not 8938// been updated to support tail calls. 8947"Target doesn't support calls with kcfi operand bundles.");
8948 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8955auto *Token = Bundle->Inputs[0].get();
8970// Set the pointer authentication info if we have it. 8974"This target doesn't support calls with ptrauth operand bundles.");
8978 std::pair<SDValue, SDValue> Result =
lowerInvokable(CLI, EHPadBB);
8980if (Result.first.getNode()) {
8985// The last element of CLI.InVals has the SDValue for swifterror return. 8986// Here we copy it to a virtual register and update SwiftErrorMap for 8989// Get the last element of InVals. 9000// Check to see if this load can be trivially constant folded, e.g. if the 9001// input is from a string literal. 9002if (
constConstant *LoadInput = dyn_cast<Constant>(PtrVal)) {
9003// Cast pointer to the type we really want to load. 9014// Otherwise, we have to emit the load. If the pointer is to unfoldable but 9015// still constant memory, the input chain can be the entry node. 9017bool ConstantMemory =
false;
9019// Do not serialize (non-volatile) loads of constant memory with anything. 9022 ConstantMemory =
true;
9024// Do not serialize non-volatile loads against each other. 9038/// Record the value for an instruction that produces an integer result, 9039/// converting the type where necessary. 9040void SelectionDAGBuilder::processIntegerCallValue(
constInstruction &
I,
9049/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return 9050/// true and lower it. Otherwise return false, and it will be lowered like a 9052/// The caller already checked that \p I calls the appropriate LibFunc with a 9053/// correct prototype. 9054bool SelectionDAGBuilder::visitMemCmpBCmpCall(
constCallInst &
I) {
9055constValue *
LHS =
I.getArgOperand(0), *
RHS =
I.getArgOperand(1);
9069if (Res.first.getNode()) {
9070 processIntegerCallValue(
I, Res.first,
true);
9075// memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 9076// memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 9080// If the target has a fast compare for the given size, it will return a 9081// preferred load type for that size. Require that the load VT is legal and 9082// that the target supports unaligned loads of that type. Otherwise, return 9084auto hasFastLoadsAndCompare = [&](
unsigned NumBits) {
9088// TODO: Handle 5 byte compare as 4-byte + 1 byte. 9089// TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. 9090// TODO: Check alignment of src and dest ptrs. 9102// This turns into unaligned loads. We only do this if the target natively 9103// supports the MVT we'll be loading or if it is small enough (<= 4) that 9104// we'll only produce a small number of byte loads. 9107switch (NumBitsToCompare) {
9119 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9129// Bitcast to a wide integer type if the loads are vectors. 9137 processIntegerCallValue(
I, Cmp,
false);
9141/// See if we can lower a memchr call into an optimized form. If so, return 9142/// true and lower it. Otherwise return false, and it will be lowered like a 9144/// The caller already checked that \p I calls the appropriate LibFunc with a 9145/// correct prototype. 9146bool SelectionDAGBuilder::visitMemChrCall(
constCallInst &
I) {
9147constValue *Src =
I.getArgOperand(0);
9152 std::pair<SDValue, SDValue> Res =
9156if (Res.first.getNode()) {
9165/// See if we can lower a mempcpy call into an optimized form. If so, return 9166/// true and lower it. Otherwise return false, and it will be lowered like a 9168/// The caller already checked that \p I calls the appropriate LibFunc with a 9169/// correct prototype. 9170bool SelectionDAGBuilder::visitMemPCpyCall(
constCallInst &
I) {
9177// DAG::getMemcpy needs Alignment to be defined. 9178Align Alignment = std::min(DstAlign, SrcAlign);
9182// In the mempcpy context we need to pass in a false value for isTailCall 9183// because the return pointer needs to be adjusted by the size of 9184// the copied memory. 9187 Root, sdl, Dst, Src,
Size, Alignment,
false,
false,
/*CI=*/nullptr,
9191"** memcpy should not be lowered as TailCall in mempcpy context **");
9194// Check if Size needs to be truncated or extended. 9197// Adjust return pointer to point just past the last dst byte. 9204/// See if we can lower a strcpy call into an optimized form. If so, return 9205/// true and lower it, otherwise return false and it will be lowered like a 9207/// The caller already checked that \p I calls the appropriate LibFunc with a 9208/// correct prototype. 9209bool SelectionDAGBuilder::visitStrCpyCall(
constCallInst &
I,
bool isStpcpy) {
9210constValue *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9213 std::pair<SDValue, SDValue> Res =
9218if (Res.first.getNode()) {
9227/// See if we can lower a strcmp call into an optimized form. If so, return 9228/// true and lower it, otherwise return false and it will be lowered like a 9230/// The caller already checked that \p I calls the appropriate LibFunc with a 9231/// correct prototype. 9232bool SelectionDAGBuilder::visitStrCmpCall(
constCallInst &
I) {
9233constValue *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9236 std::pair<SDValue, SDValue> Res =
9241if (Res.first.getNode()) {
9242 processIntegerCallValue(
I, Res.first,
true);
9250/// See if we can lower a strlen call into an optimized form. If so, return 9251/// true and lower it, otherwise return false and it will be lowered like a 9253/// The caller already checked that \p I calls the appropriate LibFunc with a 9254/// correct prototype. 9255bool SelectionDAGBuilder::visitStrLenCall(
constCallInst &
I) {
9256constValue *Arg0 =
I.getArgOperand(0);
9259 std::pair<SDValue, SDValue> Res =
9262if (Res.first.getNode()) {
9263 processIntegerCallValue(
I, Res.first,
false);
9271/// See if we can lower a strnlen call into an optimized form. If so, return 9272/// true and lower it, otherwise return false and it will be lowered like a 9274/// The caller already checked that \p I calls the appropriate LibFunc with a 9275/// correct prototype. 9276bool SelectionDAGBuilder::visitStrNLenCall(
constCallInst &
I) {
9277constValue *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9280 std::pair<SDValue, SDValue> Res =
9284if (Res.first.getNode()) {
9285 processIntegerCallValue(
I, Res.first,
false);
9293/// See if we can lower a unary floating-point operation into an SDNode with 9294/// the specified Opcode. If so, return true and lower it, otherwise return 9295/// false and it will be lowered like a normal call. 9296/// The caller already checked that \p I calls the appropriate LibFunc with a 9297/// correct prototype. 9298bool SelectionDAGBuilder::visitUnaryFloatCall(
constCallInst &
I,
9300// We already checked this call's prototype; verify it doesn't modify errno. 9301if (!
I.onlyReadsMemory())
9305Flags.copyFMF(cast<FPMathOperator>(
I));
9313/// See if we can lower a binary floating-point operation into an SDNode with 9314/// the specified Opcode. If so, return true and lower it. Otherwise return 9315/// false, and it will be lowered like a normal call. 9316/// The caller already checked that \p I calls the appropriate LibFunc with a 9317/// correct prototype. 9318bool SelectionDAGBuilder::visitBinaryFloatCall(
constCallInst &
I,
9320// We already checked this call's prototype; verify it doesn't modify errno. 9321if (!
I.onlyReadsMemory())
9325Flags.copyFMF(cast<FPMathOperator>(
I));
9334void SelectionDAGBuilder::visitCall(
constCallInst &
I) {
9335// Handle inline assembly differently. 9336if (
I.isInlineAsm()) {
9344if (
F->isDeclaration()) {
9345// Is this an LLVM intrinsic or a target-specific intrinsic? 9346unsigned IID =
F->getIntrinsicID();
9349 IID =
II->getIntrinsicID(
F);
9352 visitIntrinsicCall(
I, IID);
9357// Check for well-known libc/libm calls. If the function is internal, it 9358// can't be a library call. Don't do the check if marked as nobuiltin for 9359// some reason or the call site requires strict floating point semantics. 9361if (!
I.isNoBuiltin() && !
I.isStrictFP() && !
F->hasLocalLinkage() &&
9367if (visitMemCmpBCmpCall(
I))
9370case LibFunc_copysign:
9371case LibFunc_copysignf:
9372case LibFunc_copysignl:
9373// We already checked this call's prototype; verify it doesn't modify 9375if (
I.onlyReadsMemory()) {
9379LHS.getValueType(), LHS, RHS));
9401case LibFunc_fminimum_num:
9402case LibFunc_fminimum_numf:
9403case LibFunc_fminimum_numl:
9407case LibFunc_fmaximum_num:
9408case LibFunc_fmaximum_numf:
9409case LibFunc_fmaximum_numl:
9476case LibFunc_sqrt_finite:
9477case LibFunc_sqrtf_finite:
9478case LibFunc_sqrtl_finite:
9488case LibFunc_nearbyint:
9489case LibFunc_nearbyintf:
9490case LibFunc_nearbyintl:
9543if (visitMemCmpBCmpCall(
I))
9546case LibFunc_mempcpy:
9547if (visitMemPCpyCall(
I))
9551if (visitMemChrCall(
I))
9555if (visitStrCpyCall(
I,
false))
9559if (visitStrCpyCall(
I,
true))
9563if (visitStrCmpCall(
I))
9567if (visitStrLenCall(
I))
9570case LibFunc_strnlen:
9571if (visitStrNLenCall(
I))
9583// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't 9584// have to do anything here to lower funclet bundles. 9585// CFGuardTarget bundles are lowered in LowerCallTo. 9586assert(!
I.hasOperandBundlesOtherThan(
9587 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9588 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9589 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9590 LLVMContext::OB_convergencectrl}) &&
9591"Cannot lower calls with arbitrary operand bundles!");
9595if (
I.hasDeoptState())
9598// Check if we can potentially perform a tail call. More detailed checking 9599// is be done within LowerCallTo, after more information about the call is 9609// Gather the call ptrauth data from the operand bundle: 9610// [ i32 <key>, i64 <discriminator> ] 9611constauto *Key = cast<ConstantInt>(PAB->Inputs[0]);
9612constValue *Discriminator = PAB->Inputs[1];
9614assert(Key->getType()->isIntegerTy(32) &&
"Invalid ptrauth key");
9615assert(Discriminator->getType()->isIntegerTy(64) &&
9616"Invalid ptrauth discriminator");
9618// Look through ptrauth constants to find the raw callee. 9619// Do a direct unauthenticated call if we found it and everything matches. 9620if (
constauto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))
9621if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
9626// Functions should never be ptrauth-called directly. 9627assert(!isa<Function>(CalleeV) &&
"invalid direct ptrauth call");
9629// Otherwise, do an authenticated indirect call. 9639/// AsmOperandInfo - This contains information for each constraint that we are 9643 /// CallOperand - If this is the result output operand or a clobber 9644 /// this is null, otherwise it is the incoming operand to the CallInst. 9645 /// This gets modified as the asm is processed. 9648 /// AssignedRegs - If this is a register or register class operand, this 9649 /// contains the set of register corresponding to the operand. 9656 /// Whether or not this operand accesses memory 9658// Indirect operand accesses access memory. 9662for (
constauto &Code : Codes)
9671}
// end anonymous namespace 9673/// Make sure that the output operand \p OpInfo and its corresponding input 9674/// operand \p MatchingOpInfo have compatible constraint types (otherwise error 9677 SDISelAsmOperandInfo &MatchingOpInfo,
9679if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9685 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9687 OpInfo.ConstraintVT);
9688 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9690 MatchingOpInfo.ConstraintVT);
9691constbool OutOpIsIntOrFP =
9692 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
9693constbool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
9694 MatchingOpInfo.ConstraintVT.isFloatingPoint();
9695if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
9696// FIXME: error out in a more elegant fashion 9698" with a matching output constraint of" 9699" incompatible type!");
9701 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9704/// Get a direct memory input to behave well as an indirect operand. 9705/// This may introduce stores, hence the need for a \p Chain. 9706/// \return The (possibly updated) chain. 9708 SDISelAsmOperandInfo &OpInfo,
9712// If we don't have an indirect input, put it in the constpool if we can, 9713// otherwise spill it to a stack slot. 9714// TODO: This isn't quite right. We need to handle these according to 9715// the addressing mode that the constraint wants. Also, this may take 9716// an additional register for the computation and we don't want that 9719// If the operand is a float, integer, or vector constant, spill to a 9720// constant pool entry to get its address. 9721constValue *OpVal = OpInfo.CallOperandVal;
9722if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9723 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
9729// Otherwise, create a stack slot and emit a store to it before the asm. 9739DL.getPrefTypeAlign(Ty),
false,
9742 Chain = DAG.
getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9745 OpInfo.CallOperand = StackSlot;
9750/// GetRegistersForValue - Assign registers (virtual or physical) for the 9751/// specified operand. We prefer to assign virtual registers, to allow the 9752/// register allocator to handle the assignment process. However, if the asm 9753/// uses features that we can't model on machineinstrs, we have SDISel do the 9754/// allocation. This produces generally horrible, but correct, code. 9756/// OpInfo describes the operand 9757/// RefOpInfo describes the matching operand if any, the operand otherwise 9758static std::optional<unsigned>
9760 SDISelAsmOperandInfo &OpInfo,
9761 SDISelAsmOperandInfo &RefOpInfo) {
9769// No work to do for memory/address operands. 9774// If this is a constraint for a single physreg, or a constraint for a 9775// register class, find it. 9776unsigned AssignedReg;
9779 &
TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9780// RC is unset only on failure. Return immediately. 9784// Get the actual register value type. This is important, because the user 9785// may have asked for (e.g.) the AX register in i32 type. We need to 9786// remember that AX is actually i16 to get the right extension. 9787constMVT RegVT = *
TRI.legalclasstypes_begin(*RC);
9789if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9790// If this is an FP operand in an integer register (or visa versa), or more 9791// generally if the operand value disagrees with the register class we plan 9792// to stick it in, fix the operand type. 9794// If this is an input value, the bitcast to the new type is done now. 9795// Bitcast for output value is done at the end of visitInlineAsm(). 9798 !
TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9799// Try to convert to the first EVT that the reg class contains. If the 9800// types are identical size, use a bitcast to convert (e.g. two differing 9801// vector types). Note: output bitcast is done at the end of 9803if (RegVT.
getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9804// Exclude indirect inputs while they are unsupported because the code 9805// to perform the load is missing and thus OpInfo.CallOperand still 9806// refers to the input address rather than the pointed-to value. 9808 OpInfo.CallOperand =
9810 OpInfo.ConstraintVT = RegVT;
9811// If the operand is an FP value and we want it in integer registers, 9812// use the corresponding integer type. This turns an f64 value into 9813// i64, which can be passed with two i32 values on a 32-bit machine. 9814 }
elseif (RegVT.
isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9817 OpInfo.CallOperand =
9819 OpInfo.ConstraintVT = VT;
9824// No need to allocate a matching input constraint since the constraint it's 9825// matching to has already been allocated. 9826if (OpInfo.isMatchingInputConstraint())
9829EVT ValueVT = OpInfo.ConstraintVT;
9830if (OpInfo.ConstraintVT == MVT::Other)
9833// Initialize NumRegs. 9834unsigned NumRegs = 1;
9835if (OpInfo.ConstraintVT != MVT::Other)
9838// If this is a constraint for a specific physical register, like {r17}, 9841// If this associated to a specific register, initialize iterator to correct 9842// place. If virtual, make sure we have enough registers 9844// Initialize iterator if necessary 9848// Do not check for single registers. 9850I = std::find(
I, RC->
end(), AssignedReg);
9851if (
I == RC->
end()) {
9852// RC does not contain the selected register, which indicates a 9853// mismatch between the register and the required type/bitwidth. 9854return {AssignedReg};
9858for (; NumRegs; --NumRegs, ++
I) {
9859assert(
I != RC->
end() &&
"Ran out of registers to allocate!");
9864 OpInfo.AssignedRegs =
RegsForValue(Regs, RegVT, ValueVT);
9870const std::vector<SDValue> &AsmNodeOperands) {
9871// Scan until we find the definition we already emitted of this operand. 9873for (; OperandNo; --OperandNo) {
9874// Advance to the next operand. 9875unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9878 (
F.isRegDefKind() ||
F.isRegDefEarlyClobberKind() ||
F.isMemKind()) &&
9879"Skipped past definitions?");
9880 CurOp +=
F.getNumOperandRegisters() + 1;
9891explicit ExtraFlags(
constCallBase &Call) {
9893if (
IA->hasSideEffects())
9895if (
IA->isAlignStack())
9897if (
Call.isConvergent())
9903// Ideally, we would only check against memory constraints. However, the 9904// meaning of an Other constraint can be target-specific and we can't easily 9905// reason about it. Therefore, be conservative and set MayLoad/MayStore 9906// for Other constraints as well. 9921}
// end anonymous namespace 9925if (
auto *GA = dyn_cast<GlobalAddressSDNode>(
Op)) {
9926auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9928// In normal "call dllimport func" instruction (non-inlineasm) it force 9929// indirect access by specifing call opcode. And usually specially print 9930// asm with indirect symbol (i.g: "*") according to opcode. Inline asm can 9931// not do in this way now. (In fact, this is similar with "Data Access" 9932// action). So here we ignore dllimport function. 9940/// visitInlineAsm - Handle a call to an InlineAsm object. 9941void SelectionDAGBuilder::visitInlineAsm(
constCallBase &Call,
9945 /// ConstraintOperands - Information about all of the constraints. 9952// First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack, 9953// AsmDialect, MayLoad, MayStore). 9954bool HasSideEffect =
IA->hasSideEffects();
9955 ExtraFlags ExtraInfo(Call);
9957for (
auto &
T : TargetConstraints) {
9958 ConstraintOperands.
push_back(SDISelAsmOperandInfo(
T));
9959 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.
back();
9961if (OpInfo.CallOperandVal)
9962 OpInfo.CallOperand =
getValue(OpInfo.CallOperandVal);
9965 HasSideEffect = OpInfo.hasMemory(TLI);
9967// Determine if this InlineAsm MayLoad or MayStore based on the constraints. 9968// FIXME: Could we compute this on OpInfo rather than T? 9970// Compute the constraint code and ConstraintType to use. 9974 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
9975// We've delayed emitting a diagnostic like the "n" constraint because 9976// inlining could cause an integer showing up. 9977return emitInlineAsmError(Call,
"constraint '" +
Twine(
T.ConstraintCode) +
9978"' expects an integer constant " 9981 ExtraInfo.update(
T);
9984// We won't need to flush pending loads if this asm doesn't touch 9985// memory and is nonvolatile. 9988bool EmitEHLabels = isa<InvokeInst>(Call);
9990assert(EHPadBB &&
"InvokeInst must have an EHPadBB");
9992bool IsCallBr = isa<CallBrInst>(Call);
9994if (IsCallBr || EmitEHLabels) {
9995// If this is a callbr or invoke we need to flush pending exports since 9996// inlineasm_br and invoke are terminators. 9997// We need to do this before nodes are glued to the inlineasm_br node. 10003 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
10008IA->collectAsmStrs(AsmStrs);
10010// Second pass over the constraints: compute which constraint option to use. 10011for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10015// If this is an output operand with a matching input operand, look up the 10016// matching input. If their types mismatch, e.g. one is an integer, the 10017// other is floating point, or their sizes are different, flag it as an 10019if (OpInfo.hasMatchingInput()) {
10020 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
10024// Compute the constraint code and ConstraintType to use. 10032// In Linux PIC model, there are 4 cases about value/label addressing: 10034// 1: Function call or Label jmp inside the module. 10035// 2: Data access (such as global variable, static variable) inside module. 10036// 3: Function call or Label jmp outside the module. 10037// 4: Data access (such as global variable) outside the module. 10039// Due to current llvm inline asm architecture designed to not "recognize" 10040// the asm code, there are quite troubles for us to treat mem addressing 10041// differently for same value/adress used in different instuctions. 10042// For example, in pic model, call a func may in plt way or direclty 10043// pc-related, but lea/mov a function adress may use got. 10045// Here we try to "recognize" function call for the case 1 and case 3 in 10046// inline asm. And try to adjust the constraint for them. 10048// TODO: Due to current inline asm didn't encourage to jmp to the outsider 10049// label, so here we don't handle jmp function label now, but we need to 10050// enhance it (especilly in PIC model) if we meet meaningful requirements. 10051if (OpInfo.isIndirect &&
isFunction(OpInfo.CallOperand) &&
10054 OpInfo.isIndirect =
false;
10058// If this is a memory input, and if the operand is not indirect, do what we 10059// need to provide an address for the memory input. 10061 !OpInfo.isIndirect) {
10062assert((OpInfo.isMultipleAlternative ||
10064"Can only indirectify direct input operands!");
10066// Memory operands really want the address of the value. 10069// There is no longer a Value* corresponding to this operand. 10070 OpInfo.CallOperandVal =
nullptr;
10072// It is now an indirect operand. 10073 OpInfo.isIndirect =
true;
10078// AsmNodeOperands - The operands for the ISD::INLINEASM node. 10079 std::vector<SDValue> AsmNodeOperands;
10080 AsmNodeOperands.push_back(
SDValue());
// reserve space for input chain 10084// If we have a !srcloc metadata node associated with it, we want to attach 10085// this to the ultimately generated inline asm machineinstr. To do this, we 10086// pass in the third operand as this (potentially null) inline asm MDNode. 10087constMDNode *SrcLoc =
Call.getMetadata(
"srcloc");
10090// Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore 10091// bits as operand 3. 10095// Third pass: Loop over operands to prepare DAG-level operands.. As part of 10096// this, assign virtual and physical registers for inputs and otput. 10097for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10098// Assign Registers. 10099 SDISelAsmOperandInfo &RefOpInfo =
10100 OpInfo.isMatchingInputConstraint()
10101 ? ConstraintOperands[OpInfo.getMatchedOperand()]
10103constauto RegError =
10109 emitInlineAsmError(Call,
"register '" +
Twine(
RegName) +
10110"' allocated for constraint '" +
10111Twine(OpInfo.ConstraintCode) +
10112"' does not match required type");
10116auto DetectWriteToReservedRegister = [&]() {
10119for (
unsigned Reg : OpInfo.AssignedRegs.Regs) {
10121TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
10123 emitInlineAsmError(Call,
"write to reserved register '" +
10132 !OpInfo.isMatchingInputConstraint())) &&
10133"Only address as input operand is allowed.");
10135switch (OpInfo.Type) {
10141"Failed to convert memory constraint code to constraint id.");
10143// Add information to the INLINEASM node to know about this output. 10145 OpFlags.setMemConstraint(ConstraintID);
10148 AsmNodeOperands.push_back(OpInfo.CallOperand);
10150// Otherwise, this outputs to a register (directly for C_Register / 10151// C_RegisterClass, and a target-defined fashion for 10152// C_Immediate/C_Other). Find a register that we can use. 10153if (OpInfo.AssignedRegs.Regs.empty()) {
10154 emitInlineAsmError(
10155 Call,
"couldn't allocate output register for constraint '" +
10156Twine(OpInfo.ConstraintCode) +
"'");
10160if (DetectWriteToReservedRegister())
10163// Add information to the INLINEASM node to know that this register is 10165 OpInfo.AssignedRegs.AddInlineAsmOperands(
10174SDValue InOperandVal = OpInfo.CallOperand;
10176if (OpInfo.isMatchingInputConstraint()) {
10177// If this is required to match an output register we have already set, 10178// just use its register. 10182if (
Flag.isRegDefKind() ||
Flag.isRegDefEarlyClobberKind()) {
10183if (OpInfo.isIndirect) {
10184// This happens on gcc/testsuite/gcc.dg/pr8788-1.c 10185 emitInlineAsmError(Call,
"inline asm not supported yet: " 10186"don't know how to handle tied " 10187"indirect register inputs");
10195auto *
R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
10197MVT RegVT =
R->getSimpleValueType(0);
10201 :
TRI.getMinimalPhysRegClass(TiedReg);
10202for (
unsigned i = 0, e =
Flag.getNumOperandRegisters(); i != e; ++i)
10208// Use the produced MatchedRegs object to 10209 MatchedRegs.getCopyToRegs(InOperandVal,
DAG, dl, Chain, &Glue, &Call);
10211 OpInfo.getMatchedOperand(), dl,
DAG,
10216assert(
Flag.isMemKind() &&
"Unknown matching constraint!");
10218"Unexpected number of operands");
10219// Add information to the INLINEASM node to know about this input. 10220// See InlineAsm.h isUseOperandTiedToDef. 10221Flag.clearMemConstraint();
10222Flag.setMatchingOp(OpInfo.getMatchedOperand());
10225 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
10229// Treat indirect 'X' constraint as memory. 10236 std::vector<SDValue> Ops;
10241if (isa<ConstantSDNode>(InOperandVal)) {
10242 emitInlineAsmError(Call,
"value out of range for constraint '" +
10243Twine(OpInfo.ConstraintCode) +
"'");
10247 emitInlineAsmError(Call,
10248"invalid operand for inline asm constraint '" +
10249Twine(OpInfo.ConstraintCode) +
"'");
10253// Add information to the INLINEASM node to know about this input. 10262assert((OpInfo.isIndirect ||
10264"Operand must be indirect to be a mem!");
10267"Memory operands expect pointer values");
10272"Failed to convert memory constraint code to constraint id.");
10274// Add information to the INLINEASM node to know about this input. 10276 ResOpType.setMemConstraint(ConstraintID);
10280 AsmNodeOperands.push_back(InOperandVal);
10288"Failed to convert memory constraint code to constraint id.");
10294auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
10301// Add information to the INLINEASM node to know about this input. 10302 ResOpType.setMemConstraint(ConstraintID);
10304 AsmNodeOperands.push_back(
10307 AsmNodeOperands.push_back(AsmOp);
10313 emitInlineAsmError(Call,
"unknown asm constraint '" +
10314Twine(OpInfo.ConstraintCode) +
"'");
10318// TODO: Support this. 10319if (OpInfo.isIndirect) {
10320 emitInlineAsmError(
10321 Call,
"Don't know how to handle indirect register inputs yet " 10322"for constraint '" +
10323Twine(OpInfo.ConstraintCode) +
"'");
10327// Copy the input into the appropriate registers. 10328if (OpInfo.AssignedRegs.Regs.empty()) {
10329 emitInlineAsmError(Call,
10330"couldn't allocate input reg for constraint '" +
10331Twine(OpInfo.ConstraintCode) +
"'");
10335if (DetectWriteToReservedRegister())
10340 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal,
DAG, dl, Chain, &Glue,
10344 0, dl,
DAG, AsmNodeOperands);
10348// Add the clobbered value to the operand list, so that the register 10349// allocator is aware that the physreg got clobbered. 10350if (!OpInfo.AssignedRegs.Regs.empty())
10358// Finish up input operands. Set the input chain and add the flag last. 10360if (Glue.
getNode()) AsmNodeOperands.push_back(Glue);
10364DAG.
getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
10367// Do additional work to generate outputs. 10375if (
StructType *StructResult = dyn_cast<StructType>(CallResultType))
10376 ResultTypes = StructResult->elements();
10377elseif (!CallResultType->
isVoidTy())
10378 ResultTypes =
ArrayRef(CallResultType);
10380auto CurResultType = ResultTypes.
begin();
10381auto handleRegAssign = [&](
SDValueV) {
10382assert(CurResultType != ResultTypes.
end() &&
"Unexpected value");
10383assert((*CurResultType)->isSized() &&
"Unexpected unsized type");
10386// If the type of the inline asm call site return value is different but has 10387// same size as the type of the asm output bitcast it. One example of this 10388// is for vectors with different width / number of elements. This can 10389// happen for register classes that can contain multiple different value 10390// types. The preg or vreg allocated may not have the same VT as was 10393// This can also happen for a return value that disagrees with the register 10394// class it is put in, eg. a double in a general-purpose register on a 10396if (ResultVT !=
V.getValueType() &&
10399elseif (ResultVT !=
V.getValueType() && ResultVT.
isInteger() &&
10400V.getValueType().isInteger()) {
10401// If a result value was tied to an input value, the computed result 10402// may have a wider width than the expected result. Extract the 10403// relevant portion. 10406assert(ResultVT ==
V.getValueType() &&
"Asm result value mismatch!");
10411// Deal with output operands. 10412for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10415// Skip trivial output operands. 10416if (OpInfo.AssignedRegs.Regs.empty())
10419switch (OpInfo.ConstraintType) {
10423 Chain, &Glue, &Call);
10431break;
// Already handled. 10433break;
// Silence warning. 10435assert(
false &&
"Unexpected unknown constraint");
10438// Indirect output manifest as stores. Record output chains. 10439if (OpInfo.isIndirect) {
10440constValue *
Ptr = OpInfo.CallOperandVal;
10441assert(
Ptr &&
"Expected value CallOperandVal for indirect asm operand");
10446// generate CopyFromRegs to associated registers. 10447assert(!
Call.getType()->isVoidTy() &&
"Bad inline asm!");
10450 handleRegAssign(V);
10452 handleRegAssign(Val);
10458if (!ResultValues.
empty()) {
10459assert(CurResultType == ResultTypes.
end() &&
10460"Mismatch in number of ResultTypes");
10462"Mismatch in number of output operands in asm result");
10469// Collect store chains. 10470if (!OutChains.
empty())
10474 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10477// Only Update Root if inline assembly has a memory effect. 10478if (ResultValues.
empty() || HasSideEffect || !OutChains.
empty() || IsCallBr ||
10483void SelectionDAGBuilder::emitInlineAsmError(
constCallBase &Call,
10484constTwine &Message) {
10488// Make sure we leave the DAG in a valid state 10493if (ValueVTs.
empty())
10497for (
constEVT &VT : ValueVTs)
10503void SelectionDAGBuilder::visitVAStart(
constCallInst &
I) {
10510void SelectionDAGBuilder::visitVAArg(
constVAArgInst &
I) {
10516DL.getABITypeAlign(
I.getType()).value());
10519if (
I.getType()->isPointerTy())
10525void SelectionDAGBuilder::visitVAEnd(
constCallInst &
I) {
10532void SelectionDAGBuilder::visitVACopy(
constCallInst &
I) {
10544 std::optional<ConstantRange> CR =
getRange(
I);
10546if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
10549APIntLo = CR->getUnsignedMin();
10550if (!
Lo.isMinValue())
10553APIntHi = CR->getUnsignedMax();
10554unsigned Bits = std::max(
Hi.getActiveBits(),
10563unsigned NumVals =
Op.getNode()->getNumValues();
10570for (
unsignedI = 1;
I != NumVals; ++
I)
10576/// Populate a CallLowerinInfo (into \p CLI) based on the properties of 10577/// the call being lowered. 10579/// This is a helper for lowering intrinsics that follow a target calling 10580/// convention or require stack pointer adjustment. Only a subset of the 10581/// intrinsic's operands need to participate in the calling convention. 10584unsigned ArgIdx,
unsigned NumArgs,
SDValue Callee,
Type *ReturnTy,
10587 Args.reserve(NumArgs);
10589// Populate the argument list. 10590// Attributes for args start at offset 1, after the return attribute. 10591for (
unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10592 ArgI != ArgE; ++ArgI) {
10593constValue *V = Call->getOperand(ArgI);
10595assert(!V->getType()->isEmptyTy() &&
"Empty type passed to intrinsic.");
10599 Entry.Ty = V->getType();
10600 Entry.setAttributes(Call, ArgI);
10601 Args.push_back(Entry);
10606 .
setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10614/// Add a stack map intrinsic call's live variable operands to a stackmap 10615/// or patchpoint target node's operand list. 10617/// Constants are converted to TargetConstants purely as an optimization to 10618/// avoid constant materialization and register allocation. 10620/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not 10621/// generate addess computation nodes, and so FinalizeISel can convert the 10622/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids 10623/// address materialization and register allocation, but may also be required 10624/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an 10625/// alloca in the entry block, then the runtime may assume that the alloca's 10626/// StackMap location can be read immediately after compilation and that the 10627/// location is valid at any point during execution (this is similar to the 10628/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were 10629/// only available in a register, then the runtime would need to trap when 10630/// execution reaches the StackMap in order to read the alloca's location. 10635for (
unsignedI = StartIdx;
I < Call.arg_size();
I++) {
10638// Things on the stack are pointer-typed, meaning that they are already 10639// legal and can be emitted directly to target nodes. 10643// Otherwise emit a target independent node to be legalised. 10649/// Lower llvm.experimental.stackmap. 10650void SelectionDAGBuilder::visitStackmap(
constCallInst &CI) {
10651// void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>, 10652// [live variables...]) 10662// The stackmap intrinsic only records the live variables (the arguments 10663// passed to it) and emits NOPS (if requested). Unlike the patchpoint 10664// intrinsic, this won't be lowered to a function call. This means we don't 10665// have to worry about calling conventions and target specific lowering code. 10666// Instead we perform the call lowering right here. 10668// chain, flag = CALLSEQ_START(chain, 0, 0) 10669// chain, flag = STACKMAP(id, nbytes, ..., chain, flag) 10670// chain, flag = CALLSEQ_END(chain, 0, 0, flag) 10675// Add the STACKMAP operands, starting with DAG house-keeping. 10679// Add the <id>, <numShadowBytes> operands. 10681// These do not require legalisation, and can be emitted directly to target 10684assert(
ID.getValueType() == MVT::i64);
10695// Add the live variables. 10698// Create the STACKMAP node. 10705// Stackmaps don't generate values, so nothing goes into the NodeMap. 10707// Set the root to the target-lowered call chain. 10710// Inform the Frame Information that we have a stackmap in this function. 10714/// Lower llvm.experimental.patchpoint directly to its target opcode. 10715void SelectionDAGBuilder::visitPatchpoint(
constCallBase &CB,
10717// <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>, 10722// [live variables...]) 10730// Handle immediate and symbolic callees. 10731if (
auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10734elseif (
auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10736SDLoc(SymbolicCallee),
10737 SymbolicCallee->getValueType(0));
10739// Get the real number of arguments participating in the call <numArgs> 10743// Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> 10744// Intrinsics include all meta-operands up to but not including CC. 10747"Not enough arguments provided to the patchpoint intrinsic");
10749// For AnyRegCC the arguments are lowered later on manually. 10750unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10765 /// Get a call instruction from the call sequence chain. 10766 /// Tail calls are not allowed. 10768"Expected a callseq node.");
10770bool HasGlue =
Call->getGluedNode();
10772// Replace the target specific call node with the patchable intrinsic. 10778// Optionally, push the glue (if any). 10782// Push the register mask info. 10788// Add the <id> and <numBytes> constants. 10797// Adjust <numArgs> to account for any arguments that have been passed on the 10799// Call Node: Chain, Target, {Args}, RegMask, [Glue] 10800unsigned NumCallRegArgs =
Call->getNumOperands() - (HasGlue ? 4 : 3);
10801 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10804// Add the calling convention 10807// Add the arguments we omitted previously. The register allocator should 10808// place these in any free register. 10810for (
unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i !=
e; ++i)
10813// Push the arguments from the call instruction. 10817// Push live variables for the stack map. 10821if (IsAnyRegCC && HasDef) {
10822// Create the return types based on the intrinsic definition 10826assert(ValueVTs.
size() == 1 &&
"Expected only one return value type.");
10828// There is always a chain and a glue type at the end 10835// Replace the target specific call node with a PATCHPOINT node. 10838// Update the NodeMap. 10846// Fixup the consumers of the intrinsic. The chain and glue may be used in the 10847// call sequence. Furthermore the location of the chain and glue can change 10848// when the AnyReg calling convention is used and the intrinsic returns a 10850if (IsAnyRegCC && HasDef) {
10858// Inform the Frame Information that we have a patchpoint in this function. 10862void SelectionDAGBuilder::visitVectorReduce(
constCallInst &
I,
10863unsigned Intrinsic) {
10867if (
I.arg_size() > 1)
10873if (
auto *FPMO = dyn_cast<FPMathOperator>(&
I))
10876switch (Intrinsic) {
10877case Intrinsic::vector_reduce_fadd:
10885case Intrinsic::vector_reduce_fmul:
10893case Intrinsic::vector_reduce_add:
10896case Intrinsic::vector_reduce_mul:
10899case Intrinsic::vector_reduce_and:
10902case Intrinsic::vector_reduce_or:
10905case Intrinsic::vector_reduce_xor:
10908case Intrinsic::vector_reduce_smax:
10911case Intrinsic::vector_reduce_smin:
10914case Intrinsic::vector_reduce_umax:
10917case Intrinsic::vector_reduce_umin:
10920case Intrinsic::vector_reduce_fmax:
10923case Intrinsic::vector_reduce_fmin:
10926case Intrinsic::vector_reduce_fmaximum:
10929case Intrinsic::vector_reduce_fminimum:
10938/// Returns an AttributeList representing the attributes applied to the return 10939/// value of the given call. 10943 Attrs.push_back(Attribute::SExt);
10945 Attrs.push_back(Attribute::ZExt);
10947 Attrs.push_back(Attribute::InReg);
10953/// TargetLowering::LowerCallTo - This is the default LowerCallTo 10954/// implementation, which just calls LowerCall. 10955/// FIXME: When all targets are 10956/// migrated to using LowerCall, this hook should be integrated into SDISel. 10957std::pair<SDValue, SDValue>
10959// Handle the incoming return values from the call. 10967// If we are lowering a libcall after legalization, split the return type. 10970 RetTys.
swap(OldRetTys);
10971 Offsets.swap(OldOffsets);
10973for (
size_t i = 0, e = OldRetTys.
size(); i != e; ++i) {
10974EVT RetVT = OldRetTys[i];
10979 RetTys.
append(NumRegs, RegisterVT);
10980for (
unsigned j = 0; j != NumRegs; ++j)
10993int DemoteStackIdx = -100;
10995// FIXME: equivalent assert? 10996// assert(!CS.hasInAllocaArgument() && 10997// "sret demotion is incompatible with inalloca"); 11003Type *StackSlotPtrType =
11008 Entry.Node = DemoteStackSlot;
11009 Entry.Ty = StackSlotPtrType;
11010 Entry.IsSExt =
false;
11011 Entry.IsZExt =
false;
11012 Entry.IsInReg =
false;
11013 Entry.IsSRet =
true;
11014 Entry.IsNest =
false;
11015 Entry.IsByVal =
false;
11016 Entry.IsByRef =
false;
11017 Entry.IsReturned =
false;
11018 Entry.IsSwiftSelf =
false;
11019 Entry.IsSwiftAsync =
false;
11020 Entry.IsSwiftError =
false;
11021 Entry.IsCFGuardTarget =
false;
11022 Entry.Alignment = Alignment;
11028// sret demotion isn't compatible with tail-calls, since the sret argument 11029// points into the callers stack frame. 11034for (
unsignedI = 0, E = RetTys.
size();
I != E; ++
I) {
11036if (NeedsRegBlock) {
11037 Flags.setInConsecutiveRegs();
11038if (
I == RetTys.
size() - 1)
11039 Flags.setInConsecutiveRegsLast();
11046for (
unsigned i = 0; i != NumRegs; ++i) {
11048 MyFlags.
Flags = Flags;
11049 MyFlags.
VT = RegisterVT;
11050 MyFlags.
ArgVT = VT;
11055 cast<PointerType>(CLI.
RetTy)->getAddressSpace());
11063 CLI.
Ins.push_back(MyFlags);
11068// We push in swifterror return as the last element of CLI.Ins. 11077 CLI.
Ins.push_back(MyFlags);
11082// Handle all of the outgoing arguments. 11085for (
unsigned i = 0, e = Args.size(); i != e; ++i) {
11088// FIXME: Split arguments if CLI.IsPostTypeLegalization 11089Type *FinalType = Args[i].Ty;
11090if (Args[i].IsByVal)
11091 FinalType = Args[i].IndirectType;
11094for (
unsignedValue = 0, NumValues = ValueVTs.
size();
Value != NumValues;
11099 Args[i].Node.getResNo() +
Value);
11102// Certain targets (such as MIPS), may have a different ABI alignment 11103// for a type depending on the context. Give the target a chance to 11104// specify the alignment it wants. 11106 Flags.setOrigAlign(OriginalAlignment);
11108if (Args[i].Ty->isPointerTy()) {
11109 Flags.setPointer();
11110 Flags.setPointerAddrSpace(
11117if (Args[i].IsNoExt)
11119if (Args[i].IsInReg) {
11120// If we are using vectorcall calling convention, a structure that is 11121// passed InReg - is surely an HVA 11123 isa<StructType>(FinalType)) {
11124// The first value of a structure is marked 11126 Flags.setHvaStart();
11134if (Args[i].IsSwiftSelf)
11135 Flags.setSwiftSelf();
11136if (Args[i].IsSwiftAsync)
11137 Flags.setSwiftAsync();
11138if (Args[i].IsSwiftError)
11139 Flags.setSwiftError();
11140if (Args[i].IsCFGuardTarget)
11141 Flags.setCFGuardTarget();
11142if (Args[i].IsByVal)
11144if (Args[i].IsByRef)
11146if (Args[i].IsPreallocated) {
11147 Flags.setPreallocated();
11148// Set the byval flag for CCAssignFn callbacks that don't know about 11149// preallocated. This way we can know how many bytes we should've 11150// allocated and how many bytes a callee cleanup function will pop. If 11151// we port preallocated to more targets, we'll have to add custom 11152// preallocated handling in the various CC lowering callbacks. 11155if (Args[i].IsInAlloca) {
11156 Flags.setInAlloca();
11157// Set the byval flag for CCAssignFn callbacks that don't know about 11158// inalloca. This way we can know how many bytes we should've allocated 11159// and how many bytes a callee cleanup function will pop. If we port 11160// inalloca to more targets, we'll have to add custom inalloca handling 11161// in the various CC lowering callbacks. 11165if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11166unsigned FrameSize =
DL.getTypeAllocSize(Args[i].IndirectType);
11167 Flags.setByValSize(FrameSize);
11169// info is not there but there are cases it cannot get right. 11170if (
auto MA = Args[i].Alignment)
11174 }
elseif (
auto MA = Args[i].Alignment) {
11177 MemAlign = OriginalAlignment;
11179 Flags.setMemAlign(MemAlign);
11183 Flags.setInConsecutiveRegs();
11194elseif (Args[i].IsZExt)
11197// Conservatively only handle 'returned' on non-vectors that can be lowered, 11199if (Args[i].IsReturned && !
Op.getValueType().isVector() &&
11204 Args[i].Ty->getPointerAddressSpace())) &&
11205 RetTys.
size() == NumValues &&
"unexpected use of 'returned'");
11206// Before passing 'returned' to the target lowering code, ensure that 11207// either the register MVT and the actual EVT are the same size or that 11208// the return value and argument are extended in the same way; in these 11209// cases it's safe to pass the argument register value unchanged as the 11210// return register value (although it's at the target's option whether 11212// TODO: allow code generation to take advantage of partially preserved 11213// registers rather than clobbering the entire register when the 11214// parameter extension method is not compatible with the return 11218 CLI.
RetZExt == Args[i].IsZExt))
11219 Flags.setReturned();
11225for (
unsigned j = 0; j != NumParts; ++j) {
11226// if it isn't first piece, alignment must be 1 11227// For scalable vectors the scalable part is currently handled 11228// by individual targets, so we just use the known minimum size here. 11232 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11233if (NumParts > 1 && j == 0)
11237if (j == NumParts - 1)
11241 CLI.
Outs.push_back(MyFlags);
11242 CLI.
OutVals.push_back(Parts[j]);
11245if (NeedsRegBlock &&
Value == NumValues - 1)
11246 CLI.
Outs[CLI.
Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11253// Update CLI.InVals to use outside of this function. 11256// Verify that the target's LowerCall behaved as expected. 11258"LowerCall didn't return a valid chain!");
11260"LowerCall emitted a return value for a tail call!");
11262"LowerCall didn't emit the correct number of values!");
11264// For a tail call, the return value is merely live-out and there aren't 11265// any nodes in the DAG representing it. Return a special value to 11266// indicate that a tail call has been emitted and no more Instructions 11267// should be processed in the current block. 11274for (
unsigned i = 0, e = CLI.
Ins.size(); i != e; ++i) {
11275assert(InVals[i].
getNode() &&
"LowerCall emitted a null value!");
11276assert(
EVT(CLI.
Ins[i].VT) == InVals[i].getValueType() &&
11277"LowerCall emitted a value with the wrong type!");
11283// The instruction result is the result of loading from the 11284// hidden sret parameter. 11287unsigned NumValues = RetTys.
size();
11288 ReturnValues.
resize(NumValues);
11291// An aggregate return value cannot wrap around the address space, so 11292// offsets to its parts don't wrap either. 11295for (
unsigned i = 0; i < NumValues; ++i) {
11303 DemoteStackIdx, Offsets[i]),
11305 ReturnValues[i] = L;
11306 Chains[i] = L.getValue(1);
11311// Collect the legal value parts into potentially illegal values 11312// that correspond to the original function's return values. 11313 std::optional<ISD::NodeType> AssertOp;
11318unsigned CurReg = 0;
11319for (
EVT VT : RetTys) {
11326 CLI.
DAG, CLI.
DL, &InVals[CurReg], NumRegs, RegisterVT, VT,
nullptr,
11331// For a function returning void, there is no return value. We can't create 11332// such a node, so we just return a null return value in that case. In 11333// that case, nothing will actually look at the value. 11334if (ReturnValues.
empty())
11340return std::make_pair(Res, CLI.
Chain);
11343/// Places new result values for the node in Results (their number 11344/// and types must exactly match those of the original return values of 11345/// the node), or leaves Results empty, which indicates that the node is not 11346/// to be custom lowered after all. 11355// If the original node has one result, take the return value from 11356// LowerOperation as is. It might not be result number 0. 11357if (
N->getNumValues() == 1) {
11362// If the original node has multiple results, then the return node should 11363// have the same number of results. 11365"Lowering returned the wrong number of results!");
11367// Places new result values base on N result number. 11368for (
unsignedI = 0, E =
N->getNumValues();
I != E; ++
I)
11381 cast<RegisterSDNode>(
Op.getOperand(1))->getReg() != Reg) &&
11382"Copy from a reg to the same reg!");
11386// If this is an InlineAsm we have to match the registers required, not the 11387// notional registers required by the type. 11390 std::nullopt);
// This is not an ABI copy. 11396 ExtendType = PreferredExtendIt->second;
11399 PendingExports.push_back(Chain);
11404/// isOnlyUsedInEntryBlock - If the specified argument is only used in the 11405/// entry block, return true. This includes arguments used by switches, since 11406/// the switch may expand into multiple basic blocks. 11408// With FastISel active, we may be splitting blocks, so force creation 11409// of virtual registers for all non-dead arguments. 11411returnA->use_empty();
11414for (
constUser *U :
A->users())
11415if (cast<Instruction>(U)->
getParent() != &Entry || isa<SwitchInst>(U))
11416returnfalse;
// Use not in entry block. 11423 std::pair<const AllocaInst *, const StoreInst *>>;
11425/// Scan the entry block of the function in FuncInfo for arguments that look 11426/// like copies into a local alloca. Record any copied arguments in 11427/// ArgCopyElisionCandidates. 11432// Record the state of every static alloca used in the entry block. Argument 11433// allocas are all used in the entry block, so we need approximately as many 11434// entries as we have arguments. 11435enum StaticAllocaInfo {
Unknown, Clobbered, Elidable };
11438 StaticAllocas.
reserve(NumArgs * 2);
11440auto GetInfoIfStaticAlloca = [&](
constValue *V) -> StaticAllocaInfo * {
11443 V = V->stripPointerCasts();
11444constauto *AI = dyn_cast<AllocaInst>(V);
11445if (!AI || !AI->isStaticAlloca() || !FuncInfo->
StaticAllocaMap.count(AI))
11448return &Iter.first->second;
11451// Look for stores of arguments to static allocas. Look through bitcasts and 11452// GEPs to handle type coercions, as long as the alloca is fully initialized 11453// by the store. Any non-store use of an alloca escapes it and any subsequent 11454// unanalyzed store might write it. 11455// FIXME: Handle structs initialized with multiple stores. 11457// Look for stores, and handle non-store uses conservatively. 11458constauto *SI = dyn_cast<StoreInst>(&
I);
11460// We will look through cast uses, so ignore them completely. 11463// Ignore debug info and pseudo op intrinsics, they don't escape or store 11465if (
I.isDebugOrPseudoInst())
11467// This is an unknown instruction. Assume it escapes or writes to all 11468// static alloca operands. 11469for (
constUse &U :
I.operands()) {
11470if (StaticAllocaInfo *
Info = GetInfoIfStaticAlloca(U))
11471 *
Info = StaticAllocaInfo::Clobbered;
11476// If the stored value is a static alloca, mark it as escaped. 11477if (StaticAllocaInfo *
Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11478 *
Info = StaticAllocaInfo::Clobbered;
11480// Check if the destination is a static alloca. 11481constValue *Dst = SI->getPointerOperand()->stripPointerCasts();
11482 StaticAllocaInfo *
Info = GetInfoIfStaticAlloca(Dst);
11487// Skip allocas that have been initialized or clobbered. 11488if (*
Info != StaticAllocaInfo::Unknown)
11491// Check if the stored value is an argument, and that this store fully 11492// initializes the alloca. 11493// If the argument type has padding bits we can't directly forward a pointer 11494// as the upper bits may contain garbage. 11495// Don't elide copies from the same argument twice. 11496constValue *Val = SI->getValueOperand()->stripPointerCasts();
11497constauto *Arg = dyn_cast<Argument>(Val);
11498if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11499 Arg->getType()->isEmptyTy() ||
11500DL.getTypeStoreSize(Arg->getType()) !=
11502 !
DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11503 ArgCopyElisionCandidates.
count(Arg)) {
11504 *
Info = StaticAllocaInfo::Clobbered;
11508LLVM_DEBUG(
dbgs() <<
"Found argument copy elision candidate: " << *AI
11511// Mark this alloca and store for argument copy elision. 11512 *
Info = StaticAllocaInfo::Elidable;
11513 ArgCopyElisionCandidates.
insert({Arg, {AI, SI}});
11515// Stop scanning if we've seen all arguments. This will happen early in -O0 11516// builds, which is useful, because -O0 builds have large entry blocks and 11518if (ArgCopyElisionCandidates.
size() == NumArgs)
11523/// Try to elide argument copies from memory into a local alloca. Succeeds if 11524/// ArgVal is a load from a suitable fixed stack object. 11531// Check if this is a load from a fixed stack object. 11532auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11535auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11539// Check that the fixed stack object is the right size and alignment. 11540// Look at the alignment that the user wrote on the alloca instead of looking 11541// at the stack object. 11542auto ArgCopyIter = ArgCopyElisionCandidates.
find(&Arg);
11543assert(ArgCopyIter != ArgCopyElisionCandidates.
end());
11544constAllocaInst *AI = ArgCopyIter->second.first;
11545int FixedIndex = FINode->getIndex();
11547int OldIndex = AllocaIndex;
11551dbgs() <<
" argument copy elision failed due to bad fixed stack " 11557LLVM_DEBUG(
dbgs() <<
" argument copy elision failed: alignment of alloca " 11558"greater than stack argument alignment (" 11559 <<
DebugStr(RequiredAlignment) <<
" vs " 11564// Perform the elision. Delete the old stack object and replace its only use 11565// in the variable info map. Mark the stack object as mutable and aliased. 11567dbgs() <<
"Eliding argument copy from " << Arg <<
" to " << *AI <<
'\n' 11568 <<
" Replacing frame index " << OldIndex <<
" with " << FixedIndex
11574 AllocaIndex = FixedIndex;
11575 ArgCopyElisionFrameIndexMap.
insert({OldIndex, FixedIndex});
11576for (
SDValue ArgVal : ArgVals)
11579// Avoid emitting code for the store implementing the copy. 11580constStoreInst *SI = ArgCopyIter->second.second;
11581 ElidedArgCopyInstrs.
insert(SI);
11583// Check for uses of the argument again so that we can avoid exporting ArgVal 11584// if it is't used by anything other than the store. 11593void SelectionDAGISel::LowerArguments(
constFunction &
F) {
11599// In Naked functions we aren't going to save any registers. 11600if (
F.hasFnAttribute(Attribute::Naked))
11604// Put in an sret pointer parameter before all the other parameters. 11612Ins.push_back(RetArg);
11615// Look for stores of arguments to static allocas. Mark such arguments with a 11616// flag to ask the target to give us the memory location of that argument if 11620 ArgCopyElisionCandidates);
11622// Set up the incoming argument description vector. 11624unsigned ArgNo = Arg.getArgNo();
11627bool isArgValueUsed = !Arg.use_empty();
11628unsigned PartBase = 0;
11629Type *FinalType = Arg.getType();
11630if (Arg.hasAttribute(Attribute::ByVal))
11631 FinalType = Arg.getParamByValType();
11633 FinalType,
F.getCallingConv(),
F.isVarArg(),
DL);
11634for (
unsignedValue = 0, NumValues = ValueVTs.
size();
11641if (Arg.getType()->isPointerTy()) {
11643Flags.setPointerAddrSpace(
11644 cast<PointerType>(Arg.getType())->getAddressSpace());
11646if (Arg.hasAttribute(Attribute::ZExt))
11648if (Arg.hasAttribute(Attribute::SExt))
11650if (Arg.hasAttribute(Attribute::InReg)) {
11651// If we are using vectorcall calling convention, a structure that is 11652// passed InReg - is surely an HVA 11654 isa<StructType>(Arg.getType())) {
11655// The first value of a structure is marked 11657Flags.setHvaStart();
11663if (Arg.hasAttribute(Attribute::StructRet))
11665if (Arg.hasAttribute(Attribute::SwiftSelf))
11666Flags.setSwiftSelf();
11667if (Arg.hasAttribute(Attribute::SwiftAsync))
11668Flags.setSwiftAsync();
11669if (Arg.hasAttribute(Attribute::SwiftError))
11670Flags.setSwiftError();
11671if (Arg.hasAttribute(Attribute::ByVal))
11673if (Arg.hasAttribute(Attribute::ByRef))
11675if (Arg.hasAttribute(Attribute::InAlloca)) {
11676Flags.setInAlloca();
11677// Set the byval flag for CCAssignFn callbacks that don't know about 11678// inalloca. This way we can know how many bytes we should've allocated 11679// and how many bytes a callee cleanup function will pop. If we port 11680// inalloca to more targets, we'll have to add custom inalloca handling 11681// in the various CC lowering callbacks. 11684if (Arg.hasAttribute(Attribute::Preallocated)) {
11685Flags.setPreallocated();
11686// Set the byval flag for CCAssignFn callbacks that don't know about 11687// preallocated. This way we can know how many bytes we should've 11688// allocated and how many bytes a callee cleanup function will pop. If 11689// we port preallocated to more targets, we'll have to add custom 11690// preallocated handling in the various CC lowering callbacks. 11694// Certain targets (such as MIPS), may have a different ABI alignment 11695// for a type depending on the context. Give the target a chance to 11696// specify the alignment it wants. 11697constAlign OriginalAlignment(
11699Flags.setOrigAlign(OriginalAlignment);
11702Type *ArgMemTy =
nullptr;
11706 ArgMemTy = Arg.getPointeeInMemoryValueType();
11708uint64_t MemSize =
DL.getTypeAllocSize(ArgMemTy);
11710// For in-memory arguments, size and alignment should be passed from FE. 11711// BE will guess if this info is not there but there are cases it cannot 11713if (
auto ParamAlign = Arg.getParamStackAlign())
11714 MemAlign = *ParamAlign;
11715elseif ((ParamAlign = Arg.getParamAlign()))
11716 MemAlign = *ParamAlign;
11719if (
Flags.isByRef())
11720Flags.setByRefSize(MemSize);
11722Flags.setByValSize(MemSize);
11723 }
elseif (
auto ParamAlign = Arg.getParamStackAlign()) {
11724 MemAlign = *ParamAlign;
11726 MemAlign = OriginalAlignment;
11728Flags.setMemAlign(MemAlign);
11730if (Arg.hasAttribute(Attribute::Nest))
11733Flags.setInConsecutiveRegs();
11734if (ArgCopyElisionCandidates.
count(&Arg))
11735Flags.setCopyElisionCandidate();
11736if (Arg.hasAttribute(Attribute::Returned))
11737Flags.setReturned();
11743for (
unsigned i = 0; i != NumRegs; ++i) {
11744// For scalable vectors, use the minimum size; individual targets 11745// are responsible for handling scalable vector arguments and 11748 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11750if (NumRegs > 1 && i == 0)
11751 MyFlags.Flags.setSplit();
11752// if it isn't first piece, alignment must be 1 11754 MyFlags.Flags.setOrigAlign(
Align(1));
11755if (i == NumRegs - 1)
11756 MyFlags.Flags.setSplitEnd();
11758Ins.push_back(MyFlags);
11760if (NeedsRegBlock &&
Value == NumValues - 1)
11761Ins[
Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11766// Call the target to set up the argument values. 11769 DAG.
getRoot(),
F.getCallingConv(),
F.isVarArg(), Ins, dl, DAG, InVals);
11771// Verify that the target's LowerFormalArguments behaved as expected. 11773"LowerFormalArguments didn't return a valid chain!");
11775"LowerFormalArguments didn't emit the correct number of values!");
11777for (
unsigned i = 0, e =
Ins.size(); i != e; ++i) {
11778 assert(InVals[i].getNode() &&
11779"LowerFormalArguments emitted a null value!");
11780 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11781"LowerFormalArguments emitted a value with the wrong type!");
11785// Update the DAG with the new chain value resulting from argument lowering. 11788// Set up the argument values. 11791// Create a virtual register for the sret pointer, and put in a copy 11792// from the sret argument into it. 11795 std::optional<ISD::NodeType> AssertOp;
11798F.getCallingConv(), AssertOp);
11804FuncInfo->DemoteRegister = SRetReg;
11806SDB->DAG.getCopyToReg(NewRoot,
SDB->getCurSDLoc(), SRetReg, ArgValue);
11809// i indexes lowered arguments. Bump it past the hidden sret argument. 11819unsigned NumValues = ValueVTs.
size();
11823bool ArgHasUses = !Arg.use_empty();
11825// Elide the copying store if the target loaded this argument from a 11826// suitable fixed stack object. 11827if (Ins[i].
Flags.isCopyElisionCandidate()) {
11828unsigned NumParts = 0;
11829for (
EVT VT : ValueVTs)
11831F.getCallingConv(), VT);
11835ArrayRef(&InVals[i], NumParts), ArgHasUses);
11838// If this argument is unused then remember its value. It is used to generate 11839// debugging information. 11840bool isSwiftErrorArg =
11842 Arg.hasAttribute(Attribute::SwiftError);
11843if (!ArgHasUses && !isSwiftErrorArg) {
11844SDB->setUnusedArgValue(&Arg, InVals[i]);
11846// Also remember any frame index for use in FastISel. 11848 dyn_cast<FrameIndexSDNode>(InVals[i].
getNode()))
11849FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11852for (
unsigned Val = 0; Val != NumValues; ++Val) {
11853EVT VT = ValueVTs[Val];
11855F.getCallingConv(), VT);
11859// Even an apparent 'unused' swifterror argument needs to be returned. So 11860// we do generate a copy for it that can be used on return from the 11862if (ArgHasUses || isSwiftErrorArg) {
11863 std::optional<ISD::NodeType> AssertOp;
11864if (Arg.hasAttribute(Attribute::SExt))
11866elseif (Arg.hasAttribute(Attribute::ZExt))
11870 PartVT, VT,
nullptr, NewRoot,
11871F.getCallingConv(), AssertOp));
11877// We don't need to do anything else for unused arguments. 11878if (ArgValues.
empty())
11881// Note down frame index. 11883 dyn_cast<FrameIndexSDNode>(ArgValues[0].
getNode()))
11884FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11887SDB->getCurSDLoc());
11889SDB->setValue(&Arg, Res);
11891// We want to associate the argument with the frame index, among 11892// involved operands, that correspond to the lowest address. The 11893// getCopyFromParts function, called earlier, is swapping the order of 11894// the operands to BUILD_PAIR depending on endianness. The result of 11895// that swapping is that the least significant bits of the argument will 11896// be in the first operand of the BUILD_PAIR node, and the most 11897// significant bits will be in the second operand. 11902 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11903FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11906// Analyses past this point are naive and don't expect an assertion. 11910// Update the SwiftErrorVRegDefMap. 11913if (
Reg.isVirtual())
11918// If this argument is live outside of the entry block, insert a copy from 11919// wherever we got it to the vreg that other BB's will reference it as. 11921// If we can, though, try to skip creating an unnecessary vreg. 11922// FIXME: This isn't very clean... it would be nice to make this more 11925if (
Reg.isVirtual()) {
11931FuncInfo->InitializeRegForValue(&Arg);
11932SDB->CopyToExportRegsIfNeeded(&Arg);
11936if (!Chains.
empty()) {
11943assert(i == InVals.
size() &&
"Argument register count mismatch!");
11945// If any argument copy elisions occurred and we have debug info, update the 11946// stale frame indices used in the dbg.declare variable info table. 11947if (!ArgCopyElisionFrameIndexMap.
empty()) {
11950autoI = ArgCopyElisionFrameIndexMap.
find(
VI.getStackSlot());
11951if (
I != ArgCopyElisionFrameIndexMap.
end())
11952VI.updateStackSlot(
I->second);
11956// Finally, if the target has anything special to do, allow it to do so. 11960/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to 11961/// ensure constants are generated when needed. Remember the virtual registers 11962/// that need to be added to the Machine PHI nodes as input. We cannot just 11963/// directly add them, because expansion might result in multiple MBB's for one 11964/// BB. As such, the start of the BB might correspond to a different MBB than 11967SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(
constBasicBlock *LLVMBB) {
11972// Check PHI nodes in successors that expect a value to be available from this 11975if (!isa<PHINode>(SuccBB->begin()))
continue;
11978// If this terminator has multiple identical successors (common for 11979// switches), only handle each succ once. 11980if (!SuccsHandled.
insert(SuccMBB).second)
11985// At this point we know that there is a 1-1 correspondence between LLVM PHI 11986// nodes and Machine PHI nodes, but the incoming operands have not been 11988for (
constPHINode &PN : SuccBB->phis()) {
11989// Ignore dead phi's. 11994if (PN.getType()->isEmptyTy())
11998constValue *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12000if (
constauto *
C = dyn_cast<Constant>(PHIOp)) {
12004// We need to zero/sign extend ConstantInt phi operands to match 12005// assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo. 12007if (
auto *CI = dyn_cast<ConstantInt>(
C))
12019assert(isa<AllocaInst>(PHIOp) &&
12021"Didn't codegen value into a register!??");
12027// Remember that this register needs to added to the machine PHI node as 12028// the input for this MBB. 12031for (
EVT VT : ValueVTs) {
12033for (
unsigned i = 0; i != NumRegisters; ++i)
12035 std::make_pair(&*
MBBI++, Reg + i));
12036Reg += NumRegisters;
12051/// During lowering new call nodes can be created (such as memset, etc.). 12052/// Those will become new roots of the current DAG, but complications arise 12053/// when they are tail calls. In such cases, the call lowering will update 12054/// the root, but the builder still needs to know that a tail call has been 12055/// lowered in order to avoid generating an additional return. 12056void SelectionDAGBuilder::updateDAGForMaybeTailCall(
SDValue MaybeTC) {
12057// If the node is null, we do have a tail call. 12058if (MaybeTC.
getNode() !=
nullptr)
12073unsignedSize =
W.LastCluster -
W.FirstCluster + 1;
12077if (
Size == 2 &&
W.MBB == SwitchMBB) {
12078// If any two of the cases has the same destination, and if one value 12079// is the same as the other, but has one bit unset that the other has set, 12080// use bit manipulation to do two compares at once. For example: 12081// "if (X == 6 || X == 4)" -> "if ((X|2) == 6)" 12082// TODO: This could be extended to merge any 2 cases in switches with 3 12084// TODO: Handle cases where W.CaseBB != SwitchBB. 12090constAPInt &SmallValue =
Small.Low->getValue();
12091constAPInt &BigValue =
Big.Low->getValue();
12093// Check that there is only one bit different. 12094APInt CommonBit = BigValue ^ SmallValue;
12106// Update successor info. 12107// Both Small and Big will jump to Small.BB, so we sum up the 12109 addSuccessorWithProb(SwitchMBB,
Small.MBB,
Small.Prob +
Big.Prob);
12111 addSuccessorWithProb(
12112 SwitchMBB, DefaultMBB,
12113// The default destination is the first successor in IR. 12116 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12118// Insert the true branch. 12122// Insert the false branch. 12133// Here, we order cases by probability so the most likely case will be 12134// checked first. However, two clusters can have the same probability in 12135// which case their relative ordering is non-deterministic. So we use Low 12136// as a tie-breaker as clusters are guaranteed to never overlap. 12139 return a.Prob != b.Prob ?
12141 a.Low->getValue().slt(b.Low->getValue());
12144// Rearrange the case blocks so that the last one falls through if possible 12145// without changing the order of probabilities. 12148if (
I->Prob >
W.LastCluster->Prob)
12150if (
I->Kind ==
CC_Range &&
I->MBB == NextMBB) {
12157// Compute total probability. 12161 UnhandledProbs +=
I->Prob;
12165bool FallthroughUnreachable =
false;
12167if (
I ==
W.LastCluster) {
12168// For the last cluster, fall through to the default destination. 12169 Fallthrough = DefaultMBB;
12170 FallthroughUnreachable = isa<UnreachableInst>(
12174 CurMF->
insert(BBI, Fallthrough);
12175// Put Cond in a virtual register to make it available from the new blocks. 12178 UnhandledProbs -=
I->Prob;
12182// FIXME: Optimize away range check based on pivot comparisons. 12186// The jump block hasn't been inserted yet; insert it here. 12188 CurMF->
insert(BBI, JumpMBB);
12190auto JumpProb =
I->Prob;
12191auto FallthroughProb = UnhandledProbs;
12193// If the default statement is a target of the jump table, we evenly 12194// distribute the default probability to successors of CurMBB. Also 12195// update the probability on the edge from JumpMBB to Fallthrough. 12199if (*SI == DefaultMBB) {
12200 JumpProb += DefaultProb / 2;
12201 FallthroughProb -= DefaultProb / 2;
12208// If the default clause is unreachable, propagate that knowledge into 12209// JTH->FallthroughUnreachable which will use it to suppress the range 12212// However, don't do this if we're doing branch target enforcement, 12213// because a table branch _without_ a range check can be a tempting JOP 12214// gadget - out-of-bounds inputs that are impossible in correct 12215// execution become possible again if an attacker can influence the 12216// control flow. So if an attacker doesn't already have a BTI bypass 12217// available, we don't want them to be able to get one out of this 12219if (FallthroughUnreachable) {
12226 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12227 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12230// The jump table header will be inserted in our current block, do the 12231// range check, and fall through to our fallthrough block. 12233JT->Default = Fallthrough;
// FIXME: Move Default to JumpTableHeader. 12235// If we're in the right place, emit the jump table header right now. 12236if (CurMBB == SwitchMBB) {
12243// FIXME: Optimize away range check based on pivot comparisons. 12246// The bit test blocks haven't been inserted yet; insert them here. 12250// Fill in fields of the BitTestBlock. 12255// If the cases in bit test don't form a contiguous range, we evenly 12256// distribute the probability on the edge to Fallthrough to two 12257// successors of CurMBB. 12259 BTB->
Prob += DefaultProb / 2;
12263if (FallthroughUnreachable)
12266// If we're in the right place, emit the bit test header right now. 12267if (CurMBB == SwitchMBB) {
12276if (
I->Low ==
I->High) {
12277// Check Cond == I->Low. 12283// Check I->Low <= Cond <= I->High. 12290// If Fallthrough is unreachable, fold away the comparison. 12291if (FallthroughUnreachable)
12294// The false probability is the sum of all unhandled cases. 12295CaseBlock CB(
CC, LHS, RHS, MHS,
I->MBB, Fallthrough, CurMBB,
12298if (CurMBB == SwitchMBB)
12301SL->SwitchCases.push_back(CB);
12306 CurMBB = Fallthrough;
12310void SelectionDAGBuilder::splitWorkItem(
SwitchWorkList &WorkList,
12314assert(
W.FirstCluster->Low->getValue().slt(
W.LastCluster->Low->getValue()) &&
12315"Clusters not sorted?");
12316assert(
W.LastCluster -
W.FirstCluster + 1 >= 2 &&
"Too small to split!");
12318auto [LastLeft, FirstRight, LeftProb, RightProb] =
12319SL->computeSplitWorkItemInfo(W);
12321// Use the first element on the right as pivot since we will make less-than 12322// comparisons against it. 12324assert(PivotCluster >
W.FirstCluster);
12325assert(PivotCluster <=
W.LastCluster);
12332// New blocks will be inserted immediately after the current one. 12336// We will branch to the LHS if Value < Pivot. If LHS is a single cluster, 12337// we can branch to its destination directly if it's squeezed exactly in 12338// between the known lower bound and Pivot - 1. 12340if (FirstLeft == LastLeft && FirstLeft->Kind ==
CC_Range &&
12341 FirstLeft->Low ==
W.GE &&
12342 (FirstLeft->High->getValue() + 1LL) == Pivot->
getValue()) {
12343 LeftMBB = FirstLeft->MBB;
12348 {LeftMBB, FirstLeft, LastLeft,
W.GE, Pivot,
W.DefaultProb / 2});
12349// Put Cond in a virtual register to make it available from the new blocks. 12353// Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a 12354// single cluster, RHS.Low == Pivot, and we can branch to its destination 12355// directly if RHS.High equals the current upper bound. 12357if (FirstRight == LastRight && FirstRight->Kind ==
CC_Range &&
12358W.LT && (FirstRight->High->getValue() + 1ULL) ==
W.LT->getValue()) {
12359 RightMBB = FirstRight->MBB;
12364 {RightMBB, FirstRight, LastRight, Pivot,
W.LT,
W.DefaultProb / 2});
12365// Put Cond in a virtual register to make it available from the new blocks. 12369// Create the CaseBlock record that will be used to lower the branch. 12373if (
W.MBB == SwitchMBB)
12376SL->SwitchCases.push_back(CB);
12379// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb 12380// from the swith statement. 12392// Try to peel the top probability case if it exceeds the threshold. 12393// Return current MachineBasicBlock for the switch statement if the peeling 12395// If the peeling is performed, return the newly created MachineBasicBlock 12396// for the peeled switch statement. Also update Clusters to remove the peeled 12397// case. PeeledCaseProb is the BranchProbability for the peeled case. 12402// Don't perform if there is only one cluster or optimizing for size. 12409unsigned PeeledCaseIndex = 0;
12410bool SwitchPeeled =
false;
12411for (
unsigned Index = 0;
Index < Clusters.size(); ++
Index) {
12413if (
CC.Prob < TopCaseProb)
12415 TopCaseProb =
CC.Prob;
12416 PeeledCaseIndex =
Index;
12417 SwitchPeeled =
true;
12423 << TopCaseProb <<
"\n");
12425// Record the MBB for the peeled switch statement. 12433auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12435nullptr,
nullptr, TopCaseProb.
getCompl()};
12436 lowerWorkItem(W,
SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12438 Clusters.erase(PeeledCaseIt);
12441dbgs() <<
"Scale the probablity for one cluster, before scaling: " 12446 PeeledCaseProb = TopCaseProb;
12447return PeeledSwitchMBB;
12450void SelectionDAGBuilder::visitSwitch(
constSwitchInst &SI) {
12451// Extract cases from the switch. 12454 Clusters.reserve(
SI.getNumCases());
12455for (
autoI :
SI.cases()) {
12466// Cluster adjacent cases with the same destination. We do this at all 12467// optimization levels because it's cheap to do and will make codegen faster 12468// if there are many clusters. 12471// The branch probablity of the peeled case. 12474 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12476// If there is only the default destination, jump there directly. 12478if (Clusters.empty()) {
12479assert(PeeledSwitchMBB == SwitchMBB);
12481if (DefaultMBB != NextBlock(SwitchMBB)) {
12490SL->findBitTestClusters(Clusters, &SI);
12493dbgs() <<
"Case clusters: ";
12500C.Low->getValue().print(
dbgs(),
true);
12501if (
C.Low !=
C.High) {
12503C.High->getValue().print(
dbgs(),
true);
12510assert(!Clusters.empty());
12514auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12515// Scale the branchprobability for DefaultMBB if the peel occurs and 12516// DefaultMBB is not replaced. 12521 {PeeledSwitchMBB,
First,
Last,
nullptr,
nullptr, DefaultProb});
12523while (!WorkList.
empty()) {
12525unsigned NumClusters =
W.LastCluster -
W.FirstCluster + 1;
12529// For optimized builds, lower large range as a balanced binary tree. 12530 splitWorkItem(WorkList, W,
SI.getCondition(), SwitchMBB);
12534 lowerWorkItem(W,
SI.getCondition(), SwitchMBB, DefaultMBB);
12538void SelectionDAGBuilder::visitStepVector(
constCallInst &
I) {
12545void SelectionDAGBuilder::visitVectorReverse(
constCallInst &
I) {
12551assert(VT ==
V.getValueType() &&
"Malformed vector.reverse!");
12558// Use VECTOR_SHUFFLE for the fixed-length vector 12559// to maintain existing behavior. 12562for (
unsigned i = 0; i != NumElts; ++i)
12563Mask.push_back(NumElts - 1 - i);
12568void SelectionDAGBuilder::visitVectorDeinterleave(
constCallInst &
I) {
12576// ISD Node needs the input vectors split into two equal parts 12582// Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing 12583// legalisation and combines. 12599void SelectionDAGBuilder::visitVectorInterleave(
constCallInst &
I) {
12607// Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing 12608// legalisation and combines. 12624void SelectionDAGBuilder::visitFreeze(
constFreezeInst &
I) {
12628unsigned NumValues = ValueVTs.
size();
12629if (NumValues == 0)
return;
12634for (
unsigned i = 0; i != NumValues; ++i)
12642void SelectionDAGBuilder::visitVectorSplice(
constCallInst &
I) {
12649 int64_t
Imm = cast<ConstantInt>(
I.getOperand(2))->getSExtValue();
12651// VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node. 12664// Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors. 12666for (
unsigned i = 0; i < NumElts; ++i)
12671// Consider the following MIR after SelectionDAG, which produces output in 12672// phyregs in the first case or virtregs in the second case. 12674// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx 12675// %5:gr32 = COPY $ebx 12676// %6:gr32 = COPY $edx 12677// %1:gr32 = COPY %6:gr32 12678// %0:gr32 = COPY %5:gr32 12680// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32 12681// %1:gr32 = COPY %6:gr32 12682// %0:gr32 = COPY %5:gr32 12684// Given %0, we'd like to return $ebx in the first case and %5 in the second. 12685// Given %1, we'd like to return $edx in the first case and %6 in the second. 12687// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap 12688// to a single virtreg (such as %0). The remaining outputs monotonically 12689// increase in virtreg number from there. If a callbr has no outputs, then it 12690// should not have a corresponding callbr landingpad; in fact, the callbr 12691// landingpad would not even be able to refer to such a callbr. 12694// There is definitely at least one copy. 12695assert(
MI->getOpcode() == TargetOpcode::COPY &&
12696"start of copy chain MUST be COPY");
12697 Reg =
MI->getOperand(1).getReg();
12698MI =
MRI.def_begin(Reg)->getParent();
12699// There may be an optional second copy. 12700if (
MI->getOpcode() == TargetOpcode::COPY) {
12701assert(Reg.isVirtual() &&
"expected COPY of virtual register");
12702 Reg =
MI->getOperand(1).getReg();
12703assert(Reg.isPhysical() &&
"expected COPY of physical register");
12704MI =
MRI.def_begin(Reg)->getParent();
12706// The start of the chain must be an INLINEASM_BR. 12707assert(
MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12708"end of copy chain MUST be INLINEASM_BR");
12712// We must do this walk rather than the simpler 12713// setValue(&I, getCopyFromRegs(CBR, CBR->getType())); 12714// otherwise we will end up with copies of virtregs only valid along direct 12716void SelectionDAGBuilder::visitCallBrLandingPad(
constCallInst &
I) {
12720 cast<CallBrInst>(
I.getParent()->getUniquePredecessor()->getTerminator());
12729// Re-parse the asm constraints string. 12732for (
auto &
T : TargetConstraints) {
12733 SDISelAsmOperandInfo OpInfo(
T);
12737// Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the 12738// individual constraint. 12741switch (OpInfo.ConstraintType) {
12744// Fill in OpInfo.AssignedRegs.Regs. 12747// getRegistersForValue may produce 1 to many registers based on whether 12748// the OpInfo.ConstraintVT is legal on the target or not. 12749for (
Register &Reg : OpInfo.AssignedRegs.Regs) {
12753// Update the assigned registers to use the original defs. 12757SDValueV = OpInfo.AssignedRegs.getCopyFromRegs(
12760 ResultVTs.
push_back(OpInfo.ConstraintVT);
12769 ResultVTs.
push_back(OpInfo.ConstraintVT);
unsigned const MachineRegisterInfo * MRI
static unsigned getIntrinsicID(const SDNode *N)
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
const HexagonInstrInfo * TII
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::optional< ConstantRange > getRange(Value *V, const InstrInfoQuery &IIQ)
Helper method to get range from metadata or attribute.
static bool isUndef(const MachineInstr &MI)
unsigned const TargetRegisterInfo * TRI
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, ISD::MemIndexType &IndexType, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static void findWasmUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const unsigned char *MatcherTable, unsigned &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
static SymbolRef::Type getType(const Symbol *Sym)
This pass exposes codegen information to IR-level passes.
support::ulittle16_t & Lo
support::ulittle16_t & Hi
Class for arbitrary precision integers.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
This class represents an incoming formal argument to a Function.
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
This class represents the atomic memcpy intrinsic i.e.
an instruction that atomically reads a memory location, combines it with another value,...
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
LLVM Basic Block Representation.
InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
InstListType::const_iterator getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Instruction & back() const
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
This class represents a no-op cast from one type to another.
bool test(unsigned Idx) const
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
size_type size() const
size - Returns the number of bits in this bitvector.
The address of a basic block.
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
static BranchProbability getUnknown()
uint32_t getNumerator() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
A constant value that is initialized with an expression using other constant values.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
A signed pointer, in the ptrauth sense.
This class represents a range of values.
uint64_t getZExtValue() const
Constant Vector Declarations.
This is an important base class in LLVM.
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
unsigned getNonMetadataArgCount() const
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Base class for variables.
std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
iterator_range< location_op_iterator > getValues() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
bool isKillLocation() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LocationType getType() const
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
DILocation * getInlinedAt() const
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
bool allowReassoc() const
Flag queries.
An instruction for ordering other memory operations.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
Register CreateRegs(const Value *V)
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
MachineBasicBlock * getMBB(const BasicBlock *BB) const
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
BitVector DescribedArgs
Bitvector with a bit set if corresponding argument is described in ArgDbgValues.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
Register InitializeRegForValue(const Value *V)
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned ExceptionSelectorVirtReg
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
unsigned getCurrentCallSite()
Get the call site currently being processed, if any. Return zero if none.
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
MachineRegisterInfo * RegInfo
Register CreateReg(MVT VT, bool isDivergent=false)
CreateReg - Allocate a single virtual register for the given type.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
Register getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Garbage collection metadata for a single function.
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void setIsEHCatchretTarget(bool V=true)
Indicates if this is a target block of a catchret.
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
succ_iterator succ_begin()
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
Description of the location of a variable whose Address is valid and unchanging during function execu...
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
void setCallsUnwindInit(bool b)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setHasEHCatchret(bool V)
void setCallsEHReturn(bool b)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
auto getInStackSlotVariableDbgInfo()
Returns the collection of variables for which we have debug info and that have been assigned a stack ...
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
bool hasEHFunclets() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Representation for a specific memory location.
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Root of the metadata hierarchy.
A Module instance is used to store all the information related to an LLVM module.
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
void CopyValueToVirtualRegister(const Value *V, unsigned Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
DenseMap< const Constant *, unsigned > ConstantsOut
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li)
DebugLoc getCurDebugLoc() const
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
SDLoc getCurSDLoc() const
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
void LowerDeoptimizingReturn()
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
virtual void emitFunctionEntryCode()
SwiftErrorValueTracking * SwiftError
std::unique_ptr< SelectionDAGBuilder > SDB
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
Help to insert SDNodeFlags automatically in transforming.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
BlockFrequencyInfo * getBFI() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
void addMMRAMetadata(const SDNode *Node, MDNode *MMRA)
Set MMRAMetadata to be associated with Node.
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getRegister(Register Reg, EVT VT)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
ProfileSummaryInfo * getPSI() const
SDValue getTargetFrameIndex(int FI, EVT VT)
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getPartialReduceAdd(SDLoc DL, EVT ReducedTy, SDValue Op1, SDValue Op2)
Create the DAG equivalent of vector_partial_reduce where Op1 and Op2 are its operands and ReducedTY i...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
void clear()
Clear the memory usage of this object.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
TypeSize getElementOffset(unsigned Idx) const
Class to represent struct types.
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
TargetIntrinsicInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC, bool ZeroIsPoison, const ConstantRange *VScaleRange) const
Return the minimum number of bits required to hold the maximum possible number of trailing zero vecto...
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const
Return true if the @llvm.experimental.vector.match intrinsic should be expanded for vector type ‘VT’ ...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandPartialReductionIntrinsic(const IntrinsicInst *I) const
Return true if the @llvm.experimental.vector.partial.reduce.
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportPtrAuthBundles() const
Return true if the target supports ptrauth operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual MVT getJumpTableRegTy(const DataLayout &DL) const
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
bool hasConditionalLoadStoreForType(Type *Ty=nullptr) const
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
TypeID
Definitions of all of the base types for the Type system.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isTokenTy() const
Return true if this is 'token'.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ VECTOR_FIND_LAST_ACTIVE
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SSUBO
Same for subtraction.
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ PtrAuthGlobalAddress
A ptrauth constant.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Flag
These should be considered private to the implementation of the MCInstrDesc class.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
VScaleVal_match m_VScale()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Offsets
Offsets in bytes from the start of the input buffer.
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
ExceptionBehavior
Exception behavior used for floating point operations.
@ ebStrict
This corresponds to "fpexcept.strict".
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
@ ebIgnore
This corresponds to "fpexcept.ignore".
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
int popcount(T Value) noexcept
Count the number of set bits in a value.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
gep_type_iterator gep_type_end(const User *GEP)
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
void sort(IteratorTy Start, IteratorTy End)
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, const LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
@ Default
The result values are uniform if and only if all operands are uniform.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
static const fltSemantics & IEEEsingle() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
void setPointerAddrSpace(unsigned AS)
void setOrigAlign(Align A)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
bool isABIMangled() const
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
MachineBasicBlock * Default
BranchProbability DefaultProb
MachineBasicBlock * Parent
bool FallthroughUnreachable
MachineBasicBlock * ThisBB
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
BranchProbability TrueProb
BranchProbability FalseProb
MachineBasicBlock * TrueBB
MachineBasicBlock * FalseBB
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
A cluster of case labels.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
MachineBasicBlock * HeaderBB
bool FallthroughUnreachable
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
bool IsPostTypeLegalization
SmallVector< SDValue, 4 > InVals
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)
