Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitda4fe6c

Browse files
committed
[SelectionDAG] Widen <2 x T> vector types for atomic load
Vector types of 2 elements must be widened. This change does thisfor vector types of atomic load in SelectionDAGso that it can translate aligned vectors of >1 size.
1 parente9ba6ed commitda4fe6c

File tree

3 files changed

+361
-23
lines changed

3 files changed

+361
-23
lines changed

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1070,6 +1070,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
10701070
SDValueWidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
10711071
SDValueWidenVecRes_INSERT_SUBVECTOR(SDNode *N);
10721072
SDValueWidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
1073+
SDValueWidenVecRes_ATOMIC_LOAD(AtomicSDNode *N);
10731074
SDValueWidenVecRes_LOAD(SDNode* N);
10741075
SDValueWidenVecRes_VP_LOAD(VPLoadSDNode *N);
10751076
SDValueWidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N);

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 74 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4699,6 +4699,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
46994699
break;
47004700
case ISD::EXTRACT_SUBVECTOR: Res =WidenVecRes_EXTRACT_SUBVECTOR(N);break;
47014701
case ISD::INSERT_VECTOR_ELT: Res =WidenVecRes_INSERT_VECTOR_ELT(N);break;
4702+
case ISD::ATOMIC_LOAD:
4703+
Res =WidenVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
4704+
break;
47024705
case ISD::LOAD: Res =WidenVecRes_LOAD(N);break;
47034706
case ISD::STEP_VECTOR:
47044707
case ISD::SPLAT_VECTOR:
@@ -6080,6 +6083,74 @@ SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
60806083
N->getOperand(1), N->getOperand(2));
60816084
}
60826085

6086+
/// Either return the same load or provide appropriate casts
6087+
/// from the load and return that.
6088+
static SDValuecoerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6089+
TypeSize LdWidth, TypeSize FirstVTWidth,
6090+
SDLoc dl, SelectionDAG &DAG) {
6091+
assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
6092+
TypeSize WidenWidth = WidenVT.getSizeInBits();
6093+
if (!FirstVT.isVector()) {
6094+
unsigned NumElts =
6095+
WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6096+
EVT NewVecVT =EVT::getVectorVT(*DAG.getContext(), FirstVT, NumElts);
6097+
SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
6098+
return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
6099+
}
6100+
assert(FirstVT == WidenVT);
6101+
return LdOp;
6102+
}
6103+
6104+
static std::optional<EVT>findMemType(SelectionDAG &DAG,
6105+
const TargetLowering &TLI,unsigned Width,
6106+
EVT WidenVT,unsigned Align,
6107+
unsigned WidenEx);
6108+
6109+
SDValueDAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6110+
EVT WidenVT =
6111+
TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
6112+
EVT LdVT = LD->getMemoryVT();
6113+
SDLocdl(LD);
6114+
assert(LdVT.isVector() && WidenVT.isVector() &&"Expected vectors");
6115+
assert(LdVT.isScalableVector() == WidenVT.isScalableVector() &&
6116+
"Must be scalable");
6117+
assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType() &&
6118+
"Expected equivalent element types");
6119+
6120+
// Load information
6121+
SDValue Chain = LD->getChain();
6122+
SDValue BasePtr = LD->getBasePtr();
6123+
MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
6124+
AAMDNodes AAInfo = LD->getAAInfo();
6125+
6126+
TypeSize LdWidth = LdVT.getSizeInBits();
6127+
TypeSize WidenWidth = WidenVT.getSizeInBits();
6128+
TypeSize WidthDiff = WidenWidth - LdWidth;
6129+
6130+
// Find the vector type that can load from.
6131+
std::optional<EVT> FirstVT =
6132+
findMemType(DAG, TLI, LdWidth.getKnownMinValue(), WidenVT,/*LdAlign=*/0,
6133+
WidthDiff.getKnownMinValue());
6134+
6135+
if (!FirstVT)
6136+
returnSDValue();
6137+
6138+
SmallVector<EVT,8> MemVTs;
6139+
TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6140+
6141+
SDValue LdOp = DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, *FirstVT, *FirstVT,
6142+
Chain, BasePtr, LD->getMemOperand());
6143+
6144+
// Load the element with one instruction.
6145+
SDValue Result =coerceLoadedValue(LdOp, *FirstVT, WidenVT, LdWidth,
6146+
FirstVTWidth, dl, DAG);
6147+
6148+
// Modified the chain - switch anything that used the old chain to use
6149+
// the new one.
6150+
ReplaceValueWith(SDValue(LD,1), LdOp.getValue(1));
6151+
return Result;
6152+
}
6153+
60836154
SDValueDAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
60846155
LoadSDNode *LD = cast<LoadSDNode>(N);
60856156
ISD::LoadExtType ExtType = LD->getExtensionType();
@@ -7988,29 +8059,9 @@ SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
79888059
LdChain.push_back(LdOp.getValue(1));
79898060

79908061
// Check if we can load the element with one instruction.
7991-
if (MemVTs.empty()) {
7992-
assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth));
7993-
if (!FirstVT->isVector()) {
7994-
unsigned NumElts =
7995-
WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
7996-
EVT NewVecVT =EVT::getVectorVT(*DAG.getContext(), *FirstVT, NumElts);
7997-
SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
7998-
return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
7999-
}
8000-
if (FirstVT == WidenVT)
8001-
return LdOp;
8002-
8003-
// TODO: We don't currently have any tests that exercise this code path.
8004-
assert(WidenWidth.getFixedValue() % FirstVTWidth.getFixedValue() ==0);
8005-
unsigned NumConcat =
8006-
WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
8007-
SmallVector<SDValue,16>ConcatOps(NumConcat);
8008-
SDValue UndefVal = DAG.getUNDEF(*FirstVT);
8009-
ConcatOps[0] = LdOp;
8010-
for (unsigned i =1; i != NumConcat; ++i)
8011-
ConcatOps[i] = UndefVal;
8012-
return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
8013-
}
8062+
if (MemVTs.empty())
8063+
returncoerceLoadedValue(LdOp, *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8064+
DAG);
80148065

80158066
// Load vector by using multiple loads from largest vector to scalar.
80168067
SmallVector<SDValue,16> LdOps;

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp