Movatterモバイル変換

Go to the documentation of this file.

1//===- ProfDataUtils.cpp - Utility functions for MD_prof Metadata ---------===//

2//

3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4// See https://llvm.org/LICENSE.txt for license information.

5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6//

7//===----------------------------------------------------------------------===//

8//

9// This file implements utilities for working with Profiling Metadata.

10//

11//===----------------------------------------------------------------------===//

13#include "llvm/IR/ProfDataUtils.h"

14#include "llvm/ADT/SmallVector.h"

15#include "llvm/IR/Constants.h"

16#include "llvm/IR/Function.h"

17#include "llvm/IR/Instructions.h"

18#include "llvm/IR/LLVMContext.h"

19#include "llvm/IR/MDBuilder.h"

20#include "llvm/IR/Metadata.h"

21#include "llvm/IR/ProfDataUtils.h"

23using namespacellvm;

25namespace{

27// MD_prof nodes have the following layout

28//

29// In general:

30// { String name, Array of i32 }

31//

32// In terms of Types:

33// { MDString, [i32, i32, ...]}

34//

35// Concretely for Branch Weights

36// { "branch_weights", [i32 1, i32 10000]}

37//

38// We maintain some constants here to ensure that we access the branch weights

39// correctly, and can change the behavior in the future if the layout changes

41// the minimum number of operands for MD_prof nodes with branch weights

42constexprunsigned MinBWOps = 3;

44// the minimum number of operands for MD_prof nodes with value profiles

45constexprunsigned MinVPOps = 5;

47// We may want to add support for other MD_prof types, so provide an abstraction

48// for checking the metadata type.

49bool isTargetMD(constMDNode *ProfData,constchar *Name,unsigned MinOps) {

50// TODO: This routine may be simplified if MD_prof used an enum instead of a

51// string to differentiate the types of MD_prof nodes.

52if (!ProfData || !Name || MinOps < 2)

53returnfalse;

55unsigned NOps = ProfData->getNumOperands();

56if (NOps < MinOps)

57returnfalse;

59auto *ProfDataName = dyn_cast<MDString>(ProfData->getOperand(0));

60if (!ProfDataName)

61returnfalse;

63return ProfDataName->getString() ==Name;

64}

66template <typenameT,

67typename =typename std::enable_if<std::is_arithmetic_v<T>>>

68staticvoid extractFromBranchWeightMD(constMDNode *ProfileData,

69SmallVectorImpl<T> &Weights) {

70assert(isBranchWeightMD(ProfileData) &&"wrong metadata");

72unsigned NOps = ProfileData->getNumOperands();

73unsigned WeightsIdx =getBranchWeightOffset(ProfileData);

74assert(WeightsIdx < NOps &&"Weights Index must be less than NOps.");

75 Weights.resize(NOps - WeightsIdx);

77for (unsignedIdx = WeightsIdx, E = NOps;Idx != E; ++Idx) {

78ConstantInt *Weight =

79 mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx));

80assert(Weight &&"Malformed branch_weight in MD_prof node");

81assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) &&

82"Too many bits for MD_prof branch_weight");

83 Weights[Idx - WeightsIdx] = Weight->getZExtValue();

84 }

85}

87}// namespace

89namespacellvm {

91boolhasProfMD(constInstruction &I) {

92returnI.hasMetadata(LLVMContext::MD_prof);

93}

95boolisBranchWeightMD(constMDNode *ProfileData) {

96return isTargetMD(ProfileData,"branch_weights", MinBWOps);

97}

99boolisValueProfileMD(constMDNode *ProfileData) {

100return isTargetMD(ProfileData,"VP", MinVPOps);

101}

102

103boolhasBranchWeightMD(constInstruction &I) {

104auto *ProfileData =I.getMetadata(LLVMContext::MD_prof);

105returnisBranchWeightMD(ProfileData);

106}

107

108boolhasCountTypeMD(constInstruction &I) {

109auto *ProfileData =I.getMetadata(LLVMContext::MD_prof);

110// Value profiles record count-type information.

111if (isValueProfileMD(ProfileData))

112returntrue;

113// Conservatively assume non CallBase instruction only get taken/not-taken

114// branch probability, so not interpret them as count.

115return isa<CallBase>(I) && !isBranchWeightMD(ProfileData);

116}

117

118boolhasValidBranchWeightMD(constInstruction &I) {

119returngetValidBranchWeightMDNode(I);

120}

121

122boolhasBranchWeightOrigin(constInstruction &I) {

123auto *ProfileData =I.getMetadata(LLVMContext::MD_prof);

124returnhasBranchWeightOrigin(ProfileData);

125}

126

127boolhasBranchWeightOrigin(constMDNode *ProfileData) {

128if (!isBranchWeightMD(ProfileData))

129returnfalse;

130auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(1));

131// NOTE: if we ever have more types of branch weight provenance,

132// we need to check the string value is "expected". For now, we

133// supply a more generic API, and avoid the spurious comparisons.

134assert(ProfDataName ==nullptr || ProfDataName->getString() =="expected");

135return ProfDataName !=nullptr;

136}

137

138unsignedgetBranchWeightOffset(constMDNode *ProfileData) {

139returnhasBranchWeightOrigin(ProfileData) ? 2 : 1;

140}

141

142unsignedgetNumBranchWeights(constMDNode &ProfileData) {

143return ProfileData.getNumOperands() -getBranchWeightOffset(&ProfileData);

144}

145

146MDNode *getBranchWeightMDNode(constInstruction &I) {

147auto *ProfileData =I.getMetadata(LLVMContext::MD_prof);

148if (!isBranchWeightMD(ProfileData))

149returnnullptr;

150return ProfileData;

151}

152

153MDNode *getValidBranchWeightMDNode(constInstruction &I) {

154auto *ProfileData =getBranchWeightMDNode(I);

155if (ProfileData &&getNumBranchWeights(*ProfileData) ==I.getNumSuccessors())

156return ProfileData;

157returnnullptr;

158}

159

160voidextractFromBranchWeightMD32(constMDNode *ProfileData,

161SmallVectorImpl<uint32_t> &Weights) {

162 extractFromBranchWeightMD(ProfileData, Weights);

163}

164

165voidextractFromBranchWeightMD64(constMDNode *ProfileData,

166SmallVectorImpl<uint64_t> &Weights) {

167 extractFromBranchWeightMD(ProfileData, Weights);

168}

169

170boolextractBranchWeights(constMDNode *ProfileData,

171SmallVectorImpl<uint32_t> &Weights) {

172if (!isBranchWeightMD(ProfileData))

173returnfalse;

174 extractFromBranchWeightMD(ProfileData, Weights);

175returntrue;

176}

177

178boolextractBranchWeights(constInstruction &I,

179SmallVectorImpl<uint32_t> &Weights) {

180auto *ProfileData =I.getMetadata(LLVMContext::MD_prof);

181returnextractBranchWeights(ProfileData, Weights);

182}

183

184boolextractBranchWeights(constInstruction &I,uint64_t &TrueVal,

185uint64_t &FalseVal) {

186assert((I.getOpcode() == Instruction::Br ||

187I.getOpcode() == Instruction::Select) &&

188"Looking for branch weights on something besides branch, select, or "

189"switch");

190

191SmallVector<uint32_t, 2> Weights;

192auto *ProfileData =I.getMetadata(LLVMContext::MD_prof);

193if (!extractBranchWeights(ProfileData, Weights))

194returnfalse;

195

196if (Weights.size() > 2)

197returnfalse;

198

199 TrueVal = Weights[0];

200 FalseVal = Weights[1];

201returntrue;

202}

203

204boolextractProfTotalWeight(constMDNode *ProfileData,uint64_t &TotalVal) {

205 TotalVal = 0;

206if (!ProfileData)

207returnfalse;

208

209auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0));

210if (!ProfDataName)

211returnfalse;

212

213if (ProfDataName->getString() =="branch_weights") {

214unsignedOffset =getBranchWeightOffset(ProfileData);

215for (unsignedIdx =Offset;Idx < ProfileData->getNumOperands(); ++Idx) {

216auto *V = mdconst::extract<ConstantInt>(ProfileData->getOperand(Idx));

217 TotalVal += V->getValue().getZExtValue();

218 }

219returntrue;

220 }

221

222if (ProfDataName->getString() =="VP" && ProfileData->getNumOperands() > 3) {

223 TotalVal = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2))

224 ->getValue()

225 .getZExtValue();

226returntrue;

227 }

228returnfalse;

229}

230

231boolextractProfTotalWeight(constInstruction &I,uint64_t &TotalVal) {

232returnextractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal);

233}

234

235voidsetBranchWeights(Instruction &I,ArrayRef<uint32_t> Weights,

236bool IsExpected) {

237MDBuilder MDB(I.getContext());

238MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected);

239I.setMetadata(LLVMContext::MD_prof, BranchWeights);

240}

241

242voidscaleProfData(Instruction &I,uint64_t S,uint64_t T) {

243assert(T != 0 &&"Caller should guarantee");

244auto *ProfileData =I.getMetadata(LLVMContext::MD_prof);

245if (ProfileData ==nullptr)

246return;

247

248auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0));

249if (!ProfDataName || (ProfDataName->getString() !="branch_weights" &&

250 ProfDataName->getString() !="VP"))

251return;

252

253if (!hasCountTypeMD(I))

254return;

255

256LLVMContext &C =I.getContext();

257

258MDBuilder MDB(C);

259SmallVector<Metadata *, 3> Vals;

260 Vals.push_back(ProfileData->getOperand(0));

261APInt APS(128, S), APT(128,T);

262if (ProfDataName->getString() =="branch_weights" &&

263 ProfileData->getNumOperands() > 0) {

264// Using APInt::div may be expensive, but most cases should fit 64 bits.

265APInt Val(128,

266 mdconst::dyn_extract<ConstantInt>(

267 ProfileData->getOperand(getBranchWeightOffset(ProfileData)))

268 ->getValue()

269 .getZExtValue());

270 Val *= APS;

271 Vals.push_back(MDB.createConstant(ConstantInt::get(

272Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX))));

273 }elseif (ProfDataName->getString() =="VP")

274for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) {

275// The first value is the key of the value profile, which will not change.

276 Vals.push_back(ProfileData->getOperand(i));

277uint64_t Count =

278 mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i + 1))

279 ->getValue()

280 .getZExtValue();

281// Don't scale the magic number.

282if (Count ==NOMORE_ICP_MAGICNUM) {

283 Vals.push_back(ProfileData->getOperand(i + 1));

284continue;

285 }

286// Using APInt::div may be expensive, but most cases should fit 64 bits.

287APInt Val(128, Count);

288 Val *= APS;

289 Vals.push_back(MDB.createConstant(ConstantInt::get(

290Type::getInt64Ty(C), Val.udiv(APT).getLimitedValue())));

291 }

292I.setMetadata(LLVMContext::MD_prof,MDNode::get(C, Vals));

293}

294

295}// namespace llvm

Constants.h

This file contains the declarations for the subclasses of Constant, which represent the different fla...

Idx

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

Definition:DeadArgumentElimination.cpp:353

Name

std::string Name

Definition:ELFObjHandler.cpp:77

Function.h

Instructions.h

LLVMContext.h

#define I(x, y, z)

Definition:MD5.cpp:58

MDBuilder.h

Metadata.h

This file contains the declarations for metadata subclasses.

ProfDataUtils.h

This file contains the declarations for profiling metadata utility functions.

assert

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

SmallVector.h

This file defines the SmallVector class.

llvm::APInt

Class for arbitrary precision integers.

Definition:APInt.h:78

llvm::APInt::udiv

APInt udiv(const APInt &RHS) const

Unsigned division operation.

Definition:APInt.cpp:1547

llvm::APInt::getActiveBits

unsigned getActiveBits() const

Compute the number of active bits in the value.

Definition:APInt.h:1492

llvm::APInt::getLimitedValue

uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const

If this value is smaller than the specified limit, return it, otherwise return the limit value.

Definition:APInt.h:475

llvm::ArrayRef

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Definition:ArrayRef.h:41

llvm::ConstantInt

This is the shared class of boolean and integer constants.

Definition:Constants.h:83

llvm::ConstantInt::getZExtValue

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

Definition:Constants.h:157

llvm::ConstantInt::getValue

const APInt & getValue() const

Return the constant as an APInt value reference.

Definition:Constants.h:148

llvm::Instruction

Definition:Instruction.h:68

llvm::LLVMContext

This is an important class for using LLVM in a threaded context.

Definition:LLVMContext.h:67

llvm::MDBuilder

Definition:MDBuilder.h:36

llvm::MDBuilder::createConstant

ConstantAsMetadata * createConstant(Constant *C)

Return the given constant as metadata.

Definition:MDBuilder.cpp:24

llvm::MDBuilder::createBranchWeights

MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)

Return metadata containing two branch weights.

Definition:MDBuilder.cpp:37

llvm::MDNode

Metadata node.

Definition:Metadata.h:1073

llvm::MDNode::getOperand

const MDOperand & getOperand(unsigned I) const

Definition:Metadata.h:1434

llvm::MDNode::get

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

Definition:Metadata.h:1549

llvm::MDNode::getNumOperands

unsigned getNumOperands() const

Return number of MDNode operands.

Definition:Metadata.h:1440

llvm::SmallVectorBase::size

size_t size() const

Definition:SmallVector.h:78

llvm::SmallVectorImpl

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

Definition:SmallVector.h:573

llvm::SmallVectorImpl::resize

void resize(size_type N)

Definition:SmallVector.h:638

llvm::SmallVectorTemplateBase::push_back

void push_back(const T &Elt)

Definition:SmallVector.h:413

llvm::SmallVector

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

Definition:SmallVector.h:1196

llvm::Type::getInt32Ty

static IntegerType * getInt32Ty(LLVMContext &C)

llvm::Type::getInt64Ty

static IntegerType * getInt64Ty(LLVMContext &C)

uint64_t

llvm::CallingConv::C

@ C

The default llvm calling convention, compatible with C.

Definition:CallingConv.h:34

llvm

This is an optimization pass for GlobalISel generic memory operations.

Definition:AddressRanges.h:18

llvm::Offset

@ Offset

Definition:DWP.cpp:480

llvm::extractProfTotalWeight

bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights)

Retrieve the total of all weights from MD_prof data.

Definition:ProfDataUtils.cpp:204

llvm::getBranchWeightOffset

unsigned getBranchWeightOffset(const MDNode *ProfileData)

Return the offset to the first branch weight data.

Definition:ProfDataUtils.cpp:138

llvm::isBranchWeightMD

bool isBranchWeightMD(const MDNode *ProfileData)

Checks if an MDNode contains Branch Weight Metadata.

Definition:ProfDataUtils.cpp:95

llvm::getBranchWeightMDNode

MDNode * getBranchWeightMDNode(const Instruction &I)

Get the branch weights metadata node.

Definition:ProfDataUtils.cpp:146

llvm::hasBranchWeightOrigin

bool hasBranchWeightOrigin(const Instruction &I)

Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.

Definition:ProfDataUtils.cpp:122

llvm::setBranchWeights

void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

Definition:ProfDataUtils.cpp:235

llvm::getValidBranchWeightMDNode

MDNode * getValidBranchWeightMDNode(const Instruction &I)

Get the valid branch weights metadata node.

Definition:ProfDataUtils.cpp:153

llvm::hasValidBranchWeightMD

bool hasValidBranchWeightMD(const Instruction &I)

Checks if an instructions has valid Branch Weight Metadata.

Definition:ProfDataUtils.cpp:118

llvm::isValueProfileMD

bool isValueProfileMD(const MDNode *ProfileData)

Definition:ProfDataUtils.cpp:99

llvm::hasCountTypeMD

bool hasCountTypeMD(const Instruction &I)

Definition:ProfDataUtils.cpp:108

llvm::getNumBranchWeights

unsigned getNumBranchWeights(const MDNode &ProfileData)

Definition:ProfDataUtils.cpp:142

llvm::extractFromBranchWeightMD32

void extractFromBranchWeightMD32(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...

Definition:ProfDataUtils.cpp:160

llvm::hasProfMD

bool hasProfMD(const Instruction &I)

Checks if an Instruction has MD_prof Metadata.

Definition:ProfDataUtils.cpp:91

llvm::extractBranchWeights

bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Extract branch weights from MD_prof metadata.

Definition:ProfDataUtils.cpp:170

llvm::hasBranchWeightMD

bool hasBranchWeightMD(const Instruction &I)

Checks if an instructions has Branch Weight Metadata.

Definition:ProfDataUtils.cpp:103

llvm::NOMORE_ICP_MAGICNUM

const uint64_t NOMORE_ICP_MAGICNUM

Magic number in the value profile metadata showing a target has been promoted for the instruction and...

Definition:Metadata.h:57

llvm::scaleProfData

void scaleProfData(Instruction &I, uint64_t S, uint64_t T)

Scaling the profile data attached to 'I' using the ratio of S/T.

Definition:ProfDataUtils.cpp:242

llvm::extractFromBranchWeightMD64

void extractFromBranchWeightMD64(const MDNode *ProfileData, SmallVectorImpl< uint64_t > &Weights)

Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...

Definition:ProfDataUtils.cpp:165