Movatterモバイル変換

Go to the documentation of this file.

1//===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//

2//

3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4// See https://llvm.org/LICENSE.txt for license information.

5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6//

7//===----------------------------------------------------------------------===//

8//

9// This file contains support for reading profiling data for clang's

10// instrumentation based PGO and coverage.

11//

12//===----------------------------------------------------------------------===//

14#include "llvm/ProfileData/InstrProfReader.h"

15#include "llvm/ADT/ArrayRef.h"

16#include "llvm/ADT/DenseMap.h"

17#include "llvm/ADT/StringExtras.h"

18#include "llvm/ADT/StringRef.h"

19#include "llvm/IR/ProfileSummary.h"

20#include "llvm/ProfileData/InstrProf.h"

21#include "llvm/ProfileData/MemProf.h"

22#include "llvm/ProfileData/ProfileCommon.h"

23#include "llvm/ProfileData/SymbolRemappingReader.h"

24#include "llvm/Support/Endian.h"

25#include "llvm/Support/Error.h"

26#include "llvm/Support/ErrorOr.h"

27#include "llvm/Support/FormatVariadic.h"

28#include "llvm/Support/MemoryBuffer.h"

29#include "llvm/Support/VirtualFileSystem.h"

30#include <algorithm>

31#include <cstddef>

32#include <cstdint>

33#include <limits>

34#include <memory>

35#include <optional>

36#include <system_error>

37#include <utility>

38#include <vector>

40using namespacellvm;

42// Extracts the variant information from the top 32 bits in the version and

43// returns an enum specifying the variants present.

44staticInstrProfKind getProfileKindFromVersion(uint64_t Version) {

45InstrProfKind ProfileKind = InstrProfKind::Unknown;

46if (Version & VARIANT_MASK_IR_PROF) {

47 ProfileKind |= InstrProfKind::IRInstrumentation;

48 }

49if (Version & VARIANT_MASK_CSIR_PROF) {

50 ProfileKind |= InstrProfKind::ContextSensitive;

51 }

52if (Version & VARIANT_MASK_INSTR_ENTRY) {

53 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;

54 }

55if (Version & VARIANT_MASK_INSTR_LOOP_ENTRIES) {

56 ProfileKind |= InstrProfKind::LoopEntriesInstrumentation;

57 }

58if (Version & VARIANT_MASK_BYTE_COVERAGE) {

59 ProfileKind |= InstrProfKind::SingleByteCoverage;

60 }

61if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {

62 ProfileKind |= InstrProfKind::FunctionEntryOnly;

63 }

64if (Version & VARIANT_MASK_MEMPROF) {

65 ProfileKind |= InstrProfKind::MemProf;

66 }

67if (Version & VARIANT_MASK_TEMPORAL_PROF) {

68 ProfileKind |= InstrProfKind::TemporalProfile;

69 }

70return ProfileKind;

71}

73staticExpected<std::unique_ptr<MemoryBuffer>>

74setupMemoryBuffer(constTwine &Filename,vfs::FileSystem &FS) {

75auto BufferOrErr = Filename.str() =="-" ?MemoryBuffer::getSTDIN()

76 : FS.getBufferForFile(Filename);

77if (std::error_code EC = BufferOrErr.getError())

78returnerrorCodeToError(EC);

79return std::move(BufferOrErr.get());

80}

82staticError initializeReader(InstrProfReader &Reader) {

83return Reader.readHeader();

84}

86/// Read a list of binary ids from a profile that consist of

87/// a. uint64_t binary id length

88/// b. uint8_t binary id data

89/// c. uint8_t padding (if necessary)

90/// This function is shared between raw and indexed profiles.

91/// Raw profiles are in host-endian format, and indexed profiles are in

92/// little-endian format. So, this function takes an argument indicating the

93/// associated endian format to read the binary ids correctly.

94staticError

95readBinaryIdsInternal(constMemoryBuffer &DataBuffer,

96ArrayRef<uint8_t> BinaryIdsBuffer,

97 std::vector<llvm::object::BuildID> &BinaryIds,

98constllvm::endianness Endian) {

99using namespacesupport;

100

101constuint64_t BinaryIdsSize = BinaryIdsBuffer.size();

102constuint8_t *BinaryIdsStart = BinaryIdsBuffer.data();

103

104if (BinaryIdsSize == 0)

105returnError::success();

106

107constuint8_t *BI = BinaryIdsStart;

108constuint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;

109constuint8_t *End =

110reinterpret_cast<constuint8_t *>(DataBuffer.getBufferEnd());

111

112while (BI < BIEnd) {

113size_t Remaining = BIEnd - BI;

114// There should be enough left to read the binary id length.

115if (Remaining <sizeof(uint64_t))

116return make_error<InstrProfError>(

117 instrprof_error::malformed,

118"not enough data to read binary id length");

119

120uint64_t BILen = endian::readNext<uint64_t>(BI,Endian);

121if (BILen == 0)

122return make_error<InstrProfError>(instrprof_error::malformed,

123"binary id length is 0");

124

125 Remaining = BIEnd - BI;

126// There should be enough left to read the binary id data.

127if (Remaining <alignToPowerOf2(BILen,sizeof(uint64_t)))

128return make_error<InstrProfError>(

129 instrprof_error::malformed,"not enough data to read binary id data");

130

131// Add binary id to the binary ids list.

132 BinaryIds.push_back(object::BuildID(BI, BI + BILen));

133

134// Increment by binary id data length, which aligned to the size of uint64.

135 BI +=alignToPowerOf2(BILen,sizeof(uint64_t));

136if (BI >End)

137return make_error<InstrProfError>(

138 instrprof_error::malformed,

139"binary id section is greater than buffer size");

140 }

141

142returnError::success();

143}

144

145staticvoidprintBinaryIdsInternal(raw_ostream &OS,

146ArrayRef<llvm::object::BuildID> BinaryIds) {

147OS <<"Binary IDs: \n";

148for (constauto &BI : BinaryIds) {

149for (autoI : BI)

150OS <<format("%02x",I);

151OS <<"\n";

152 }

153}

154

155Expected<std::unique_ptr<InstrProfReader>>InstrProfReader::create(

156constTwine &Path,vfs::FileSystem &FS,

157constInstrProfCorrelator *Correlator,

158constobject::BuildIDFetcher *BIDFetcher,

159constInstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind,

160 std::function<void(Error)> Warn) {

161// Set up the buffer to read.

162auto BufferOrError =setupMemoryBuffer(Path, FS);

163if (Error E = BufferOrError.takeError())

164return std::move(E);

165returnInstrProfReader::create(std::move(BufferOrError.get()), Correlator,

166 BIDFetcher, BIDFetcherCorrelatorKind, Warn);

167}

168

169Expected<std::unique_ptr<InstrProfReader>>InstrProfReader::create(

170 std::unique_ptr<MemoryBuffer> Buffer,constInstrProfCorrelator *Correlator,

171constobject::BuildIDFetcher *BIDFetcher,

172constInstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind,

173 std::function<void(Error)> Warn) {

174if (Buffer->getBufferSize() == 0)

175return make_error<InstrProfError>(instrprof_error::empty_raw_profile);

176

177 std::unique_ptr<InstrProfReader> Result;

178// Create the reader.

179if (IndexedInstrProfReader::hasFormat(*Buffer))

180 Result.reset(newIndexedInstrProfReader(std::move(Buffer)));

181elseif (RawInstrProfReader64::hasFormat(*Buffer))

182 Result.reset(newRawInstrProfReader64(std::move(Buffer), Correlator,

183 BIDFetcher, BIDFetcherCorrelatorKind,

184 Warn));

185elseif (RawInstrProfReader32::hasFormat(*Buffer))

186 Result.reset(newRawInstrProfReader32(std::move(Buffer), Correlator,

187 BIDFetcher, BIDFetcherCorrelatorKind,

188 Warn));

189elseif (TextInstrProfReader::hasFormat(*Buffer))

190 Result.reset(newTextInstrProfReader(std::move(Buffer)));

191else

192return make_error<InstrProfError>(instrprof_error::unrecognized_format);

193

194// Initialize the reader and return the result.

195if (Error E =initializeReader(*Result))

196return std::move(E);

197

198return std::move(Result);

199}

200

201Expected<std::unique_ptr<IndexedInstrProfReader>>

202IndexedInstrProfReader::create(constTwine &Path,vfs::FileSystem &FS,

203constTwine &RemappingPath) {

204// Set up the buffer to read.

205auto BufferOrError =setupMemoryBuffer(Path, FS);

206if (Error E = BufferOrError.takeError())

207return std::move(E);

208

209// Set up the remapping buffer if requested.

210 std::unique_ptr<MemoryBuffer> RemappingBuffer;

211 std::string RemappingPathStr = RemappingPath.str();

212if (!RemappingPathStr.empty()) {

213auto RemappingBufferOrError =setupMemoryBuffer(RemappingPathStr, FS);

214if (Error E = RemappingBufferOrError.takeError())

215return std::move(E);

216 RemappingBuffer = std::move(RemappingBufferOrError.get());

217 }

218

219returnIndexedInstrProfReader::create(std::move(BufferOrError.get()),

220 std::move(RemappingBuffer));

221}

222

223Expected<std::unique_ptr<IndexedInstrProfReader>>

224IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,

225 std::unique_ptr<MemoryBuffer> RemappingBuffer) {

226// Create the reader.

227if (!IndexedInstrProfReader::hasFormat(*Buffer))

228return make_error<InstrProfError>(instrprof_error::bad_magic);

229auto Result = std::make_unique<IndexedInstrProfReader>(

230 std::move(Buffer), std::move(RemappingBuffer));

231

232// Initialize the reader and return the result.

233if (Error E =initializeReader(*Result))

234return std::move(E);

235

236return std::move(Result);

237}

238

239boolTextInstrProfReader::hasFormat(constMemoryBuffer &Buffer) {

240// Verify that this really looks like plain ASCII text by checking a

241// 'reasonable' number of characters (up to profile magic size).

242size_tcount = std::min(Buffer.getBufferSize(),sizeof(uint64_t));

243StringRef buffer = Buffer.getBufferStart();

244returncount == 0 ||

245 std::all_of(buffer.begin(), buffer.begin() +count,

246 [](char c) { return isPrint(c) || isSpace(c); });

247}

248

249// Read the profile variant flag from the header: ":FE" means this is a FE

250// generated profile. ":IR" means this is an IR level profile. Other strings

251// with a leading ':' will be reported an error format.

252Error TextInstrProfReader::readHeader() {

253Symtab.reset(newInstrProfSymtab());

254

255while (Line->starts_with(":")) {

256StringRef Str = Line->substr(1);

257if (Str.equals_insensitive("ir"))

258 ProfileKind |=InstrProfKind::IRInstrumentation;

259elseif (Str.equals_insensitive("fe"))

260 ProfileKind |=InstrProfKind::FrontendInstrumentation;

261elseif (Str.equals_insensitive("csir")) {

262 ProfileKind |=InstrProfKind::IRInstrumentation;

263 ProfileKind |=InstrProfKind::ContextSensitive;

264 }elseif (Str.equals_insensitive("entry_first"))

265 ProfileKind |=InstrProfKind::FunctionEntryInstrumentation;

266elseif (Str.equals_insensitive("not_entry_first"))

267 ProfileKind &=~InstrProfKind ::FunctionEntryInstrumentation;

268elseif (Str.equals_insensitive("instrument_loop_entries"))

269 ProfileKind |=InstrProfKind::LoopEntriesInstrumentation;

270elseif (Str.equals_insensitive("single_byte_coverage"))

271 ProfileKind |=InstrProfKind::SingleByteCoverage;

272elseif (Str.equals_insensitive("temporal_prof_traces")) {

273 ProfileKind |=InstrProfKind::TemporalProfile;

274if (auto Err = readTemporalProfTraceData())

275returnerror(std::move(Err));

276 }else

277returnerror(instrprof_error::bad_header);

278 ++Line;

279 }

280returnsuccess();

281}

282

283/// Temporal profile trace data is stored in the header immediately after

284/// ":temporal_prof_traces". The first integer is the number of traces, the

285/// second integer is the stream size, then the following lines are the actual

286/// traces which consist of a weight and a comma separated list of function

287/// names.

288Error TextInstrProfReader::readTemporalProfTraceData() {

289if ((++Line).is_at_end())

290returnerror(instrprof_error::eof);

291

292uint32_t NumTraces;

293if (Line->getAsInteger(0, NumTraces))

294returnerror(instrprof_error::malformed);

295

296if ((++Line).is_at_end())

297returnerror(instrprof_error::eof);

298

299if (Line->getAsInteger(0,TemporalProfTraceStreamSize))

300returnerror(instrprof_error::malformed);

301

302for (uint32_t i = 0; i < NumTraces; i++) {

303if ((++Line).is_at_end())

304returnerror(instrprof_error::eof);

305

306TemporalProfTraceTy Trace;

307if (Line->getAsInteger(0,Trace.Weight))

308returnerror(instrprof_error::malformed);

309

310if ((++Line).is_at_end())

311returnerror(instrprof_error::eof);

312

313SmallVector<StringRef> FuncNames;

314 Line->split(FuncNames,",",/*MaxSplit=*/-1,/*KeepEmpty=*/false);

315for (auto &FuncName : FuncNames)

316Trace.FunctionNameRefs.push_back(

317IndexedInstrProf::ComputeHash(FuncName.trim()));

318TemporalProfTraces.push_back(std::move(Trace));

319 }

320returnsuccess();

321}

322

323Error

324TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {

325

326#define CHECK_LINE_END(Line) \

327 if (Line.is_at_end()) \

328 return error(instrprof_error::truncated);

329#define READ_NUM(Str, Dst) \

330 if ((Str).getAsInteger(10, (Dst))) \

331 return error(instrprof_error::malformed);

332#define VP_READ_ADVANCE(Val) \

333 CHECK_LINE_END(Line); \

334 uint32_t Val; \

335 READ_NUM((*Line), (Val)); \

336 Line++;

337

338if (Line.is_at_end())

339returnsuccess();

340

341uint32_t NumValueKinds;

342if (Line->getAsInteger(10, NumValueKinds)) {

343// No value profile data

344returnsuccess();

345 }

346if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)

347returnerror(instrprof_error::malformed,

348"number of value kinds is invalid");

349 Line++;

350

351for (uint32_t VK = 0; VK < NumValueKinds; VK++) {

352VP_READ_ADVANCE(ValueKind);

353if (ValueKind > IPVK_Last)

354returnerror(instrprof_error::malformed,"value kind is invalid");

355 ;

356VP_READ_ADVANCE(NumValueSites);

357if (!NumValueSites)

358continue;

359

360Record.reserveSites(VK, NumValueSites);

361for (uint32_t S = 0; S < NumValueSites; S++) {

362VP_READ_ADVANCE(NumValueData);

363

364 std::vector<InstrProfValueData> CurrentValues;

365for (uint32_t V = 0;V < NumValueData;V++) {

366CHECK_LINE_END(Line);

367 std::pair<StringRef, StringRef> VD = Line->rsplit(':');

368uint64_t TakenCount,Value;

369if (ValueKind == IPVK_IndirectCallTarget) {

370if (InstrProfSymtab::isExternalSymbol(VD.first)) {

371Value = 0;

372 }else {

373if (Error E =Symtab->addFuncName(VD.first))

374return E;

375Value =IndexedInstrProf::ComputeHash(VD.first);

376 }

377 }elseif (ValueKind == IPVK_VTableTarget) {

378if (InstrProfSymtab::isExternalSymbol(VD.first))

379Value = 0;

380else {

381if (Error E =Symtab->addVTableName(VD.first))

382return E;

383Value =IndexedInstrProf::ComputeHash(VD.first);

384 }

385 }else {

386READ_NUM(VD.first,Value);

387 }

388READ_NUM(VD.second, TakenCount);

389 CurrentValues.push_back({Value, TakenCount});

390 Line++;

391 }

392assert(CurrentValues.size() == NumValueData);

393Record.addValueData(ValueKind, S, CurrentValues,nullptr);

394 }

395 }

396returnsuccess();

397

398#undef CHECK_LINE_END

399#undef READ_NUM

400#undef VP_READ_ADVANCE

401}

402

403Error TextInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {

404// Skip empty lines and comments.

405while (!Line.is_at_end() && (Line->empty() || Line->starts_with("#")))

406 ++Line;

407// If we hit EOF while looking for a name, we're done.

408if (Line.is_at_end()) {

409returnerror(instrprof_error::eof);

410 }

411

412// Read the function name.

413Record.Name = *Line++;

414if (Error E =Symtab->addFuncName(Record.Name))

415returnerror(std::move(E));

416

417// Read the function hash.

418if (Line.is_at_end())

419returnerror(instrprof_error::truncated);

420if ((Line++)->getAsInteger(0,Record.Hash))

421returnerror(instrprof_error::malformed,

422"function hash is not a valid integer");

423

424// Read the number of counters.

425uint64_t NumCounters;

426if (Line.is_at_end())

427returnerror(instrprof_error::truncated);

428if ((Line++)->getAsInteger(10, NumCounters))

429returnerror(instrprof_error::malformed,

430"number of counters is not a valid integer");

431if (NumCounters == 0)

432returnerror(instrprof_error::malformed,"number of counters is zero");

433

434// Read each counter and fill our internal storage with the values.

435Record.Clear();

436Record.Counts.reserve(NumCounters);

437for (uint64_t I = 0;I < NumCounters; ++I) {

438if (Line.is_at_end())

439returnerror(instrprof_error::truncated);

440uint64_t Count;

441if ((Line++)->getAsInteger(10, Count))

442returnerror(instrprof_error::malformed,"count is invalid");

443Record.Counts.push_back(Count);

444 }

445

446// Bitmap byte information is indicated with special character.

447if (Line->starts_with("$")) {

448Record.BitmapBytes.clear();

449// Read the number of bitmap bytes.

450uint64_t NumBitmapBytes;

451if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes))

452returnerror(instrprof_error::malformed,

453"number of bitmap bytes is not a valid integer");

454if (NumBitmapBytes != 0) {

455// Read each bitmap and fill our internal storage with the values.

456Record.BitmapBytes.reserve(NumBitmapBytes);

457for (uint8_t I = 0;I < NumBitmapBytes; ++I) {

458if (Line.is_at_end())

459returnerror(instrprof_error::truncated);

460uint8_t BitmapByte;

461if ((Line++)->getAsInteger(0, BitmapByte))

462returnerror(instrprof_error::malformed,

463"bitmap byte is not a valid integer");

464Record.BitmapBytes.push_back(BitmapByte);

465 }

466 }

467 }

468

469// Check if value profile data exists and read it if so.

470if (Error E = readValueProfileData(Record))

471returnerror(std::move(E));

472

473returnsuccess();

474}

475

476template <class IntPtrT>

477InstrProfKind RawInstrProfReader<IntPtrT>::getProfileKind() const{

478returngetProfileKindFromVersion(Version);

479}

480

481template <class IntPtrT>

482SmallVector<TemporalProfTraceTy> &

483RawInstrProfReader<IntPtrT>::getTemporalProfTraces(

484 std::optional<uint64_t> Weight) {

485if (TemporalProfTimestamps.empty()) {

486assert(TemporalProfTraces.empty());

487return TemporalProfTraces;

488 }

489// Sort functions by their timestamps to build the trace.

490 std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end());

491TemporalProfTraceTy Trace;

492if (Weight)

493Trace.Weight = *Weight;

494for (auto &[TimestampValue, NameRef] : TemporalProfTimestamps)

495Trace.FunctionNameRefs.push_back(NameRef);

496 TemporalProfTraces = {std::move(Trace)};

497return TemporalProfTraces;

498}

499

500template <class IntPtrT>

501boolRawInstrProfReader<IntPtrT>::hasFormat(constMemoryBuffer &DataBuffer) {

502if (DataBuffer.getBufferSize() <sizeof(uint64_t))

503returnfalse;

504uint64_t Magic =

505 *reinterpret_cast<constuint64_t *>(DataBuffer.getBufferStart());

506return RawInstrProf::getMagic<IntPtrT>() == Magic ||

507llvm::byteswap(RawInstrProf::getMagic<IntPtrT>()) == Magic;

508}

509

510template <class IntPtrT>

511Error RawInstrProfReader<IntPtrT>::readHeader() {

512if (!hasFormat(*DataBuffer))

513returnerror(instrprof_error::bad_magic);

514if (DataBuffer->getBufferSize() <sizeof(RawInstrProf::Header))

515returnerror(instrprof_error::bad_header);

516auto *Header =reinterpret_cast<constRawInstrProf::Header *>(

517 DataBuffer->getBufferStart());

518 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();

519return readHeader(*Header);

520}

521

522template <class IntPtrT>

523Error RawInstrProfReader<IntPtrT>::readNextHeader(constchar *CurrentPos) {

524constchar *End = DataBuffer->getBufferEnd();

525// Skip zero padding between profiles.

526while (CurrentPos !=End && *CurrentPos == 0)

527 ++CurrentPos;

528// If there's nothing left, we're done.

529if (CurrentPos ==End)

530return make_error<InstrProfError>(instrprof_error::eof);

531// If there isn't enough space for another header, this is probably just

532// garbage at the end of the file.

533if (CurrentPos +sizeof(RawInstrProf::Header) >End)

534return make_error<InstrProfError>(instrprof_error::malformed,

535"not enough space for another header");

536// The writer ensures each profile is padded to start at an aligned address.

537if (reinterpret_cast<size_t>(CurrentPos) %alignof(uint64_t))

538return make_error<InstrProfError>(instrprof_error::malformed,

539"insufficient padding");

540// The magic should have the same byte order as in the previous header.

541uint64_t Magic = *reinterpret_cast<constuint64_t *>(CurrentPos);

542if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))

543return make_error<InstrProfError>(instrprof_error::bad_magic);

544

545// There's another profile to read, so we need to process the header.

546auto *Header =reinterpret_cast<constRawInstrProf::Header *>(CurrentPos);

547return readHeader(*Header);

548}

549

550template <class IntPtrT>

551Error RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {

552if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart),

553StringRef(VNamesStart, VNamesEnd - VNamesStart)))

554returnerror(std::move(E));

555for (constRawInstrProf::ProfileData<IntPtrT> *I =Data;I != DataEnd; ++I) {

556constIntPtrT FPtr = swap(I->FunctionPointer);

557if (!FPtr)

558continue;

559 Symtab.mapAddress(FPtr, swap(I->NameRef));

560 }

561

562if (VTableBegin !=nullptr && VTableEnd !=nullptr) {

563for (constRawInstrProf::VTableProfileData<IntPtrT> *I = VTableBegin;

564I != VTableEnd; ++I) {

565constIntPtrT VPtr =swap(I->VTablePointer);

566if (!VPtr)

567continue;

568// Map both begin and end address to the name hash, since the instrumented

569// address could be somewhere in the middle.

570// VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks

571// the end of vtable address.

572 Symtab.mapVTableAddress(VPtr, VPtr +swap(I->VTableSize),

573swap(I->VTableNameHash));

574 }

575 }

576returnsuccess();

577}

578

579template <class IntPtrT>

580Error RawInstrProfReader<IntPtrT>::readHeader(

581constRawInstrProf::Header &Header) {

582Version =swap(Header.Version);

583if (GET_VERSION(Version) !=RawInstrProf::Version)

584returnerror(instrprof_error::raw_profile_version_mismatch,

585 ("Profile uses raw profile format version = " +

586Twine(GET_VERSION(Version)) +

587"; expected version = " +Twine(RawInstrProf::Version) +

588"\nPLEASE update this tool to version in the raw profile, or "

589"regenerate raw profile with expected version.")

590 .str());

591

592uint64_t BinaryIdSize =swap(Header.BinaryIdsSize);

593// Binary id start just after the header if exists.

594constuint8_t *BinaryIdStart =

595reinterpret_cast<constuint8_t *>(&Header) +sizeof(RawInstrProf::Header);

596constuint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;

597constuint8_t *BufferEnd = (constuint8_t *)DataBuffer->getBufferEnd();

598if (BinaryIdSize %sizeof(uint64_t) || BinaryIdEnd > BufferEnd)

599returnerror(instrprof_error::bad_header);

600ArrayRef<uint8_t> BinaryIdsBuffer(BinaryIdStart, BinaryIdSize);

601if (!BinaryIdsBuffer.empty()) {

602if (Error Err =readBinaryIdsInternal(*DataBuffer, BinaryIdsBuffer,

603 BinaryIds, getDataEndianness()))

604return Err;

605 }

606

607 CountersDelta =swap(Header.CountersDelta);

608 BitmapDelta =swap(Header.BitmapDelta);

609 NamesDelta =swap(Header.NamesDelta);

610auto NumData =swap(Header.NumData);

611auto PaddingBytesBeforeCounters =swap(Header.PaddingBytesBeforeCounters);

612auto CountersSize =swap(Header.NumCounters) * getCounterTypeSize();

613auto PaddingBytesAfterCounters =swap(Header.PaddingBytesAfterCounters);

614auto NumBitmapBytes =swap(Header.NumBitmapBytes);

615auto PaddingBytesAfterBitmapBytes =swap(Header.PaddingBytesAfterBitmapBytes);

616auto NamesSize =swap(Header.NamesSize);

617auto VTableNameSize =swap(Header.VNamesSize);

618auto NumVTables =swap(Header.NumVTables);

619 ValueKindLast =swap(Header.ValueKindLast);

620

621auto DataSize = NumData *sizeof(RawInstrProf::ProfileData<IntPtrT>);

622auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);

623auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);

624

625auto VTableSectionSize =

626 NumVTables *sizeof(RawInstrProf::VTableProfileData<IntPtrT>);

627auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);

628

629// Profile data starts after profile header and binary ids if exist.

630ptrdiff_t DataOffset =sizeof(RawInstrProf::Header) + BinaryIdSize;

631ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;

632ptrdiff_t BitmapOffset =

633 CountersOffset + CountersSize + PaddingBytesAfterCounters;

634ptrdiff_t NamesOffset =

635 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;

636ptrdiff_t VTableProfDataOffset =

637 NamesOffset + NamesSize + PaddingBytesAfterNames;

638ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +

639 PaddingBytesAfterVTableProfData;

640ptrdiff_t ValueDataOffset =

641 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;

642

643auto *Start =reinterpret_cast<constchar *>(&Header);

644if (Start + ValueDataOffset > DataBuffer->getBufferEnd())

645returnerror(instrprof_error::bad_header);

646

647if (BIDFetcher) {

648 std::vector<object::BuildID> BinaryIDs;

649if (Error E = readBinaryIds(BinaryIDs))

650return E;

651if (auto E =InstrProfCorrelator::get("", BIDFetcherCorrelatorKind,

652 BIDFetcher, BinaryIDs)

653 .moveInto(BIDFetcherCorrelator)) {

654return E;

655 }

656if (auto Err = BIDFetcherCorrelator->correlateProfileData(0))

657return Err;

658 }

659

660if (Correlator) {

661// These sizes in the raw file are zero because we constructed them in the

662// Correlator.

663if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&

664 NamesDelta == 0))

665returnerror(instrprof_error::unexpected_correlation_info);

666Data = Correlator->getDataPointer();

667 DataEnd =Data + Correlator->getDataSize();

668 NamesStart = Correlator->getNamesPointer();

669 NamesEnd = NamesStart + Correlator->getNamesSize();

670 }elseif (BIDFetcherCorrelator) {

671InstrProfCorrelatorImpl<IntPtrT> *BIDFetcherCorrelatorImpl =

672 dyn_cast_or_null<InstrProfCorrelatorImpl<IntPtrT>>(

673 BIDFetcherCorrelator.get());

674Data = BIDFetcherCorrelatorImpl->getDataPointer();

675 DataEnd =Data + BIDFetcherCorrelatorImpl->getDataSize();

676 NamesStart = BIDFetcherCorrelatorImpl->getNamesPointer();

677 NamesEnd = NamesStart + BIDFetcherCorrelatorImpl->getNamesSize();

678 }else {

679Data =reinterpret_cast<constRawInstrProf::ProfileData<IntPtrT> *>(

680 Start + DataOffset);

681 DataEnd =Data + NumData;

682 VTableBegin =

683reinterpret_cast<constRawInstrProf::VTableProfileData<IntPtrT> *>(

684 Start + VTableProfDataOffset);

685 VTableEnd = VTableBegin + NumVTables;

686 NamesStart = Start + NamesOffset;

687 NamesEnd = NamesStart + NamesSize;

688 VNamesStart = Start + VTableNameOffset;

689 VNamesEnd = VNamesStart + VTableNameSize;

690 }

691

692 CountersStart = Start + CountersOffset;

693 CountersEnd = CountersStart + CountersSize;

694 BitmapStart = Start + BitmapOffset;

695 BitmapEnd = BitmapStart + NumBitmapBytes;

696 ValueDataStart =reinterpret_cast<constuint8_t *>(Start + ValueDataOffset);

697

698 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();

699if (Error E = createSymtab(*NewSymtab))

700return E;

701

702 Symtab = std::move(NewSymtab);

703returnsuccess();

704}

705

706template <class IntPtrT>

707Error RawInstrProfReader<IntPtrT>::readName(NamedInstrProfRecord &Record) {

708Record.Name =getName(Data->NameRef);

709returnsuccess();

710}

711

712template <class IntPtrT>

713Error RawInstrProfReader<IntPtrT>::readFuncHash(NamedInstrProfRecord &Record) {

714Record.Hash =swap(Data->FuncHash);

715returnsuccess();

716}

717

718template <class IntPtrT>

719Error RawInstrProfReader<IntPtrT>::readRawCounts(

720InstrProfRecord &Record) {

721uint32_t NumCounters =swap(Data->NumCounters);

722if (NumCounters == 0)

723returnerror(instrprof_error::malformed,"number of counters is zero");

724

725ptrdiff_t CounterBaseOffset =swap(Data->CounterPtr) - CountersDelta;

726if (CounterBaseOffset < 0)

727returnerror(

728instrprof_error::malformed,

729 ("counter offset " +Twine(CounterBaseOffset) +" is negative").str());

730

731if (CounterBaseOffset >= CountersEnd - CountersStart)

732returnerror(instrprof_error::malformed,

733 ("counter offset " +Twine(CounterBaseOffset) +

734" is greater than the maximum counter offset " +

735Twine(CountersEnd - CountersStart - 1))

736 .str());

737

738uint64_t MaxNumCounters =

739 (CountersEnd - (CountersStart + CounterBaseOffset)) /

740 getCounterTypeSize();

741if (NumCounters > MaxNumCounters)

742returnerror(instrprof_error::malformed,

743 ("number of counters " +Twine(NumCounters) +

744" is greater than the maximum number of counters " +

745Twine(MaxNumCounters))

746 .str());

747

748Record.Counts.clear();

749Record.Counts.reserve(NumCounters);

750for (uint32_t I = 0;I < NumCounters;I++) {

751constchar *Ptr =

752 CountersStart + CounterBaseOffset +I * getCounterTypeSize();

753if (I == 0 && hasTemporalProfile()) {

754uint64_t TimestampValue =swap(*reinterpret_cast<constuint64_t *>(Ptr));

755if (TimestampValue != 0 &&

756 TimestampValue != std::numeric_limits<uint64_t>::max()) {

757 TemporalProfTimestamps.emplace_back(TimestampValue,

758swap(Data->NameRef));

759 TemporalProfTraceStreamSize = 1;

760 }

761if (hasSingleByteCoverage()) {

762// In coverage mode, getCounterTypeSize() returns 1 byte but our

763// timestamp field has size uint64_t. Increment I so that the next

764// iteration of this for loop points to the byte after the timestamp

765// field, i.e., I += 8.

766I += 7;

767 }

768continue;

769 }

770if (hasSingleByteCoverage()) {

771// A value of zero signifies the block is covered.

772Record.Counts.push_back(*Ptr == 0 ? 1 : 0);

773 }else {

774uint64_t CounterValue =swap(*reinterpret_cast<constuint64_t *>(Ptr));

775if (CounterValue > MaxCounterValue && Warn)

776 Warn(make_error<InstrProfError>(

777instrprof_error::counter_value_too_large,Twine(CounterValue)));

778

779Record.Counts.push_back(CounterValue);

780 }

781 }

782

783returnsuccess();

784}

785

786template <class IntPtrT>

787Error RawInstrProfReader<IntPtrT>::readRawBitmapBytes(InstrProfRecord &Record) {

788uint32_t NumBitmapBytes =swap(Data->NumBitmapBytes);

789

790Record.BitmapBytes.clear();

791Record.BitmapBytes.reserve(NumBitmapBytes);

792

793// It's possible MCDC is either not enabled or only used for some functions

794// and not others. So if we record 0 bytes, just move on.

795if (NumBitmapBytes == 0)

796returnsuccess();

797

798// BitmapDelta decreases as we advance to the next data record.

799ptrdiff_t BitmapOffset =swap(Data->BitmapPtr) - BitmapDelta;

800if (BitmapOffset < 0)

801returnerror(

802instrprof_error::malformed,

803 ("bitmap offset " +Twine(BitmapOffset) +" is negative").str());

804

805if (BitmapOffset >= BitmapEnd - BitmapStart)

806returnerror(instrprof_error::malformed,

807 ("bitmap offset " +Twine(BitmapOffset) +

808" is greater than the maximum bitmap offset " +

809Twine(BitmapEnd - BitmapStart - 1))

810 .str());

811

812uint64_t MaxNumBitmapBytes =

813 (BitmapEnd - (BitmapStart + BitmapOffset)) /sizeof(uint8_t);

814if (NumBitmapBytes > MaxNumBitmapBytes)

815returnerror(instrprof_error::malformed,

816 ("number of bitmap bytes " +Twine(NumBitmapBytes) +

817" is greater than the maximum number of bitmap bytes " +

818Twine(MaxNumBitmapBytes))

819 .str());

820

821for (uint32_t I = 0;I < NumBitmapBytes;I++) {

822constchar *Ptr = BitmapStart + BitmapOffset +I;

823Record.BitmapBytes.push_back(swap(*Ptr));

824 }

825

826returnsuccess();

827}

828

829template <class IntPtrT>

830Error RawInstrProfReader<IntPtrT>::readValueProfilingData(

831InstrProfRecord &Record) {

832Record.clearValueData();

833 CurValueDataSize = 0;

834// Need to match the logic in value profile dumper code in compiler-rt:

835uint32_t NumValueKinds = 0;

836for (uint32_t I = 0;I < IPVK_Last + 1;I++)

837 NumValueKinds += (Data->NumValueSites[I] != 0);

838

839if (!NumValueKinds)

840returnsuccess();

841

842Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr =

843 ValueProfData::getValueProfData(

844 ValueDataStart, (constunsignedchar *)DataBuffer->getBufferEnd(),

845 getDataEndianness());

846

847if (Error E = VDataPtrOrErr.takeError())

848return E;

849

850// Note that besides deserialization, this also performs the conversion for

851// indirect call targets. The function pointers from the raw profile are

852// remapped into function name hashes.

853 VDataPtrOrErr.get()->deserializeTo(Record, Symtab.get());

854 CurValueDataSize = VDataPtrOrErr.get()->getSize();

855returnsuccess();

856}

857

858template <class IntPtrT>

859Error RawInstrProfReader<IntPtrT>::readNextRecord(NamedInstrProfRecord &Record) {

860// Keep reading profiles that consist of only headers and no profile data and

861// counters.

862while (atEnd())

863// At this point, ValueDataStart field points to the next header.

864if (Error E = readNextHeader(getNextHeaderPos()))

865returnerror(std::move(E));

866

867// Read name and set it in Record.

868if (Error E = readName(Record))

869returnerror(std::move(E));

870

871// Read FuncHash and set it in Record.

872if (Error E = readFuncHash(Record))

873returnerror(std::move(E));

874

875// Read raw counts and set Record.

876if (Error E = readRawCounts(Record))

877returnerror(std::move(E));

878

879// Read raw bitmap bytes and set Record.

880if (Error E = readRawBitmapBytes(Record))

881returnerror(std::move(E));

882

883// Read value data and set Record.

884if (Error E = readValueProfilingData(Record))

885returnerror(std::move(E));

886

887// Iterate.

888 advanceData();

889returnsuccess();

890}

891

892template <class IntPtrT>

893Error RawInstrProfReader<IntPtrT>::readBinaryIds(

894 std::vector<llvm::object::BuildID> &BinaryIds) {

895 BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(),

896 this->BinaryIds.end());

897returnError::success();

898}

899

900template <class IntPtrT>

901Error RawInstrProfReader<IntPtrT>::printBinaryIds(raw_ostream &OS) {

902if (!BinaryIds.empty())

903printBinaryIdsInternal(OS, BinaryIds);

904returnError::success();

905}

906

907namespacellvm {

908

909templateclassRawInstrProfReader<uint32_t>;

910templateclassRawInstrProfReader<uint64_t>;

911

912}// end namespace llvm

913

914InstrProfLookupTrait::hash_value_type

915InstrProfLookupTrait::ComputeHash(StringRef K) {

916returnIndexedInstrProf::ComputeHash(HashType, K);

917}

918

919usingdata_type =InstrProfLookupTrait::data_type;

920usingoffset_type =InstrProfLookupTrait::offset_type;

921

922boolInstrProfLookupTrait::readValueProfilingData(

923constunsignedchar *&D,constunsignedchar *constEnd) {

924Expected<std::unique_ptr<ValueProfData>> VDataPtrOrErr =

925 ValueProfData::getValueProfData(D,End, ValueProfDataEndianness);

926

927if (VDataPtrOrErr.takeError())

928returnfalse;

929

930 VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(),nullptr);

931D += VDataPtrOrErr.get()->TotalSize;

932

933returntrue;

934}

935

936data_type InstrProfLookupTrait::ReadData(StringRef K,constunsignedchar *D,

937offset_type N) {

938using namespacesupport;

939

940// Check if the data is corrupt. If so, don't try to read it.

941if (N %sizeof(uint64_t))

942returndata_type();

943

944 DataBuffer.clear();

945 std::vector<uint64_t> CounterBuffer;

946 std::vector<uint8_t> BitmapByteBuffer;

947

948constunsignedchar *End =D +N;

949while (D <End) {

950// Read hash.

951if (D +sizeof(uint64_t) >=End)

952returndata_type();

953uint64_t Hash = endian::readNext<uint64_t, llvm::endianness::little>(D);

954

955// Initialize number of counters for GET_VERSION(FormatVersion) == 1.

956uint64_t CountsSize =N /sizeof(uint64_t) - 1;

957// If format version is different then read the number of counters.

958if (GET_VERSION(FormatVersion) !=IndexedInstrProf::ProfVersion::Version1) {

959if (D +sizeof(uint64_t) >End)

960returndata_type();

961 CountsSize = endian::readNext<uint64_t, llvm::endianness::little>(D);

962 }

963// Read counter values.

964if (D + CountsSize *sizeof(uint64_t) >End)

965returndata_type();

966

967 CounterBuffer.clear();

968 CounterBuffer.reserve(CountsSize);

969for (uint64_t J = 0; J < CountsSize; ++J)

970 CounterBuffer.push_back(

971 endian::readNext<uint64_t, llvm::endianness::little>(D));

972

973// Read bitmap bytes for GET_VERSION(FormatVersion) > 10.

974if (GET_VERSION(FormatVersion) >IndexedInstrProf::ProfVersion::Version10) {

975uint64_t BitmapBytes = 0;

976if (D +sizeof(uint64_t) >End)

977returndata_type();

978 BitmapBytes = endian::readNext<uint64_t, llvm::endianness::little>(D);

979// Read bitmap byte values.

980if (D + BitmapBytes *sizeof(uint8_t) >End)

981returndata_type();

982 BitmapByteBuffer.clear();

983 BitmapByteBuffer.reserve(BitmapBytes);

984for (uint64_t J = 0; J < BitmapBytes; ++J)

985 BitmapByteBuffer.push_back(static_cast<uint8_t>(

986 endian::readNext<uint64_t, llvm::endianness::little>(D)));

987 }

988

989 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer),

990 std::move(BitmapByteBuffer));

991

992// Read value profiling data.

993if (GET_VERSION(FormatVersion) >IndexedInstrProf::ProfVersion::Version2 &&

994 !readValueProfilingData(D,End)) {

995 DataBuffer.clear();

996returndata_type();

997 }

998 }

999return DataBuffer;

1000}

1001

1002template <typename HashTableImpl>

1003Error InstrProfReaderIndex<HashTableImpl>::getRecords(

1004StringRef FuncName,ArrayRef<NamedInstrProfRecord> &Data) {

1005auto Iter = HashTable->find(FuncName);

1006if (Iter == HashTable->end())

1007return make_error<InstrProfError>(instrprof_error::unknown_function);

1008

1009Data = (*Iter);

1010if (Data.empty())

1011return make_error<InstrProfError>(instrprof_error::malformed,

1012"profile data is empty");

1013

1014returnError::success();

1015}

1016

1017template <typename HashTableImpl>

1018Error InstrProfReaderIndex<HashTableImpl>::getRecords(

1019ArrayRef<NamedInstrProfRecord> &Data) {

1020if (atEnd())

1021return make_error<InstrProfError>(instrprof_error::eof);

1022

1023Data = *RecordIterator;

1024

1025if (Data.empty())

1026return make_error<InstrProfError>(instrprof_error::malformed,

1027"profile data is empty");

1028

1029returnError::success();

1030}

1031

1032template <typename HashTableImpl>

1033InstrProfReaderIndex<HashTableImpl>::InstrProfReaderIndex(

1034constunsignedchar *Buckets,constunsignedchar *const Payload,

1035constunsignedchar *constBase,IndexedInstrProf::HashT HashType,

1036uint64_t Version) {

1037 FormatVersion =Version;

1038 HashTable.reset(HashTableImpl::Create(

1039 Buckets, Payload,Base,

1040typename HashTableImpl::InfoType(HashType,Version)));

1041 RecordIterator = HashTable->data_begin();

1042}

1043

1044template <typename HashTableImpl>

1045InstrProfKind InstrProfReaderIndex<HashTableImpl>::getProfileKind() const{

1046returngetProfileKindFromVersion(FormatVersion);

1047}

1048

1049namespace{

1050/// A remapper that does not apply any remappings.

1051classInstrProfReaderNullRemapper :publicInstrProfReaderRemapper {

1052InstrProfReaderIndexBase &Underlying;

1053

1054public:

1055 InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)

1056 : Underlying(Underlying) {}

1057

1058Error getRecords(StringRef FuncName,

1059ArrayRef<NamedInstrProfRecord> &Data) override{

1060return Underlying.getRecords(FuncName, Data);

1061 }

1062};

1063}// namespace

1064

1065/// A remapper that applies remappings based on a symbol remapping file.

1066template <typename HashTableImpl>

1067classllvm::InstrProfReaderItaniumRemapper

1068 :publicInstrProfReaderRemapper {

1069public:

1070InstrProfReaderItaniumRemapper(

1071 std::unique_ptr<MemoryBuffer> RemapBuffer,

1072InstrProfReaderIndex<HashTableImpl> &Underlying)

1073 : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {

1074 }

1075

1076 /// Extract the original function name from a PGO function name.

1077staticStringRef extractName(StringRef Name) {

1078// We can have multiple pieces separated by kGlobalIdentifierDelimiter (

1079// semicolon now and colon in older profiles); there can be pieces both

1080// before and after the mangled name. Find the first part that starts with

1081// '_Z'; we'll assume that's the mangled name we want.

1082 std::pair<StringRef, StringRef> Parts = {StringRef(),Name};

1083while (true) {

1084 Parts = Parts.second.split(GlobalIdentifierDelimiter);

1085if (Parts.first.starts_with("_Z"))

1086return Parts.first;

1087if (Parts.second.empty())

1088returnName;

1089 }

1090 }

1091

1092 /// Given a mangled name extracted from a PGO function name, and a new

1093 /// form for that mangled name, reconstitute the name.

1094staticvoidreconstituteName(StringRef OrigName,StringRef ExtractedName,

1095StringRef Replacement,

1096SmallVectorImpl<char> &Out) {

1097 Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());

1098 Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());

1099 Out.insert(Out.end(), Replacement.begin(), Replacement.end());

1100 Out.insert(Out.end(), ExtractedName.end(), OrigName.end());

1101 }

1102

1103Error populateRemappings() override{

1104if (Error E = Remappings.read(*RemapBuffer))

1105return E;

1106for (StringRef Name : Underlying.HashTable->keys()) {

1107StringRef RealName =extractName(Name);

1108if (auto Key = Remappings.insert(RealName)) {

1109// FIXME: We could theoretically map the same equivalence class to

1110// multiple names in the profile data. If that happens, we should

1111// return NamedInstrProfRecords from all of them.

1112 MappedNames.insert({Key, RealName});

1113 }

1114 }

1115returnError::success();

1116 }

1117

1118Error getRecords(StringRef FuncName,

1119ArrayRef<NamedInstrProfRecord> &Data) override{

1120StringRef RealName =extractName(FuncName);

1121if (auto Key = Remappings.lookup(RealName)) {

1122StringRef Remapped = MappedNames.lookup(Key);

1123if (!Remapped.empty()) {

1124if (RealName.begin() == FuncName.begin() &&

1125 RealName.end() == FuncName.end())

1126 FuncName = Remapped;

1127else {

1128// Try rebuilding the name from the given remapping.

1129SmallString<256> Reconstituted;

1130reconstituteName(FuncName, RealName, Remapped, Reconstituted);

1131Error E = Underlying.getRecords(Reconstituted,Data);

1132if (!E)

1133return E;

1134

1135// If we failed because the name doesn't exist, fall back to asking

1136// about the original name.

1137if (Error Unhandled =handleErrors(

1138 std::move(E), [](std::unique_ptr<InstrProfError> Err) {

1139return Err->get() ==instrprof_error::unknown_function

1140 ?Error::success()

1141 :Error(std::move(Err));

1142 }))

1143return Unhandled;

1144 }

1145 }

1146 }

1147return Underlying.getRecords(FuncName,Data);

1148 }

1149

1150private:

1151 /// The memory buffer containing the remapping configuration. Remappings

1152 /// holds pointers into this buffer.

1153 std::unique_ptr<MemoryBuffer> RemapBuffer;

1154

1155 /// The mangling remapper.

1156SymbolRemappingReader Remappings;

1157

1158 /// Mapping from mangled name keys to the name used for the key in the

1159 /// profile data.

1160 /// FIXME: Can we store a location within the on-disk hash table instead of

1161 /// redoing lookup?

1162DenseMap<SymbolRemappingReader::Key, StringRef> MappedNames;

1163

1164 /// The real profile data reader.

1165InstrProfReaderIndex<HashTableImpl> &Underlying;

1166};

1167

1168boolIndexedInstrProfReader::hasFormat(constMemoryBuffer &DataBuffer) {

1169using namespacesupport;

1170

1171if (DataBuffer.getBufferSize() < 8)

1172returnfalse;

1173uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(

1174 DataBuffer.getBufferStart());

1175// Verify that it's magical.

1176return Magic ==IndexedInstrProf::Magic;

1177}

1178

1179constunsignedchar *

1180IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,

1181constunsignedchar *Cur,bool UseCS) {

1182using namespaceIndexedInstrProf;

1183using namespacesupport;

1184

1185if (Version >=IndexedInstrProf::Version4) {

1186constIndexedInstrProf::Summary *SummaryInLE =

1187reinterpret_cast<constIndexedInstrProf::Summary *>(Cur);

1188uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(

1189 SummaryInLE->NumSummaryFields);

1190uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(

1191 SummaryInLE->NumCutoffEntries);

1192uint32_t SummarySize =

1193IndexedInstrProf::Summary::getSize(NFields, NEntries);

1194 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =

1195IndexedInstrProf::allocSummary(SummarySize);

1196

1197constuint64_t *Src =reinterpret_cast<constuint64_t *>(SummaryInLE);

1198uint64_t *Dst =reinterpret_cast<uint64_t *>(SummaryData.get());

1199for (unsignedI = 0;I < SummarySize /sizeof(uint64_t);I++)

1200 Dst[I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[I]);

1201

1202SummaryEntryVector DetailedSummary;

1203for (unsignedI = 0;I < SummaryData->NumCutoffEntries;I++) {

1204constIndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);

1205 DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,

1206 Ent.NumBlocks);

1207 }

1208 std::unique_ptr<llvm::ProfileSummary> &Summary =

1209 UseCS ? this->CS_Summary : this->Summary;

1210

1211// initialize InstrProfSummary using the SummaryData from disk.

1212 Summary = std::make_unique<ProfileSummary>(

1213 UseCS ?ProfileSummary::PSK_CSInstr :ProfileSummary::PSK_Instr,

1214 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),

1215 SummaryData->get(Summary::MaxBlockCount),

1216 SummaryData->get(Summary::MaxInternalBlockCount),

1217 SummaryData->get(Summary::MaxFunctionCount),

1218 SummaryData->get(Summary::TotalNumBlocks),

1219 SummaryData->get(Summary::TotalNumFunctions));

1220return Cur + SummarySize;

1221 }else {

1222// The older versions do not support a profile summary. This just computes

1223// an empty summary, which will not result in accurate hot/cold detection.

1224// We would need to call addRecord for all NamedInstrProfRecords to get the

1225// correct summary. However, this version is old (prior to early 2016) and

1226// has not been supporting an accurate summary for several years.

1227InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);

1228 Summary = Builder.getSummary();

1229return Cur;

1230 }

1231}

1232

1233Error IndexedMemProfReader::deserializeV2(constunsignedchar *Start,

1234constunsignedchar *Ptr) {

1235// The value returned from RecordTableGenerator.Emit.

1236constuint64_t RecordTableOffset =

1237 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1238// The offset in the stream right before invoking

1239// FrameTableGenerator.Emit.

1240constuint64_t FramePayloadOffset =

1241 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1242// The value returned from FrameTableGenerator.Emit.

1243constuint64_t FrameTableOffset =

1244 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1245

1246// The offset in the stream right before invoking

1247// CallStackTableGenerator.Emit.

1248uint64_t CallStackPayloadOffset = 0;

1249// The value returned from CallStackTableGenerator.Emit.

1250uint64_t CallStackTableOffset = 0;

1251if (Version >=memprof::Version2) {

1252 CallStackPayloadOffset =

1253 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1254 CallStackTableOffset =

1255 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1256 }

1257

1258// Read the schema.

1259auto SchemaOr =memprof::readMemProfSchema(Ptr);

1260if (!SchemaOr)

1261return SchemaOr.takeError();

1262 Schema = SchemaOr.get();

1263

1264// Now initialize the table reader with a pointer into data buffer.

1265 MemProfRecordTable.reset(MemProfRecordHashTable::Create(

1266/*Buckets=*/Start + RecordTableOffset,

1267/*Payload=*/Ptr,

1268/*Base=*/Start,memprof::RecordLookupTrait(Version, Schema)));

1269

1270// Initialize the frame table reader with the payload and bucket offsets.

1271 MemProfFrameTable.reset(MemProfFrameHashTable::Create(

1272/*Buckets=*/Start + FrameTableOffset,

1273/*Payload=*/Start + FramePayloadOffset,

1274/*Base=*/Start));

1275

1276if (Version >=memprof::Version2)

1277 MemProfCallStackTable.reset(MemProfCallStackHashTable::Create(

1278/*Buckets=*/Start + CallStackTableOffset,

1279/*Payload=*/Start + CallStackPayloadOffset,

1280/*Base=*/Start));

1281

1282returnError::success();

1283}

1284

1285Error IndexedMemProfReader::deserializeV3(constunsignedchar *Start,

1286constunsignedchar *Ptr) {

1287// The offset in the stream right before invoking

1288// CallStackTableGenerator.Emit.

1289constuint64_t CallStackPayloadOffset =

1290 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1291// The offset in the stream right before invoking RecordTableGenerator.Emit.

1292constuint64_t RecordPayloadOffset =

1293 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1294// The value returned from RecordTableGenerator.Emit.

1295constuint64_t RecordTableOffset =

1296 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1297

1298// Read the schema.

1299auto SchemaOr =memprof::readMemProfSchema(Ptr);

1300if (!SchemaOr)

1301return SchemaOr.takeError();

1302 Schema = SchemaOr.get();

1303

1304 FrameBase =Ptr;

1305 CallStackBase = Start + CallStackPayloadOffset;

1306

1307// Compute the number of elements in the radix tree array. Since we use this

1308// to reserve enough bits in a BitVector, it's totally OK if we overestimate

1309// this number a little bit because of padding just before the next section.

1310 RadixTreeSize = (RecordPayloadOffset - CallStackPayloadOffset) /

1311sizeof(memprof::LinearFrameId);

1312

1313// Now initialize the table reader with a pointer into data buffer.

1314 MemProfRecordTable.reset(MemProfRecordHashTable::Create(

1315/*Buckets=*/Start + RecordTableOffset,

1316/*Payload=*/Start + RecordPayloadOffset,

1317/*Base=*/Start,memprof::RecordLookupTrait(memprof::Version3, Schema)));

1318

1319returnError::success();

1320}

1321

1322Error IndexedMemProfReader::deserialize(constunsignedchar *Start,

1323uint64_t MemProfOffset) {

1324constunsignedchar *Ptr = Start + MemProfOffset;

1325

1326// Read the MemProf version number.

1327constuint64_t FirstWord =

1328 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1329

1330if (FirstWord ==memprof::Version2 || FirstWord ==memprof::Version3) {

1331// Everything is good. We can proceed to deserialize the rest.

1332 Version =static_cast<memprof::IndexedVersion>(FirstWord);

1333 }else {

1334return make_error<InstrProfError>(

1335instrprof_error::unsupported_version,

1336formatv("MemProf version {} not supported; "

1337"requires version between {} and {}, inclusive",

1338 FirstWord,memprof::MinimumSupportedVersion,

1339memprof::MaximumSupportedVersion));

1340 }

1341

1342switch (Version) {

1343casememprof::Version2:

1344if (Error E = deserializeV2(Start,Ptr))

1345return E;

1346break;

1347casememprof::Version3:

1348if (Error E = deserializeV3(Start,Ptr))

1349return E;

1350break;

1351 }

1352

1353returnError::success();

1354}

1355

1356Error IndexedInstrProfReader::readHeader() {

1357using namespacesupport;

1358

1359constunsignedchar *Start =

1360 (constunsignedchar *)DataBuffer->getBufferStart();

1361constunsignedchar *Cur = Start;

1362if ((constunsignedchar *)DataBuffer->getBufferEnd() - Cur < 24)

1363returnerror(instrprof_error::truncated);

1364

1365auto HeaderOr =IndexedInstrProf::Header::readFromBuffer(Start);

1366if (!HeaderOr)

1367return HeaderOr.takeError();

1368

1369constIndexedInstrProf::Header *Header = &HeaderOr.get();

1370 Cur += Header->size();

1371

1372 Cur = readSummary((IndexedInstrProf::ProfVersion)Header->Version, Cur,

1373/* UseCS */false);

1374if (Header->Version & VARIANT_MASK_CSIR_PROF)

1375 Cur = readSummary((IndexedInstrProf::ProfVersion)Header->Version, Cur,

1376/* UseCS */true);

1377// Read the hash type and start offset.

1378IndexedInstrProf::HashT HashType =

1379static_cast<IndexedInstrProf::HashT>(Header->HashType);

1380if (HashType >IndexedInstrProf::HashT::Last)

1381returnerror(instrprof_error::unsupported_hash_type);

1382

1383// The hash table with profile counts comes next.

1384auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(

1385 Start + Header->HashOffset, Cur, Start, HashType, Header->Version);

1386

1387// The MemProfOffset field in the header is only valid when the format

1388// version is higher than 8 (when it was introduced).

1389if (Header->getIndexedProfileVersion() >= 8 &&

1390 Header->Version & VARIANT_MASK_MEMPROF) {

1391if (Error E = MemProfReader.deserialize(Start, Header->MemProfOffset))

1392return E;

1393 }

1394

1395// BinaryIdOffset field in the header is only valid when the format version

1396// is higher than 9 (when it was introduced).

1397if (Header->getIndexedProfileVersion() >= 9) {

1398constunsignedchar *Ptr = Start + Header->BinaryIdOffset;

1399// Read binary ids size.

1400uint64_t BinaryIdsSize =

1401 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1402if (BinaryIdsSize %sizeof(uint64_t))

1403returnerror(instrprof_error::bad_header);

1404// Set the binary ids start.

1405 BinaryIdsBuffer =ArrayRef<uint8_t>(Ptr, BinaryIdsSize);

1406if (Ptr > (constunsignedchar *)DataBuffer->getBufferEnd())

1407return make_error<InstrProfError>(instrprof_error::malformed,

1408"corrupted binary ids");

1409 }

1410

1411if (Header->getIndexedProfileVersion() >= 12) {

1412constunsignedchar *Ptr = Start + Header->VTableNamesOffset;

1413

1414uint64_t CompressedVTableNamesLen =

1415 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1416

1417// Writer first writes the length of compressed string, and then the actual

1418// content.

1419constchar *VTableNamePtr = (constchar *)Ptr;

1420if (VTableNamePtr > (constchar *)DataBuffer->getBufferEnd())

1421return make_error<InstrProfError>(instrprof_error::truncated);

1422

1423 VTableName =StringRef(VTableNamePtr, CompressedVTableNamesLen);

1424 }

1425

1426if (Header->getIndexedProfileVersion() >= 10 &&

1427 Header->Version & VARIANT_MASK_TEMPORAL_PROF) {

1428constunsignedchar *Ptr = Start + Header->TemporalProfTracesOffset;

1429constauto *PtrEnd = (constunsignedchar *)DataBuffer->getBufferEnd();

1430// Expect at least two 64 bit fields: NumTraces, and TraceStreamSize

1431if (Ptr + 2 *sizeof(uint64_t) > PtrEnd)

1432returnerror(instrprof_error::truncated);

1433constuint64_t NumTraces =

1434 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1435TemporalProfTraceStreamSize =

1436 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1437for (unsigned i = 0; i < NumTraces; i++) {

1438// Expect at least two 64 bit fields: Weight and NumFunctions

1439if (Ptr + 2 *sizeof(uint64_t) > PtrEnd)

1440returnerror(instrprof_error::truncated);

1441TemporalProfTraceTy Trace;

1442Trace.Weight =

1443 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1444constuint64_t NumFunctions =

1445 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1446// Expect at least NumFunctions 64 bit fields

1447if (Ptr + NumFunctions *sizeof(uint64_t) > PtrEnd)

1448returnerror(instrprof_error::truncated);

1449for (unsigned j = 0; j < NumFunctions; j++) {

1450constuint64_t NameRef =

1451 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);

1452Trace.FunctionNameRefs.push_back(NameRef);

1453 }

1454TemporalProfTraces.push_back(std::move(Trace));

1455 }

1456 }

1457

1458// Load the remapping table now if requested.

1459if (RemappingBuffer) {

1460 Remapper =

1461 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(

1462 std::move(RemappingBuffer), *IndexPtr);

1463if (Error E = Remapper->populateRemappings())

1464return E;

1465 }else {

1466 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);

1467 }

1468 Index = std::move(IndexPtr);

1469

1470returnsuccess();

1471}

1472

1473InstrProfSymtab &IndexedInstrProfReader::getSymtab() {

1474if (Symtab)

1475return *Symtab;

1476

1477auto NewSymtab = std::make_unique<InstrProfSymtab>();

1478

1479if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(VTableName)) {

1480auto [ErrCode, Msg] =InstrProfError::take(std::move(E));

1481consumeError(error(ErrCode, Msg));

1482 }

1483

1484// finalizeSymtab is called inside populateSymtab.

1485if (Error E = Index->populateSymtab(*NewSymtab)) {

1486auto [ErrCode, Msg] =InstrProfError::take(std::move(E));

1487consumeError(error(ErrCode, Msg));

1488 }

1489

1490Symtab = std::move(NewSymtab);

1491return *Symtab;

1492}

1493

1494Expected<InstrProfRecord>IndexedInstrProfReader::getInstrProfRecord(

1495StringRef FuncName,uint64_t FuncHash,StringRef DeprecatedFuncName,

1496uint64_t *MismatchedFuncSum) {

1497ArrayRef<NamedInstrProfRecord>Data;

1498uint64_t FuncSum = 0;

1499auto Err = Remapper->getRecords(FuncName,Data);

1500if (Err) {

1501// If we don't find FuncName, try DeprecatedFuncName to handle profiles

1502// built by older compilers.

1503auto Err2 =

1504handleErrors(std::move(Err), [&](constInstrProfError &IE) ->Error {

1505if (IE.get() !=instrprof_error::unknown_function)

1506return make_error<InstrProfError>(IE);

1507if (auto Err = Remapper->getRecords(DeprecatedFuncName,Data))

1508return Err;

1509returnError::success();

1510 });

1511if (Err2)

1512return std::move(Err2);

1513 }

1514// Found it. Look for counters with the right hash.

1515

1516// A flag to indicate if the records are from the same type

1517// of profile (i.e cs vs nocs).

1518bool CSBitMatch =false;

1519auto getFuncSum = [](ArrayRef<uint64_t> Counts) {

1520uint64_t ValueSum = 0;

1521for (uint64_t CountValue : Counts) {

1522if (CountValue == (uint64_t)-1)

1523continue;

1524// Handle overflow -- if that happens, return max.

1525if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)

1526return std::numeric_limits<uint64_t>::max();

1527 ValueSum += CountValue;

1528 }

1529return ValueSum;

1530 };

1531

1532for (constNamedInstrProfRecord &I :Data) {

1533// Check for a match and fill the vector if there is one.

1534if (I.Hash == FuncHash)

1535return std::move(I);

1536if (NamedInstrProfRecord::hasCSFlagInHash(I.Hash) ==

1537NamedInstrProfRecord::hasCSFlagInHash(FuncHash)) {

1538 CSBitMatch =true;

1539if (MismatchedFuncSum ==nullptr)

1540continue;

1541 FuncSum = std::max(FuncSum, getFuncSum(I.Counts));

1542 }

1543 }

1544if (CSBitMatch) {

1545if (MismatchedFuncSum !=nullptr)

1546 *MismatchedFuncSum = FuncSum;

1547returnerror(instrprof_error::hash_mismatch);

1548 }

1549returnerror(instrprof_error::unknown_function);

1550}

1551

1552staticExpected<memprof::MemProfRecord>

1553getMemProfRecordV2(constmemprof::IndexedMemProfRecord &IndexedRecord,

1554MemProfFrameHashTable &MemProfFrameTable,

1555MemProfCallStackHashTable &MemProfCallStackTable) {

1556memprof::FrameIdConverter<MemProfFrameHashTable> FrameIdConv(

1557 MemProfFrameTable);

1558

1559memprof::CallStackIdConverter<MemProfCallStackHashTable> CSIdConv(

1560 MemProfCallStackTable, FrameIdConv);

1561

1562memprof::MemProfRecord Record = IndexedRecord.toMemProfRecord(CSIdConv);

1563

1564// Check that all call stack ids were successfully converted to call stacks.

1565if (CSIdConv.LastUnmappedId) {

1566return make_error<InstrProfError>(

1567instrprof_error::hash_mismatch,

1568"memprof call stack not found for call stack id " +

1569Twine(*CSIdConv.LastUnmappedId));

1570 }

1571

1572// Check that all frame ids were successfully converted to frames.

1573if (FrameIdConv.LastUnmappedId) {

1574return make_error<InstrProfError>(instrprof_error::hash_mismatch,

1575"memprof frame not found for frame id " +

1576Twine(*FrameIdConv.LastUnmappedId));

1577 }

1578

1579returnRecord;

1580}

1581

1582staticExpected<memprof::MemProfRecord>

1583getMemProfRecordV3(constmemprof::IndexedMemProfRecord &IndexedRecord,

1584constunsignedchar *FrameBase,

1585constunsignedchar *CallStackBase) {

1586memprof::LinearFrameIdConverter FrameIdConv(FrameBase);

1587memprof::LinearCallStackIdConverter CSIdConv(CallStackBase, FrameIdConv);

1588memprof::MemProfRecord Record = IndexedRecord.toMemProfRecord(CSIdConv);

1589returnRecord;

1590}

1591

1592Expected<memprof::MemProfRecord>

1593IndexedMemProfReader::getMemProfRecord(constuint64_t FuncNameHash) const{

1594// TODO: Add memprof specific errors.

1595if (MemProfRecordTable ==nullptr)

1596return make_error<InstrProfError>(instrprof_error::invalid_prof,

1597"no memprof data available in profile");

1598auto Iter = MemProfRecordTable->find(FuncNameHash);

1599if (Iter == MemProfRecordTable->end())

1600return make_error<InstrProfError>(

1601instrprof_error::unknown_function,

1602"memprof record not found for function hash " +Twine(FuncNameHash));

1603

1604constmemprof::IndexedMemProfRecord &IndexedRecord = *Iter;

1605switch (Version) {

1606casememprof::Version2:

1607assert(MemProfFrameTable &&"MemProfFrameTable must be available");

1608assert(MemProfCallStackTable &&"MemProfCallStackTable must be available");

1609returngetMemProfRecordV2(IndexedRecord, *MemProfFrameTable,

1610 *MemProfCallStackTable);

1611casememprof::Version3:

1612assert(!MemProfFrameTable &&"MemProfFrameTable must not be available");

1613assert(!MemProfCallStackTable &&

1614"MemProfCallStackTable must not be available");

1615assert(FrameBase &&"FrameBase must be available");

1616assert(CallStackBase &&"CallStackBase must be available");

1617returngetMemProfRecordV3(IndexedRecord, FrameBase, CallStackBase);

1618 }

1619

1620return make_error<InstrProfError>(

1621instrprof_error::unsupported_version,

1622formatv("MemProf version {} not supported; "

1623"requires version between {} and {}, inclusive",

1624 Version,memprof::MinimumSupportedVersion,

1625memprof::MaximumSupportedVersion));

1626}

1627

1628DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>>

1629IndexedMemProfReader::getMemProfCallerCalleePairs() const{

1630assert(MemProfRecordTable);

1631assert(Version ==memprof::Version3);

1632

1633memprof::LinearFrameIdConverter FrameIdConv(FrameBase);

1634memprof::CallerCalleePairExtractor Extractor(CallStackBase, FrameIdConv,

1635 RadixTreeSize);

1636

1637// The set of linear call stack IDs that we need to traverse from. We expect

1638// the set to be dense, so we use a BitVector.

1639BitVector Worklist(RadixTreeSize);

1640

1641// Collect the set of linear call stack IDs. Since we expect a lot of

1642// duplicates, we first collect them in the form of a bit vector before

1643// processing them.

1644for (constmemprof::IndexedMemProfRecord &IndexedRecord :

1645 MemProfRecordTable->data()) {

1646for (constmemprof::IndexedAllocationInfo &IndexedAI :

1647 IndexedRecord.AllocSites)

1648 Worklist.set(IndexedAI.CSId);

1649 }

1650

1651// Collect caller-callee pairs for each linear call stack ID in Worklist.

1652for (unsigned CS : Worklist.set_bits())

1653 Extractor(CS);

1654

1655DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>> Pairs =

1656 std::move(Extractor.CallerCalleePairs);

1657

1658// Sort each call list by the source location.

1659for (auto &[CallerGUID, CallList] : Pairs) {

1660llvm::sort(CallList);

1661 CallList.erase(llvm::unique(CallList), CallList.end());

1662 }

1663

1664return Pairs;

1665}

1666

1667memprof::AllMemProfData IndexedMemProfReader::getAllMemProfData() const{

1668memprof::AllMemProfData AllMemProfData;

1669 AllMemProfData.HeapProfileRecords.reserve(

1670 MemProfRecordTable->getNumEntries());

1671for (uint64_t Key : MemProfRecordTable->keys()) {

1672autoRecord =getMemProfRecord(Key);

1673if (Record.takeError())

1674continue;

1675memprof::GUIDMemProfRecordPair Pair;

1676 Pair.GUID = Key;

1677 Pair.Record = std::move(*Record);

1678 AllMemProfData.HeapProfileRecords.push_back(std::move(Pair));

1679 }

1680return AllMemProfData;

1681}

1682

1683Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName,

1684uint64_t FuncHash,

1685 std::vector<uint64_t> &Counts) {

1686Expected<InstrProfRecord>Record =getInstrProfRecord(FuncName, FuncHash);

1687if (Error E =Record.takeError())

1688returnerror(std::move(E));

1689

1690 Counts =Record.get().Counts;

1691returnsuccess();

1692}

1693

1694Error IndexedInstrProfReader::getFunctionBitmap(StringRef FuncName,

1695uint64_t FuncHash,

1696BitVector &Bitmap) {

1697Expected<InstrProfRecord>Record =getInstrProfRecord(FuncName, FuncHash);

1698if (Error E =Record.takeError())

1699returnerror(std::move(E));

1700

1701constauto &BitmapBytes =Record.get().BitmapBytes;

1702size_tI = 0, E = BitmapBytes.size();

1703 Bitmap.resize(E * CHAR_BIT);

1704BitVector::apply(

1705 [&](autoX) {

1706usingXTy =decltype(X);

1707alignas(XTy)uint8_t W[sizeof(X)];

1708size_tN = std::min(E -I,sizeof(W));

1709 std::memset(W, 0,sizeof(W));

1710 std::memcpy(W, &BitmapBytes[I],N);

1711I +=N;

1712returnsupport::endian::read<XTy,llvm::endianness::little,

1713support::aligned>(W);

1714 },

1715 Bitmap, Bitmap);

1716assert(I == E);

1717

1718returnsuccess();

1719}

1720

1721Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {

1722ArrayRef<NamedInstrProfRecord>Data;

1723

1724Error E = Index->getRecords(Data);

1725if (E)

1726returnerror(std::move(E));

1727

1728Record =Data[RecordIndex++];

1729if (RecordIndex >=Data.size()) {

1730 Index->advanceToNextKey();

1731 RecordIndex = 0;

1732 }

1733returnsuccess();

1734}

1735

1736Error IndexedInstrProfReader::readBinaryIds(

1737 std::vector<llvm::object::BuildID> &BinaryIds) {

1738returnreadBinaryIdsInternal(*DataBuffer, BinaryIdsBuffer, BinaryIds,

1739llvm::endianness::little);

1740}

1741

1742Error IndexedInstrProfReader::printBinaryIds(raw_ostream &OS) {

1743 std::vector<llvm::object::BuildID> BinaryIds;

1744if (Error E =readBinaryIds(BinaryIds))

1745return E;

1746printBinaryIdsInternal(OS, BinaryIds);

1747returnError::success();

1748}

1749

1750voidInstrProfReader::accumulateCounts(CountSumOrPercent &Sum,bool IsCS) {

1751uint64_t NumFuncs = 0;

1752for (constauto &Func : *this) {

1753if (isIRLevelProfile()) {

1754bool FuncIsCS =NamedInstrProfRecord::hasCSFlagInHash(Func.Hash);

1755if (FuncIsCS != IsCS)

1756continue;

1757 }

1758 Func.accumulateCounts(Sum);

1759 ++NumFuncs;

1760 }

1761 Sum.NumEntries = NumFuncs;

1762}

ArrayRef.h

static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")

DenseMap.h

This file defines the DenseMap class.

Name

std::string Name

Definition:ELFObjHandler.cpp:77

End

bool End

Definition:ELF_riscv.cpp:480

Endian.h

static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")

ErrorOr.h

Provides ErrorOr<T> smart pointer.

FormatVariadic.h

setupMemoryBuffer

static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)

Definition:InstrProfReader.cpp:74

initializeReader

static Error initializeReader(InstrProfReader &Reader)

Definition:InstrProfReader.cpp:82

READ_NUM

#define READ_NUM(Str, Dst)

CHECK_LINE_END

#define CHECK_LINE_END(Line)

readBinaryIdsInternal

static Error readBinaryIdsInternal(const MemoryBuffer &DataBuffer, ArrayRef< uint8_t > BinaryIdsBuffer, std::vector< llvm::object::BuildID > &BinaryIds, const llvm::endianness Endian)

Read a list of binary ids from a profile that consist of a.

Definition:InstrProfReader.cpp:95

VP_READ_ADVANCE

#define VP_READ_ADVANCE(Val)

getProfileKindFromVersion

static InstrProfKind getProfileKindFromVersion(uint64_t Version)

Definition:InstrProfReader.cpp:44

getMemProfRecordV2

static Expected< memprof::MemProfRecord > getMemProfRecordV2(const memprof::IndexedMemProfRecord &IndexedRecord, MemProfFrameHashTable &MemProfFrameTable, MemProfCallStackHashTable &MemProfCallStackTable)

Definition:InstrProfReader.cpp:1553

getMemProfRecordV3

static Expected< memprof::MemProfRecord > getMemProfRecordV3(const memprof::IndexedMemProfRecord &IndexedRecord, const unsigned char *FrameBase, const unsigned char *CallStackBase)

Definition:InstrProfReader.cpp:1583

printBinaryIdsInternal

static void printBinaryIdsInternal(raw_ostream &OS, ArrayRef< llvm::object::BuildID > BinaryIds)

Definition:InstrProfReader.cpp:145

InstrProfReader.h

InstrProf.h

#define I(x, y, z)

Definition:MD5.cpp:58

MemProf.h

MemoryBuffer.h

if(PassOpts->AAPipeline)

Definition:PassBuilderBindings.cpp:64

ProfileCommon.h

ProfileSummary.h

getName

static StringRef getName(Value *V)

Definition:ProvenanceAnalysisEvaluator.cpp:20

assert

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

Endian

endianness Endian

Definition:SampleProfWriter.cpp:52

raw_pwrite_stream & OS

Definition:SampleProfWriter.cpp:51

StringExtras.h

This file contains some functions that are useful when dealing with strings.

StringRef.h

error

#define error(X)

Definition:SymbolRecordMapping.cpp:14

SymbolRemappingReader.h

Ptr

@ Ptr

Definition:TargetLibraryInfo.cpp:77

VirtualFileSystem.h

Defines the virtual file system interface vfs::FileSystem.

IntPtrT

llvm::ArrayRef

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Definition:ArrayRef.h:41

llvm::ArrayRef::size

size_t size() const

size - Get the array size.

Definition:ArrayRef.h:168

llvm::ArrayRef::data

const T * data() const

Definition:ArrayRef.h:165

llvm::BitVector

Definition:BitVector.h:82

llvm::BitVector::resize

void resize(unsigned N, bool t=false)

resize - Grow or shrink the bitvector.

Definition:BitVector.h:341

llvm::BitVector::set

BitVector & set()

Definition:BitVector.h:351

llvm::BitVector::set_bits

iterator_range< const_set_bits_iterator > set_bits() const

Definition:BitVector.h:140

llvm::BitVector::apply

static BitVector & apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args)

Definition:BitVector.h:552

llvm::DenseMap

Definition:DenseMap.h:727

llvm::Error

Lightweight error class with error context and mandatory checking.

Definition:Error.h:160

llvm::Error::success

static ErrorSuccess success()

Create a success value.

Definition:Error.h:337

llvm::Expected

Tagged union holding either a T or a Error.

Definition:Error.h:481

llvm::Expected::takeError

Error takeError()

Take ownership of the stored error.

Definition:Error.h:608

llvm::Expected::get

reference get()

Returns a reference to the stored T value.

Definition:Error.h:578

llvm::IndexedInstrProfReader

Reader for the indexed binary instrprof format.

Definition:InstrProfReader.h:726

llvm::IndexedInstrProfReader::readNextRecord

Error readNextRecord(NamedInstrProfRecord &Record) override

Read a single record.

Definition:InstrProfReader.cpp:1721

llvm::IndexedInstrProfReader::create

static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")

Factory method to create an indexed reader.

Definition:InstrProfReader.cpp:202

llvm::IndexedInstrProfReader::readHeader

Error readHeader() override

Read the file header.

Definition:InstrProfReader.cpp:1356

llvm::IndexedInstrProfReader::printBinaryIds

Error printBinaryIds(raw_ostream &OS) override

Print binary ids.

Definition:InstrProfReader.cpp:1742

llvm::IndexedInstrProfReader::getFunctionBitmap

Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, BitVector &Bitmap)

Fill Bitmap with the profile data for the given function name.

Definition:InstrProfReader.cpp:1694

llvm::IndexedInstrProfReader::getSymtab

InstrProfSymtab & getSymtab() override

Return the PGO symtab.

Definition:InstrProfReader.cpp:1473

llvm::IndexedInstrProfReader::hasFormat

static bool hasFormat(const MemoryBuffer &DataBuffer)

Return true if the given buffer is in an indexed instrprof format.

Definition:InstrProfReader.cpp:1168

llvm::IndexedInstrProfReader::getInstrProfRecord

Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)

Return the NamedInstrProfRecord associated with FuncName and FuncHash.

Definition:InstrProfReader.cpp:1494

llvm::IndexedInstrProfReader::getFunctionCounts

Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)

Fill Counts with the profile data for the given function name.

Definition:InstrProfReader.cpp:1683

llvm::IndexedInstrProfReader::readBinaryIds

Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override

Read a list of binary ids.

Definition:InstrProfReader.cpp:1736

llvm::IndexedMemProfReader::deserialize

Error deserialize(const unsigned char *Start, uint64_t MemProfOffset)

Definition:InstrProfReader.cpp:1322

llvm::IndexedMemProfReader::getAllMemProfData

memprof::AllMemProfData getAllMemProfData() const

Definition:InstrProfReader.cpp:1667

llvm::IndexedMemProfReader::getMemProfRecord

Expected< memprof::MemProfRecord > getMemProfRecord(const uint64_t FuncNameHash) const

Definition:InstrProfReader.cpp:1593

llvm::IndexedMemProfReader::getMemProfCallerCalleePairs

DenseMap< uint64_t, SmallVector< memprof::CallEdgeTy, 0 > > getMemProfCallerCalleePairs() const

Definition:InstrProfReader.cpp:1629

llvm::InstrProfCorrelatorImpl

InstrProfCorrelatorImpl - A child of InstrProfCorrelator with a template pointer type so that the Pro...

Definition:InstrProfCorrelator.h:130

llvm::InstrProfCorrelatorImpl::getDataPointer

const RawInstrProf::ProfileData< IntPtrT > * getDataPointer() const

Return a pointer to the underlying ProfileData vector that this class constructs.

Definition:InstrProfCorrelator.h:137

llvm::InstrProfCorrelatorImpl::getDataSize

size_t getDataSize() const

Return the number of ProfileData elements.

Definition:InstrProfCorrelator.h:142

llvm::InstrProfCorrelator

InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.

Definition:InstrProfCorrelator.h:34

llvm::InstrProfCorrelator::getNamesPointer

const char * getNamesPointer() const

Return a pointer to the names string that this class constructs.

Definition:InstrProfCorrelator.h:58

llvm::InstrProfCorrelator::ProfCorrelatorKind

ProfCorrelatorKind

Indicate if we should use the debug info or profile metadata sections to correlate.

Definition:InstrProfCorrelator.h:38

llvm::InstrProfCorrelator::getDataSize

std::optional< size_t > getDataSize() const

Return the number of ProfileData elements.

Definition:InstrProfCorrelator.cpp:175

llvm::InstrProfCorrelator::get

static llvm::Expected< std::unique_ptr< InstrProfCorrelator > > get(StringRef Filename, ProfCorrelatorKind FileKind, const object::BuildIDFetcher *BIDFetcher=nullptr, const ArrayRef< llvm::object::BuildID > BIs={})

Definition:InstrProfCorrelator.cpp:94

llvm::InstrProfCorrelator::getNamesSize

size_t getNamesSize() const

Return the number of bytes in the names string.

Definition:InstrProfCorrelator.h:61

llvm::InstrProfError

Definition:InstrProf.h:404

llvm::InstrProfError::take

static std::pair< instrprof_error, std::string > take(Error E)

Consume an Error and return the raw enum value contained within it, and the optional error message.

Definition:InstrProf.h:425

llvm::InstrProfLookupTrait::hash_value_type

uint64_t hash_value_type

Definition:InstrProfReader.h:529

llvm::InstrProfLookupTrait::offset_type

uint64_t offset_type

Definition:InstrProfReader.h:530

llvm::InstrProfLookupTrait::ReadData

data_type ReadData(StringRef K, const unsigned char *D, offset_type N)

Definition:InstrProfReader.cpp:936

llvm::InstrProfLookupTrait::readValueProfilingData

bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)

Definition:InstrProfReader.cpp:922

llvm::InstrProfLookupTrait::ComputeHash

hash_value_type ComputeHash(StringRef K)

Definition:InstrProfReader.cpp:915

llvm::InstrProfLookupTrait::data_type

ArrayRef< NamedInstrProfRecord > data_type

Definition:InstrProfReader.h:525

llvm::InstrProfReaderIndex

Definition:InstrProfReader.h:603

llvm::InstrProfReaderIndex::getProfileKind

InstrProfKind getProfileKind() const override

Definition:InstrProfReader.cpp:1045

llvm::InstrProfReaderIndex::getRecords

Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override

Definition:InstrProfReader.cpp:1018

llvm::InstrProfReaderIndex::InstrProfReaderIndex

InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)

Definition:InstrProfReader.cpp:1033

llvm::InstrProfReaderItaniumRemapper

A remapper that applies remappings based on a symbol remapping file.

Definition:InstrProfReader.cpp:1068

llvm::InstrProfReaderItaniumRemapper::extractName

static StringRef extractName(StringRef Name)

Extract the original function name from a PGO function name.

Definition:InstrProfReader.cpp:1077

llvm::InstrProfReaderItaniumRemapper::InstrProfReaderItaniumRemapper

InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)

Definition:InstrProfReader.cpp:1070

llvm::InstrProfReaderItaniumRemapper::populateRemappings

Error populateRemappings() override

Definition:InstrProfReader.cpp:1103

llvm::InstrProfReaderItaniumRemapper::reconstituteName

static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)

Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...

Definition:InstrProfReader.cpp:1094

llvm::InstrProfReaderItaniumRemapper::getRecords

Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override

Definition:InstrProfReader.cpp:1118

llvm::InstrProfReaderRemapper

Name matcher supporting fuzzy matching of symbol names to names in profiles.

Definition:InstrProfReader.h:679

llvm::InstrProfReader

Base class and interface for reading profiling data of any known instrprof format.

Definition:InstrProfReader.h:92

llvm::InstrProfReader::Symtab

std::unique_ptr< InstrProfSymtab > Symtab

Definition:InstrProfReader.h:162

llvm::InstrProfReader::success

Error success()

Clear the current error and return a successful one.

Definition:InstrProfReader.h:186

llvm::InstrProfReader::TemporalProfTraces

SmallVector< TemporalProfTraceTy > TemporalProfTraces

A list of temporal profile traces.

Definition:InstrProfReader.h:164

llvm::InstrProfReader::TemporalProfTraceStreamSize

uint64_t TemporalProfTraceStreamSize

The total number of temporal profile traces seen.

Definition:InstrProfReader.h:166

llvm::InstrProfReader::isIRLevelProfile

virtual bool isIRLevelProfile() const =0

llvm::InstrProfReader::readHeader

virtual Error readHeader()=0

Read the header. Required before reading first record.

llvm::InstrProfReader::accumulateCounts

void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)

Compute the sum of counts and return in Sum.

Definition:InstrProfReader.cpp:1750

llvm::InstrProfReader::create

static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, const object::BuildIDFetcher *BIDFetcher=nullptr, const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind=InstrProfCorrelator::ProfCorrelatorKind::NONE, std::function< void(Error)> Warn=nullptr)

Factory method to create an appropriately typed reader for the given instrprof file.

Definition:InstrProfReader.cpp:155

llvm::InstrProfSummaryBuilder

Definition:ProfileCommon.h:79

llvm::InstrProfSymtab

A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...

Definition:InstrProf.h:460

llvm::InstrProfSymtab::isExternalSymbol

static bool isExternalSymbol(const StringRef &Symbol)

True if Symbol is the value used to represent external symbols.

Definition:InstrProf.h:651

llvm::InstrProfSymtab::mapAddress

void mapAddress(uint64_t Addr, uint64_t MD5Val)

Map a function address to its name's MD5 hash.

Definition:InstrProf.h:619

llvm::InstrProfSymtab::create

Error create(object::SectionRef &Section)

Create InstrProfSymtab from an object file section which contains function PGO names.

Definition:CoverageMappingReader.cpp:509

llvm::InstrProfSymtab::mapVTableAddress

void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val)

Map the address range (i.e., [start_address, end_address)) of a variable to its names' MD5 hash.

Definition:InstrProf.h:626

llvm::MemoryBuffer

This interface provides simple read-only access to a block of memory, and provides simple methods for...

Definition:MemoryBuffer.h:51

llvm::MemoryBuffer::getBufferSize

size_t getBufferSize() const

Definition:MemoryBuffer.h:68

llvm::MemoryBuffer::getBufferEnd

const char * getBufferEnd() const

Definition:MemoryBuffer.h:67

llvm::MemoryBuffer::getSTDIN

static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()

Read all of stdin into a file buffer, and return it.

Definition:MemoryBuffer.cpp:566

llvm::MemoryBuffer::getBufferStart

const char * getBufferStart() const

Definition:MemoryBuffer.h:66

llvm::OnDiskIterableChainedHashTable

Provides lookup and iteration over an on disk hash table.

Definition:OnDiskHashTable.h:434

llvm::OnDiskIterableChainedHashTable::Create

static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())

Create the hash table.

Definition:OnDiskHashTable.h:606

llvm::ProfileSummaryBuilder::DefaultCutoffs

static const ArrayRef< uint32_t > DefaultCutoffs

A vector of useful cutoff values for detailed summary.

Definition:ProfileCommon.h:70

llvm::ProfileSummary::PSK_CSInstr

@ PSK_CSInstr

Definition:ProfileSummary.h:47

llvm::ProfileSummary::PSK_Instr

@ PSK_Instr

Definition:ProfileSummary.h:47

llvm::RawInstrProfReader

Reader for the raw instrprof binary format from runtime.

Definition:InstrProfReader.h:325

llvm::RawInstrProfReader::readHeader

Error readHeader() override

Read the header. Required before reading first record.

Definition:InstrProfReader.cpp:511

llvm::RawInstrProfReader::readNextRecord

Error readNextRecord(NamedInstrProfRecord &Record) override

Read a single record.

Definition:InstrProfReader.cpp:859

llvm::RawInstrProfReader::printBinaryIds

Error printBinaryIds(raw_ostream &OS) override

Print binary ids.

Definition:InstrProfReader.cpp:901

llvm::RawInstrProfReader::hasFormat

static bool hasFormat(const MemoryBuffer &DataBuffer)

Definition:InstrProfReader.cpp:501

llvm::RawInstrProfReader::getProfileKind

InstrProfKind getProfileKind() const override

Returns a BitsetEnum describing the attributes of the raw instr profile.

Definition:InstrProfReader.cpp:477

llvm::RawInstrProfReader::readBinaryIds

Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override

Read a list of binary ids.

Definition:InstrProfReader.cpp:893

llvm::RawInstrProfReader::getTemporalProfTraces

SmallVector< TemporalProfTraceTy > & getTemporalProfTraces(std::optional< uint64_t > Weight={}) override

Definition:InstrProfReader.cpp:483

llvm::Record

Definition:Record.h:1596

llvm::SmallString

SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...

Definition:SmallString.h:26

llvm::SmallVectorImpl

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

Definition:SmallVector.h:573

llvm::SmallVectorImpl::reserve

void reserve(size_type N)

Definition:SmallVector.h:663

llvm::SmallVectorImpl::insert

iterator insert(iterator I, T &&Elt)

Definition:SmallVector.h:805

llvm::SmallVectorTemplateCommon::end

iterator end()

Definition:SmallVector.h:269

llvm::SmallVector

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

Definition:SmallVector.h:1196

llvm::StringRef

StringRef - Represent a constant reference to a string, i.e.

Definition:StringRef.h:51

llvm::StringRef::split

std::pair< StringRef, StringRef > split(char Separator) const

Split into two substrings around the first occurrence of a separator character.

Definition:StringRef.h:700

llvm::StringRef::getAsInteger

bool getAsInteger(unsigned Radix, T &Result) const

Parse the current string as an integer of the specified radix.

Definition:StringRef.h:470

llvm::StringRef::substr

constexpr StringRef substr(size_t Start, size_t N=npos) const

Return a reference to the substring from [Start, Start + N).

Definition:StringRef.h:571

llvm::StringRef::starts_with

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

Definition:StringRef.h:265

llvm::StringRef::empty

constexpr bool empty() const

empty - Check if the string is empty.

Definition:StringRef.h:147

llvm::StringRef::begin

iterator begin() const

Definition:StringRef.h:116

llvm::StringRef::size

constexpr size_t size() const

size - Get the string size.

Definition:StringRef.h:150

llvm::StringRef::end

iterator end() const

Definition:StringRef.h:118

llvm::StringRef::rsplit

std::pair< StringRef, StringRef > rsplit(StringRef Separator) const

Split into two substrings around the last occurrence of a separator string.

Definition:StringRef.h:733

llvm::SymbolRemappingReader

Reader for symbol remapping files.

Definition:SymbolRemappingReader.h:98

llvm::SymbolRemappingReader::insert

Key insert(StringRef FunctionName)

Construct a key for the given symbol, or return an existing one if an equivalent name has already bee...

Definition:SymbolRemappingReader.h:113

llvm::SymbolRemappingReader::lookup

Key lookup(StringRef FunctionName)

Map the given symbol name into the key for the corresponding equivalence class.

Definition:SymbolRemappingReader.h:123

llvm::SymbolRemappingReader::read

Error read(MemoryBuffer &B)

Read remappings from the given buffer, which must live as long as the remapper.

Definition:SymbolRemappingReader.cpp:28

llvm::TextInstrProfReader

Reader for the simple text based instrprof format.

Definition:InstrProfReader.h:243

llvm::TextInstrProfReader::hasFormat

static bool hasFormat(const MemoryBuffer &Buffer)

Return true if the given buffer is in text instrprof format.

Definition:InstrProfReader.cpp:239

llvm::TextInstrProfReader::readNextRecord

Error readNextRecord(NamedInstrProfRecord &Record) override

Read a single record.

Definition:InstrProfReader.cpp:403

llvm::TextInstrProfReader::readHeader

Error readHeader() override

Read the header.

Definition:InstrProfReader.cpp:252

llvm::Trace

Definition:Trace.h:30

llvm::Twine

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

Definition:Twine.h:81

llvm::Twine::str

std::string str() const

Return the twine contents as a std::string.

Definition:Twine.cpp:17

llvm::Value

LLVM Value Representation.

Definition:Value.h:74

llvm::line_iterator::is_at_end

bool is_at_end() const

Return true if we're an "end" iterator or have reached EOF.

Definition:LineIterator.h:63

llvm::memprof::RecordLookupTrait

Definition:MemProf.h:486

llvm::object::BuildIDFetcher

BuildIDFetcher searches local cache directories for debug info.

Definition:BuildID.h:39

llvm::raw_ostream

This class implements an extremely fast bulk output stream that can only output to a stream.

Definition:raw_ostream.h:52

llvm::vfs::FileSystem

The virtual file system interface.

Definition:VirtualFileSystem.h:266

llvm::IndexedInstrProf::allocSummary

std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)

Definition:InstrProf.h:1243

llvm::IndexedInstrProf::ComputeHash

uint64_t ComputeHash(StringRef K)

Definition:InstrProf.h:1124

llvm::IndexedInstrProf::HashT

HashT

Definition:InstrProf.h:1068

llvm::IndexedInstrProf::HashT::Last

@ Last

llvm::IndexedInstrProf::Magic

const uint64_t Magic

Definition:InstrProf.h:1081

llvm::IndexedInstrProf::ProfVersion

ProfVersion

Definition:InstrProf.h:1083

llvm::IndexedInstrProf::Version10

@ Version10

Definition:InstrProf.h:1112

llvm::IndexedInstrProf::Version4

@ Version4

Definition:InstrProf.h:1099

llvm::IndexedInstrProf::Version2

@ Version2

Definition:InstrProf.h:1093

llvm::IndexedInstrProf::Version1

@ Version1

Definition:InstrProf.h:1088

llvm::M68k::MemAddrModeKind::V

@ V

llvm::RawInstrProf::Version

const uint64_t Version

Definition:InstrProf.h:1266

llvm::memprof::IndexedVersion

IndexedVersion

Definition:MemProf.h:31

llvm::memprof::Version2

@ Version2

Definition:MemProf.h:33

llvm::memprof::Version3

@ Version3

Definition:MemProf.h:36

llvm::memprof::MaximumSupportedVersion

constexpr uint64_t MaximumSupportedVersion

Definition:MemProf.h:40

llvm::memprof::MinimumSupportedVersion

constexpr uint64_t MinimumSupportedVersion

Definition:MemProf.h:39

llvm::memprof::readMemProfSchema

Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)

Definition:MemProf.cpp:267

llvm::sampleprof::Base

@ Base

Definition:Discriminator.h:58

llvm::support::endian::read

value_type read(const void *memory, endianness endian)

Read a value of a particular endianness from memory.

Definition:Endian.h:58

llvm::support::aligned

@ aligned

Definition:Endian.h:29

llvm

This is an optimization pass for GlobalISel generic memory operations.

Definition:AddressRanges.h:18

llvm::RawInstrProfReader32

RawInstrProfReader< uint32_t > RawInstrProfReader32

Definition:InstrProfReader.h:501

llvm::setupMemoryBuffer

static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)

Definition:CodeGenDataReader.cpp:25

llvm::byteswap

constexpr T byteswap(T V) noexcept

Reverses the bytes in the given integer value V.

Definition:bit.h:101

llvm::handleErrors

Error handleErrors(Error E, HandlerTs &&... Hs)

Pass the ErrorInfo(s) contained in E to their respective handlers.

Definition:Error.h:954

llvm::unique

auto unique(Range &&R, Predicate P)

Definition:STLExtras.h:2055

llvm::formatv

auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)

Definition:FormatVariadic.h:252

llvm::cgdata_error::success

@ success

llvm::sort

void sort(IteratorTy Start, IteratorTy End)

Definition:STLExtras.h:1664

llvm::alignToPowerOf2

constexpr T alignToPowerOf2(U Value, V Align)

Will overflow only if result is not representable in T.

Definition:MathExtras.h:503

llvm::format

format_object< Ts... > format(const char *Fmt, const Ts &... Vals)

These are helper functions used to produce formatted output.

Definition:Format.h:125

llvm::SummaryEntryVector

std::vector< ProfileSummaryEntry > SummaryEntryVector

Definition:ProfileSummary.h:43

llvm::instrprof_error::unsupported_version

@ unsupported_version

llvm::instrprof_error::eof

@ eof

llvm::instrprof_error::raw_profile_version_mismatch

@ raw_profile_version_mismatch

llvm::instrprof_error::bad_magic

@ bad_magic

llvm::instrprof_error::counter_value_too_large

@ counter_value_too_large

llvm::instrprof_error::malformed

@ malformed

llvm::instrprof_error::truncated

@ truncated

llvm::instrprof_error::unknown_function

@ unknown_function

llvm::instrprof_error::invalid_prof

@ invalid_prof

llvm::instrprof_error::hash_mismatch

@ hash_mismatch

llvm::instrprof_error::unsupported_hash_type

@ unsupported_hash_type

llvm::instrprof_error::unrecognized_format

@ unrecognized_format

llvm::instrprof_error::bad_header

@ bad_header

llvm::instrprof_error::empty_raw_profile

@ empty_raw_profile

llvm::instrprof_error::unexpected_correlation_info

@ unexpected_correlation_info

llvm::count

auto count(R &&Range, const E &Element)

Wrapper function around std::count to count the number of times an element Element occurs in the give...

Definition:STLExtras.h:1938

llvm::GlobalIdentifierDelimiter

constexpr char GlobalIdentifierDelimiter

Definition:GlobalValue.h:46

llvm::move

OutputIt move(R &&Range, OutputIt Out)

Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.

Definition:STLExtras.h:1873

llvm::RawInstrProfReader64

RawInstrProfReader< uint64_t > RawInstrProfReader64

Definition:InstrProfReader.h:502

llvm::errorCodeToError

Error errorCodeToError(std::error_code EC)

Helper for converting an std::error_code to a Error.

Definition:Error.cpp:111

llvm::endianness

endianness

Definition:bit.h:70

llvm::endianness::little

@ little

llvm::Data

@ Data

Definition:SIMachineScheduler.h:55

llvm::consumeError

void consumeError(Error Err)

Consume a Error without doing anything.

Definition:Error.h:1069

llvm::Version

@ Version

Definition:PGOCtxProfWriter.h:22

llvm::InstrProfKind

InstrProfKind

An enum describing the attributes of an instrumented profile.

Definition:InstrProf.h:329

llvm::InstrProfKind::ContextSensitive

@ ContextSensitive

llvm::InstrProfKind::LoopEntriesInstrumentation

@ LoopEntriesInstrumentation

llvm::InstrProfKind::FunctionEntryInstrumentation

@ FunctionEntryInstrumentation

llvm::InstrProfKind::TemporalProfile

@ TemporalProfile

llvm::InstrProfKind::IRInstrumentation

@ IRInstrumentation

llvm::InstrProfKind::SingleByteCoverage

@ SingleByteCoverage

llvm::InstrProfKind::FrontendInstrumentation

@ FrontendInstrumentation

std

Implement std::hash so that hash_code can be used in STL containers.

Definition:BitVector.h:858

std::swap

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

Definition:BitVector.h:860

#define N

llvm::CountSumOrPercent

Definition:InstrProf.h:746

llvm::CountSumOrPercent::NumEntries

uint64_t NumEntries

Definition:InstrProf.h:747

llvm::IndexedInstrProf::Header

Definition:InstrProf.h:1129

llvm::IndexedInstrProf::Header::readFromBuffer

static Expected< Header > readFromBuffer(const unsigned char *Buffer)

Definition:InstrProf.cpp:1643

llvm::IndexedInstrProf::Summary::Entry

Definition:InstrProf.h:1166

llvm::IndexedInstrProf::Summary::Entry::Cutoff

uint64_t Cutoff

The required percentile of total execution count.

Definition:InstrProf.h:1167

llvm::IndexedInstrProf::Summary::Entry::NumBlocks

uint64_t NumBlocks

Number of blocks >= the minumum execution count.

Definition:InstrProf.h:1170

llvm::IndexedInstrProf::Summary::Entry::MinBlockCount

uint64_t MinBlockCount

The minimum execution count for this percentile.

Definition:InstrProf.h:1169

llvm::IndexedInstrProf::Summary

Definition:InstrProf.h:1165

llvm::IndexedInstrProf::Summary::getSize

static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)

Definition:InstrProf.h:1203

llvm::IndexedInstrProf::Summary::NumSummaryFields

uint64_t NumSummaryFields

Definition:InstrProf.h:1194

llvm::IndexedInstrProf::Summary::NumCutoffEntries

uint64_t NumCutoffEntries

Definition:InstrProf.h:1196

llvm::InstrProfReaderIndexBase

Definition:InstrProfReader.h:563

llvm::InstrProfRecord

Profiling information for a single function.

Definition:InstrProf.h:836

llvm::NamedInstrProfRecord

Definition:InstrProf.h:997

llvm::NamedInstrProfRecord::hasCSFlagInHash

static bool hasCSFlagInHash(uint64_t FuncHash)

Definition:InstrProf.h:1014

llvm::RawInstrProf::Header

Definition:InstrProf.h:1296

llvm::RawInstrProf::ProfileData

Definition:InstrProf.h:1282

llvm::RawInstrProf::VTableProfileData

Definition:InstrProf.h:1287

llvm::TemporalProfTraceTy

An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.

Definition:InstrProf.h:385

llvm::memprof::AllMemProfData

Definition:MemProfYAML.h:23

llvm::memprof::AllMemProfData::HeapProfileRecords

std::vector< GUIDMemProfRecordPair > HeapProfileRecords

Definition:MemProfYAML.h:24

llvm::memprof::CallStackIdConverter

Definition:MemProf.h:802

llvm::memprof::CallStackIdConverter::LastUnmappedId

std::optional< CallStackId > LastUnmappedId

Definition:MemProf.h:803

llvm::memprof::CallerCalleePairExtractor

Definition:MemProf.h:921

llvm::memprof::CallerCalleePairExtractor::CallerCalleePairs

DenseMap< uint64_t, SmallVector< CallEdgeTy, 0 > > CallerCalleePairs

Definition:MemProf.h:927

llvm::memprof::FrameIdConverter

Definition:MemProf.h:778

llvm::memprof::FrameIdConverter::LastUnmappedId

std::optional< FrameId > LastUnmappedId

Definition:MemProf.h:779

llvm::memprof::GUIDMemProfRecordPair

Definition:MemProfYAML.h:17

llvm::memprof::GUIDMemProfRecordPair::Record

MemProfRecord Record

Definition:MemProfYAML.h:19

llvm::memprof::GUIDMemProfRecordPair::GUID

GUIDHex64 GUID

Definition:MemProfYAML.h:18

llvm::memprof::IndexedAllocationInfo

Definition:MemProf.h:336

llvm::memprof::IndexedAllocationInfo::CSId

CallStackId CSId

Definition:MemProf.h:339

llvm::memprof::IndexedMemProfRecord

Definition:MemProf.h:393

llvm::memprof::IndexedMemProfRecord::toMemProfRecord

MemProfRecord toMemProfRecord(llvm::function_ref< std::vector< Frame >(const CallStackId)> Callback) const

Definition:MemProf.cpp:232

llvm::memprof::LinearCallStackIdConverter

Definition:MemProf.h:851

llvm::memprof::LinearFrameIdConverter

Definition:MemProf.h:836

llvm::memprof::MemProfRecord

Definition:MemProf.h:450