1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7//===----------------------------------------------------------------------===// 9// This file implements the class that reads LLVM sample profiles. It 10// supports three file formats: text, binary and gcov. 12// The textual representation is useful for debugging and testing purposes. The 13// binary representation is more compact, resulting in smaller file sizes. 15// The gcov encoding is the one generated by GCC's AutoFDO profile creation 16// tool (https://github.com/google/autofdo) 18// All three encodings can be used interchangeably as an input sample profile. 20//===----------------------------------------------------------------------===// 45#include <system_error> 49using namespacesampleprof;
51#define DEBUG_TYPE "samplepgo-reader" 53// This internal option specifies if the profile uses FS discriminators. 54// It only applies to text, and binary format profiles. 55// For ext-binary format profiles, the flag is set in the summary. 58cl::desc(
"Profile uses flow sensitive discriminators"));
60/// Dump the function profile for \p FName. 62/// \param FContext Name + context of the function to print. 63/// \param OS Stream to emit the output to. 66OS <<
"Function: " << FS.getContext().toString() <<
": " << FS;
69/// Dump all the function profiles found on stream \p OS. 71 std::vector<NameFunctionSamples> V;
80for (
constauto &
I : BodySamples) {
90if (!CallTargets.empty()) {
92for (
constauto &J : CallTargets) {
105for (
constauto &
I : CallsiteSamples)
106for (
constauto &FS :
I.second) {
126if (!BodySamples.empty())
130if (!CallsiteSamples.empty())
132 [&] { DumpCallsiteSamples(CallsiteSamples); });
136/// Dump all the function profiles found on stream \p OS in the JSON format. 138 std::vector<NameFunctionSamples> V;
142for (
constauto &
F : V)
146// Emit a newline character at the end as json::OStream doesn't emit one. 150/// Parse \p Input as function head. 152/// Parse one line of \p Input, and update function name in \p FName, 153/// function's total sample count in \p NumSamples, function's entry 154/// count in \p NumHeadSamples. 156/// \returns true if parsing is successful. 161size_t n2 = Input.
rfind(
':');
162size_t n1 = Input.
rfind(
':', n2 - 1);
163 FName = Input.
substr(0, n1);
171/// Returns true if line offset \p L is legal (only has 16 bits). 174/// Parse \p Input that contains metadata. 175/// Possible metadata: 176/// - CFG Checksum information: 177/// !CFGChecksum: 12345 178/// - CFG Checksum information: 180/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash. 202/// Parse \p Input as line sample. 204/// \param Input input line. 205/// \param LineTy Type of this line. 206/// \param Depth the depth of the inline stack. 207/// \param NumSamples total samples of the line/inlined callsite. 208/// \param LineOffset line offset to the start of the function. 209/// \param Discriminator discriminator of the line. 210/// \param TargetCountMap map from indirect call target to count. 211/// \param FunctionHash the function's CFG hash, used by pseudo probe. 213/// returns true if parsing is successful. 225if (Input[
Depth] ==
'!') {
226 LineTy = LineType::Metadata;
227// This metadata is only for manual inspection only. We already created a 228// FunctionSamples and put it in the profile map, so there is no point 229// to skip profiles even they have no use for ThinLTO. 237size_t n1 = Input.
find(
':');
239size_t n2 = Loc.
find(
'.');
253 LineTy = LineType::BodyProfile;
254size_t n3 = Rest.
find(
' ');
262// Find call targets and their sample counts. 263// Note: In some cases, there are symbols in the profile which are not 264// mangled. To accommodate such cases, use colon + integer pairs as the 267// _M_construct<char *>:1000 string_view<std::allocator<char> >:437 268// ":1000" and ":437" are used as anchor points so the string above will 270// target: _M_construct<char *> 272// target: string_view<std::allocator<char> > 284// Get the segment after the current colon. 286// Get the target symbol before the current colon. 288// Check if the word after the current colon is an integer. 295// Try to find the next colon. 302// An anchor point is found. Save the {target, count} pair 304if (n4 == Rest.
size())
306// Change n3 to the next blank space after colon + integer pair. 310 LineTy = LineType::CallSiteProfile;
312 CalleeName = Rest.
substr(0, n3);
319/// Load samples from a text file. 321/// See the documentation at the top of the file for an explanation of 322/// the expected format. 324/// \returns true if the file was loaded successfully, false otherwise. 330uint32_t TopLevelProbeProfileCount = 0;
332// DepthMetadata tracks whether we have processed metadata for the current 333// top-level or nested function profile. 336 std::vector<SampleContext *> FlatSamples;
342if (pos == LineIt->
npos || (*LineIt)[pos] ==
'#')
344// Read the header of each function. 346// Note that for function identifiers we are actually expecting 347// mangled names, but we may not always get them. This happens when 348// the compiler decides not to emit the function (e.g., it was inlined 349// and removed). In this case, the binary will not have the linkage 350// name for the function, so the profiler will emit the function's 351// unmangled name, which may contain characters like ':' and '>' in its 352// name (member functions, templates, etc). 354// The only requirement we place on the identifier, then, is that it 355// should not begin with a number. 356if ((*LineIt)[0] !=
' ') {
359if (!
ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) {
361"Expected 'mangled_name:NUM:NUM', found " + *LineIt);
383 Discriminator, FName, TargetCountMap, FunctionHash,
386"Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
390if (LineTy != LineType::Metadata &&
Depth == DepthMetadata) {
391// Metadata must be put at the end of a function profile. 393"Found non-metadata after metadata: " + *LineIt);
397// Here we handle FS discriminators. 404case LineType::CallSiteProfile: {
413case LineType::BodyProfile: {
418for (
constauto &name_count : TargetCountMap) {
420 LineOffset, Discriminator,
429case LineType::Metadata: {
434 ++TopLevelProbeProfileCount;
439 DepthMetadata =
Depth;
442 FlatSamples.push_back(&FProfile.
getContext());
446"!Flat may only be used at top level function.",
DS_Warning));
454// Honor the option to skip flat functions. Since they are already added to 455// the profile map, remove them all here. 461"Cannot have both context-sensitive and regular profile");
463assert((TopLevelProbeProfileCount == 0 ||
464 TopLevelProbeProfileCount ==
Profiles.size()) &&
465"Cannot have both probe-based profiles and regular profiles");
480// Check that the first non-comment line is a valid function header. 483if ((*LineIt)[0] !=
' ') {
486 result =
ParseHead(*LineIt, FName, NumSamples, NumHeadSamples);
494unsigned NumBytesRead = 0;
497if (Val > std::numeric_limits<T>::max()) {
501 }
elseif (
Data + NumBytesRead >
End) {
508returnstatic_cast<T>(Val);
513if (
Data + Str.size() + 1 >
End) {
519Data += Str.size() + 1;
531using namespacesupport;
532T Val = endian::readNext<T, llvm::endianness::little>(
Data);
538autoIdx = readNumber<size_t>();
539if (std::error_code EC =
Idx.getError())
541if (*
Idx >= Table.size())
549if (std::error_code EC =
Idx.getError())
558auto ContextIdx = readNumber<size_t>();
559if (std::error_code EC = ContextIdx.getError())
564 *RetIdx = *ContextIdx;
574if (std::error_code EC = FContext.getError())
579if (std::error_code EC = FName.getError())
583// Since MD5SampleContextStart may point to the profile's file data, need to 584// make sure it is reading the same value on big endian CPU. 586// Lazy computing of hash value, write back to the table to cache it. Only 587// compute the context's hash value if it is being referenced for the first 594return std::make_pair(Context, Hash);
599auto NumSamples = readNumber<uint64_t>();
600if (std::error_code EC = NumSamples.getError())
604// Read the samples in the body. 605auto NumRecords = readNumber<uint32_t>();
606if (std::error_code EC = NumRecords.getError())
610auto LineOffset = readNumber<uint64_t>();
611if (std::error_code EC = LineOffset.getError())
615return std::error_code();
618auto Discriminator = readNumber<uint64_t>();
619if (std::error_code EC = Discriminator.getError())
622auto NumSamples = readNumber<uint64_t>();
623if (std::error_code EC = NumSamples.getError())
626auto NumCalls = readNumber<uint32_t>();
627if (std::error_code EC = NumCalls.getError())
630// Here we handle FS discriminators: 633for (
uint32_t J = 0; J < *NumCalls; ++J) {
635if (std::error_code EC = CalledFunction.getError())
638auto CalledFunctionSamples = readNumber<uint64_t>();
639if (std::error_code EC = CalledFunctionSamples.getError())
643 *CalledFunction, *CalledFunctionSamples);
646 FProfile.
addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples);
649// Read all the samples for inlined function calls. 650auto NumCallsites = readNumber<uint32_t>();
651if (std::error_code EC = NumCallsites.getError())
654for (
uint32_t J = 0; J < *NumCallsites; ++J) {
655auto LineOffset = readNumber<uint64_t>();
656if (std::error_code EC = LineOffset.getError())
659auto Discriminator = readNumber<uint64_t>();
660if (std::error_code EC = Discriminator.getError())
664if (std::error_code EC = FName.getError())
667// Here we handle FS discriminators: 684auto NumHeadSamples = readNumber<uint64_t>();
685if (std::error_code EC = NumHeadSamples.getError())
689if (std::error_code EC = FContextHash.getError())
692auto &[FContext, Hash] = *FContextHash;
693// Use the cached hash value for insertion instead of recalculating it. 699if (FContext.hasContext())
732Summary->setPartialProfile(
true);
744// UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire 745// profile uses MD5 for function name matching in IPO passes. 764// If module is absent, we are using LLVM tools, and need to read all 765// profiles, so skip reading the function offset table. 771"func offset table should always be sorted in CS profile");
799// If profile is CS, the function offset section is expected to consist of 800// sequences of contexts in pre-order layout 801// (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched 802// context in the module is found, the profiles of all its callees are 803// recursively loaded. A list is needed since the order of profiles matters. 807// If the profile is MD5, use the map container to lookup functions in 808// the module. A remapper has no use on MD5 names. 812// Profile is not MD5 and if a remapper is present, the remapped name of 813// every function needed to be matched against the module, so use the list 814// container since each entry is accessed. 818// Otherwise use the map container for faster lookup. 819// TODO: If the cardinality of the function offset section is much smaller 820// than the number of functions in the module, using the list container can 821// be always faster, but we need to figure out the constant factor to 822// determine the cutoff. 850// If there are more than one function offset section, the profile associated 851// with the previous section has to be done reading before next one is read. 855autoSize = readNumber<uint64_t>();
856if (std::error_code EC =
Size.getError())
860if (UseFuncOffsetList)
867if (std::error_code EC = FContextHash.getError())
870auto &[FContext, Hash] = *FContextHash;
871autoOffset = readNumber<uint64_t>();
872if (std::error_code EC =
Offset.getError())
875if (UseFuncOffsetList)
878// Because Porfiles replace existing value with new value if collision 879// happens, we also use the latest offset so that they are consistent. 904// For each function in current module, load all context profiles for 905// the function as well as their callee contexts which can help profile 906// guided importing for ThinLTO. This can be achieved by walking 907// through an ordered context container, where contexts are laid out 908// as if they were walked in preorder of a context trie. While 909// traversing the trie, a link to the highest common ancestor node is 910// kept so that all of its decendants will be loaded. 913constauto &FContext = NameOffset.first;
919// For function in the current module, keep its farthest ancestor 920// context. This can be used to load itself and its child and 925if (!CommonContext || !CommonContext->
isPrefixOf(FContext))
926 CommonContext = &FContext;
929if (CommonContext == &FContext ||
930 (CommonContext && CommonContext->
isPrefixOf(FContext))) {
931// Load profile for the current context which originated from 932// the common ancestor. 933constuint8_t *FuncProfileAddr = Start + NameOffset.second;
945constuint8_t *FuncProfileAddr = Start + iter->second;
954StringRef FuncNameStr = FuncName.stringRef();
957constuint8_t *FuncProfileAddr = Start + NameOffset.second;
968constuint8_t *FuncProfileAddr = Start + iter->second;
978// Collect functions used by current module if the Reader has been 980// collectFuncsFromModule uses FunctionSamples::getCanonicalFnName 981// which will query FunctionSamples::HasUniqSuffix, so it has to be 982// called after FunctionSamples::HasUniqSuffix is set, i.e. after 983// NameTable section is read. 986// When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all 988if (!LoadFuncsToBeUsed) {
995// Load function profiles on demand. 1001"Cannot have both context-sensitive and regular profile");
1003"Section flag should be consistent with actual profile");
1009ProfSymList = std::make_unique<ProfileSymbolList>();
1018std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
1022End = SecStart + SecSize;
1023auto DecompressSize = readNumber<uint64_t>();
1024if (std::error_code EC = DecompressSize.getError())
1026 DecompressBufSize = *DecompressSize;
1028auto CompressSize = readNumber<uint64_t>();
1029if (std::error_code EC = CompressSize.getError())
1036size_t UCSize = DecompressBufSize;
1050// Skip empty section. 1054// Skip sections without inlined functions when SkipFlatProf is true. 1058constuint8_t *SecStart = BufStart + Entry.Offset;
1061// If the section is compressed, decompress it into a buffer 1062// DecompressBuf before reading the actual data. The pointee of 1063// 'Data' will be changed to buffer hold by DecompressBuf 1064// temporarily when reading the actual data. 1069if (std::error_code EC = decompressSection(
1070 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1072 SecStart = DecompressBuf;
1073 SecSize = DecompressBufSize;
1078if (
Data != SecStart + SecSize)
1081// Change the pointee of 'Data' from DecompressBuf to original Buffer. 1083Data = BufStart + Entry.Offset;
1091std::error_code SampleProfileReaderRawBinary::verifySPMagic(
uint64_t Magic) {
1097std::error_code SampleProfileReaderExtBinary::verifySPMagic(
uint64_t Magic) {
1104autoSize = readNumber<size_t>();
1105if (std::error_code EC =
Size.getError())
1108// Normally if useMD5 is true, the name table should have MD5 values, not 1109// strings, however in the case that ExtBinary profile has multiple name 1110// tables mixing string and MD5, all of them have to be normalized to use MD5, 1111// because optimization passes can only handle either type. 1121// If we are using strings, delay MD5 computation since only a portion of 1122// names are used by top level functions. Use 0 to indicate MD5 value is 1123// to be calculated as no known string has a MD5 value of 0. 1126for (
size_tI = 0;
I < *
Size; ++
I) {
1128if (std::error_code EC =
Name.getError())
1145bool FixedLengthMD5) {
1146if (FixedLengthMD5) {
1148errs() <<
"If FixedLengthMD5 is true, UseMD5 has to be true";
1149autoSize = readNumber<size_t>();
1150if (std::error_code EC =
Size.getError())
1154"Fixed length MD5 name table does not contain specified number of " 1161for (
size_tI = 0;
I < *
Size; ++
I) {
1162using namespacesupport;
1163uint64_t FID = endian::read<uint64_t, endianness::little, unaligned>(
1174assert(!FixedLengthMD5 &&
"FixedLengthMD5 should be unreachable here");
1175autoSize = readNumber<size_t>();
1176if (std::error_code EC =
Size.getError())
1183for (
size_tI = 0;
I < *
Size; ++
I) {
1184auto FID = readNumber<uint64_t>();
1185if (std::error_code EC = FID.getError())
1199// Read in the CS name table section, which basically contains a list of context 1200// vectors. Each element of a context vector, aka a frame, refers to the 1201// underlying raw function names that are stored in the name table, as well as 1202// a callsite identifier that only makes sense for non-leaf frames. 1204autoSize = readNumber<size_t>();
1205if (std::error_code EC =
Size.getError())
1211// Delay MD5 computation of CS context until they are needed. Use 0 to 1212// indicate MD5 value is to be calculated as no known string has a MD5 1218for (
size_tI = 0;
I < *
Size; ++
I) {
1220auto ContextSize = readNumber<uint32_t>();
1221if (std::error_code EC = ContextSize.getError())
1223for (
uint32_t J = 0; J < *ContextSize; ++J) {
1225if (std::error_code EC = FName.getError())
1227auto LineOffset = readNumber<uint64_t>();
1228if (std::error_code EC = LineOffset.getError())
1232return std::error_code();
1234auto Discriminator = readNumber<uint64_t>();
1235if (std::error_code EC = Discriminator.getError())
1239 FName.get(),
LineLocation(LineOffset.get(), Discriminator.get()));
1251auto Checksum = readNumber<uint64_t>();
1252if (std::error_code EC = Checksum.getError())
1260if (std::error_code EC =
Attributes.getError())
1267// Read all the attributes for inlined function calls. 1268auto NumCallsites = readNumber<uint32_t>();
1269if (std::error_code EC = NumCallsites.getError())
1272for (
uint32_t J = 0; J < *NumCallsites; ++J) {
1273auto LineOffset = readNumber<uint64_t>();
1274if (std::error_code EC = LineOffset.getError())
1277auto Discriminator = readNumber<uint64_t>();
1278if (std::error_code EC = Discriminator.getError())
1282if (std::error_code EC = FContextHash.getError())
1285auto &[FContext, Hash] = *FContextHash;
1291 *Discriminator))[FContext.getFunction()]);
1293if (std::error_code EC =
1315Data = R->second.first;
1316End = R->second.second;
1328if (std::error_code EC = FContextHash.getError())
1330auto &[FContext, Hash] = *FContextHash;
1334 FProfile = &It->second;
1350autoType = readUnencodedNumber<uint64_t>();
1351if (std::error_code EC =
Type.getError())
1353 Entry.Type =
static_cast<SecType>(*Type);
1355auto Flags = readUnencodedNumber<uint64_t>();
1356if (std::error_code EC = Flags.getError())
1358 Entry.Flags = *Flags;
1360autoOffset = readUnencodedNumber<uint64_t>();
1361if (std::error_code EC =
Offset.getError())
1365autoSize = readUnencodedNumber<uint64_t>();
1366if (std::error_code EC =
Size.getError())
1370 Entry.LayoutIndex =
Idx;
1376auto EntryNum = readUnencodedNumber<uint64_t>();
1377if (std::error_code EC = EntryNum.getError())
1380for (
uint64_t i = 0; i < (*EntryNum); i++)
1405if (Entry.Type ==
Type)
1412// Sections in SecHdrTable is not necessarily in the same order as 1413// sections in the profile because section like FuncOffsetTable needs 1414// to be written after section LBRProfile but needs to be read before 1415// section LBRProfile, so we cannot simply use the last entry in 1416// SecHdrTable to calculate the file size. 1419 FileSize = std::max(Entry.Offset + Entry.Size, FileSize);
1427 Flags.append(
"{compressed,");
1432 Flags.append(
"flat,");
1434switch (Entry.Type) {
1437 Flags.append(
"fixlenmd5,");
1439 Flags.append(
"md5,");
1441 Flags.append(
"uniq,");
1445 Flags.append(
"partial,");
1447 Flags.append(
"context,");
1449 Flags.append(
"preInlined,");
1451 Flags.append(
"fs-discriminator,");
1455 Flags.append(
"ordered,");
1459 Flags.append(
"probe,");
1461 Flags.append(
"attr,");
1466char &last = Flags.back();
1477OS <<
getSecName(Entry.Type) <<
" - Offset: " << Entry.Offset
1481 TotalSecsSize += Entry.Size;
1485"Size of 'header + sections' doesn't match the total size of profile");
1487OS <<
"Header Size: " << HeaderSize <<
"\n";
1488OS <<
"Total Sections Size: " << TotalSecsSize <<
"\n";
1494// Read and check the magic identifier. 1495auto Magic = readNumber<uint64_t>();
1496if (std::error_code EC = Magic.getError())
1498elseif (std::error_code EC = verifySPMagic(*Magic))
1501// Read the version number. 1502autoVersion = readNumber<uint64_t>();
1503if (std::error_code EC =
Version.getError())
1526std::error_code SampleProfileReaderBinary::readSummaryEntry(
1527 std::vector<ProfileSummaryEntry> &Entries) {
1528auto Cutoff = readNumber<uint64_t>();
1529if (std::error_code EC = Cutoff.getError())
1532auto MinBlockCount = readNumber<uint64_t>();
1533if (std::error_code EC = MinBlockCount.getError())
1536auto NumBlocks = readNumber<uint64_t>();
1537if (std::error_code EC = NumBlocks.getError())
1540 Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
1545auto TotalCount = readNumber<uint64_t>();
1546if (std::error_code EC = TotalCount.getError())
1549auto MaxBlockCount = readNumber<uint64_t>();
1550if (std::error_code EC = MaxBlockCount.getError())
1553auto MaxFunctionCount = readNumber<uint64_t>();
1554if (std::error_code EC = MaxFunctionCount.getError())
1557auto NumBlocks = readNumber<uint64_t>();
1558if (std::error_code EC = NumBlocks.getError())
1561auto NumFunctions = readNumber<uint64_t>();
1562if (std::error_code EC = NumFunctions.getError())
1565auto NumSummaryEntries = readNumber<uint64_t>();
1566if (std::error_code EC = NumSummaryEntries.getError())
1569 std::vector<ProfileSummaryEntry> Entries;
1570for (
unsigned i = 0; i < *NumSummaryEntries; i++) {
1571 std::error_code EC = readSummaryEntry(Entries);
1575Summary = std::make_unique<ProfileSummary>(
1577 *MaxFunctionCount, *NumBlocks, *NumFunctions);
1607returnstatic_cast<T>(Val);
1608 }
elseif (
sizeof(
T) <=
sizeof(
uint64_t)) {
1611returnstatic_cast<T>(Val);
1627// Read the magic identifier. 1631// Read the version number. Note - the GCC reader does not validate this 1632// version, but the profile creator generates v704. 1640// Skip the empty integer. 1673Names.push_back(std::string(Str));
1699if (InlineStack.
size() == 0)
1718if (InlineStack.
size() == 0) {
1719// If this is a top function that we have already processed, do not 1720// update its profile again. This happens in the presence of 1721// function aliases. Since these aliases share the same function 1722// body, there will be identical replicated profiles for the 1723// original function. In this case, we simply not bother updating 1724// the profile of the original function. 1730// Otherwise, we are reading an inlined instance. The top of the 1731// inline stack contains the profile of the caller. Insert this 1732// callee in the caller's CallsiteMap. 1754// The line location is encoded in the offset as: 1755// high 16 bits: line offset to the start of the function. 1756// low 16 bits: discriminator. 1764// Walk up the inline stack, adding the samples on this line to 1765// the total sample count of the callers in the chain. 1766for (
auto *CallerProfile : NewStack)
1767 CallerProfile->addTotalSamples(Count);
1769// Update the body samples for the current profile. 1773// Process the list of functions called at an indirect call site. 1774// These are all the targets that a function pointer (or virtual 1775// function) resolved at runtime. 1776for (
uint32_t J = 0; J < NumTargets; J++) {
1800// Process all the inlined callers into the current function. These 1801// are all the callsites that were inlined into this function. 1803// The offset is encoded as: 1804// high 16 bits: line offset to the start of the function. 1805// low 16 bits: discriminator. 1819/// Read a GCC AutoFDO profile. 1821/// This format is generated by the Linux Perf conversion tool at 1822/// https://github.com/google/autofdo. 1825// Read the string table. 1829// Read the source profile. 1837StringRef Magic(
reinterpret_cast<constchar *
>(
Buffer.getBufferStart()));
1838return Magic ==
"adcg*704";
1842// If the reader uses MD5 to represent string, we can't remap it because 1843// we don't know what the original function names were. 1847"Profile data remapping cannot be applied to profile data " 1848"using MD5 names (original mangled names are not available).",
1853// CSSPGO-TODO: Remapper is not yet supported. 1854// We will need to remap the entire context string. 1855assert(Remappings &&
"should be initialized while creating remapper");
1858 Sample.second.findAllNames(NamesInSample);
1859for (
auto &
Name : NamesInSample) {
1861if (
auto Key = Remappings->insert(NameStr))
1862 NameMap.insert({Key, NameStr});
1866 RemappingApplied =
true;
1869std::optional<StringRef>
1871if (
auto Key = Remappings->lookup(Fname)) {
1879/// Prepare a memory buffer for the contents of \p Filename. 1881/// \returns an error code indicating the status of the buffer. 1885 : FS.getBufferForFile(Filename);
1886if (std::error_code EC = BufferOrErr.getError())
1888auto Buffer = std::move(BufferOrErr.get());
1890return std::move(Buffer);
1893/// Create a sample profile reader based on the format of the input file. 1895/// \param Filename The file to open. 1897/// \param C The LLVM context to use to emit diagnostics. 1899/// \param P The FSDiscriminatorPass. 1901/// \param RemapFilename The file used for profile remapping. 1903/// \returns an error code indicating the status of the created reader. 1909if (std::error_code EC = BufferOrError.getError())
1911returncreate(BufferOrError.get(),
C, FS,
P, RemapFilename);
1914/// Create a sample profile remapper from the given input, to remap the 1915/// function names in the given profile data. 1917/// \param Filename The file to open. 1919/// \param Reader The profile reader the remapper is going to be applied to. 1921/// \param C The LLVM context to use to emit diagnostics. 1923/// \returns an error code indicating the status of the created reader. 1930if (std::error_code EC = BufferOrError.getError())
1932returncreate(BufferOrError.get(), Reader,
C);
1935/// Create a sample profile remapper from the given input, to remap the 1936/// function names in the given profile data. 1938/// \param B The memory buffer to create the reader from (assumes ownership). 1940/// \param C The LLVM context to use to emit diagnostics. 1942/// \param Reader The profile reader the remapper is going to be applied to. 1944/// \returns an error code indicating the status of the created reader. 1949auto Remappings = std::make_unique<SymbolRemappingReader>();
1950if (
Error E = Remappings->read(*
B)) {
1960return std::make_unique<SampleProfileReaderItaniumRemapper>(
1961 std::move(
B), std::move(Remappings), Reader);
1964/// Create a sample profile reader based on the format of the input data. 1966/// \param B The memory buffer to create the reader from (assumes ownership). 1968/// \param C The LLVM context to use to emit diagnostics. 1970/// \param P The FSDiscriminatorPass. 1972/// \param RemapFilename The file used for profile remapping. 1974/// \returns an error code indicating the status of the created reader. 1979 std::unique_ptr<SampleProfileReader> Reader;
1991if (!RemapFilename.
empty()) {
1993 RemapFilename, FS, *Reader,
C);
1994if (std::error_code EC = ReaderOrErr.getError()) {
1995 std::string Msg =
"Could not create remapper: " + EC.message();
1999 Reader->Remapper = std::move(ReaderOrErr.get());
2002if (std::error_code EC = Reader->readHeader()) {
2006 Reader->setDiscriminatorMaskedBitFrom(
P);
2008return std::move(Reader);
2011// For text and GCC file formats, we compute the summary after reading the 2012// profile. Binary format has the profile summary in its header. static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
Provides ErrorOr<T> smart pointer.
Module.h This file contains the declarations for the Module class.
This file supports working with JSON data.
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static bool ParseHead(const StringRef &Input, StringRef &FName, uint64_t &NumSamples, uint64_t &NumHeadSamples)
Parse Input as function head.
static void dumpFunctionProfileJson(const FunctionSamples &S, json::OStream &JOS, bool TopLevel=false)
static bool isOffsetLegal(unsigned L)
Returns true if line offset L is legal (only has 16 bits).
static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, DenseMap< StringRef, uint64_t > &TargetCountMap, uint64_t &FunctionHash, uint32_t &Attributes, bool &IsFlat)
Parse Input as line sample.
static cl::opt< bool > ProfileIsFSDisciminator("profile-isfs", cl::Hidden, cl::init(false), cl::desc("Profile uses flow sensitive discriminators"))
static std::string getSecFlagsStr(const SecHdrTableEntry &Entry)
static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash, uint32_t &Attributes)
Parse Input that contains metadata.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
Implements a dense probed hash-table based set.
Diagnostic information for the sample profiler.
Represents either an error or a value T.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
bool readInt(uint32_t &Val)
bool readInt64(uint64_t &Val)
bool readGCOVVersion(GCOV::GCOVVersion &version)
readGCOVVersion - Read GCOV version.
bool readString(StringRef &str)
bool readGCDAFormat()
readGCDAFormat - Check GCDA signature is valid at the beginning of buffer.
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
Root of the metadata hierarchy.
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
std::unique_ptr< ProfileSummary > computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr size_t size() const
size - Get the string size.
size_t find_last_of(char C, size_t From=npos) const
Find the last character in the string that is C, or npos if not found.
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
static constexpr size_t npos
size_t find_first_not_of(char C, size_t From=0) const
Find the first character in the string that is not C or npos if not found.
int64_t getLineNum() const
StringRef getMessage() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
json::OStream allows writing well-formed JSON without materializing all structures as json::Value ahe...
void object(Block Contents)
Emit an object whose elements are emitted in the provided Block.
void attribute(llvm::StringRef Key, const Value &Contents)
Emit an attribute whose value is self-contained (number, vector<int> etc).
void attributeArray(llvm::StringRef Key, Block Contents)
Emit an attribute whose value is an array with elements from the Block.
A forward iterator which reads text lines from a buffer.
int64_t line_number() const
Return the current line number. May return any number at EOF.
bool is_at_eof() const
Return true if we've reached EOF or are an "end" iterator.
This class implements an extremely fast bulk output stream that can only output to a stream.
This class represents a function that is read from a sample profile.
StringRef stringRef() const
Convert to StringRef.
uint64_t getHashCode() const
Get hash code of this object.
std::string str() const
Convert to a string, usually for output purpose.
Representation of the samples collected for a function.
static bool ProfileIsPreInlined
sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight=1)
uint64_t getHeadSamples() const
For top-level functions, return the total number of branch samples that have the function as the bran...
void setFunction(FunctionId NewFunctionID)
Set the name of the function.
FunctionId getFunction() const
Return the function name.
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight=1)
sampleprof_error addCalledTargetSamples(uint32_t LineOffset, uint32_t Discriminator, FunctionId Func, uint64_t Num, uint64_t Weight=1)
FunctionSamplesMap & functionSamplesAt(const LineLocation &Loc)
Return the function samples at the given callsite location.
static bool ProfileIsProbeBased
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator, uint64_t Num, uint64_t Weight=1)
void setFunctionHash(uint64_t Hash)
static bool ProfileIsFS
If this profile uses flow sensitive discriminators.
SampleContext & getContext() const
static bool HasUniqSuffix
Whether the profile contains any ".__uniq." suffix in a name.
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
const CallsiteSampleMap & getCallsiteSamples() const
Return all the callsite samples collected in the body of the function.
void setContext(const SampleContext &FContext)
const BodySampleMap & getBodySamples() const
Return all the samples collected in the body of the function.
std::pair< iterator, bool > try_emplace(const key_type &Hash, const original_key_type &Key, Ts &&...Args)
void setAllAttributes(uint32_t A)
uint64_t getHashCode() const
FunctionId getFunction() const
bool isPrefixOf(const SampleContext &That) const
This class provides operator overloads to the map container using MD5 as the key type,...
iterator find(const SampleContext &Ctx)
mapped_type & create(const SampleContext &Ctx)
size_t erase(const SampleContext &Ctx)
std::error_code readProfile(FunctionSamples &FProfile)
Read the contents of the given profile instance.
std::error_code readNameTable()
Read the whole name table.
const uint8_t * Data
Points to the current location in the buffer.
ErrorOr< StringRef > readString()
Read a string from the profile.
std::vector< FunctionId > NameTable
Function name table.
ErrorOr< T > readNumber()
Read a numeric value of type T from the profile.
ErrorOr< SampleContextFrames > readContextFromTable(size_t *RetIdx=nullptr)
Read a context indirectly via the CSNameTable.
ErrorOr< std::pair< SampleContext, uint64_t > > readSampleContextFromTable()
Read a context indirectly via the CSNameTable if the profile has context, otherwise same as readStrin...
std::error_code readHeader() override
Read and validate the file header.
const uint64_t * MD5SampleContextStart
The starting address of the table of MD5 values of sample contexts.
std::vector< SampleContextFrameVector > CSNameTable
CSNameTable is used to save full context vectors.
std::error_code readImpl() override
Read sample profiles from the associated file.
ErrorOr< FunctionId > readStringFromTable(size_t *RetIdx=nullptr)
Read a string indirectly via the name table. Optionally return the index.
std::vector< uint64_t > MD5SampleContextTable
Table to cache MD5 values of sample contexts corresponding to readSampleContextFromTable(),...
ErrorOr< size_t > readStringIndex(T &Table)
Read the string index and check whether it overflows the table.
const uint8_t * End
Points to the end of the buffer.
ErrorOr< T > readUnencodedNumber()
Read a numeric value of type T from the profile.
std::error_code readFuncProfile(const uint8_t *Start)
Read the next function profile instance.
std::error_code readSummary()
Read profile summary.
std::error_code readMagicIdent()
Read the contents of Magic number and Version number.
std::error_code readFuncOffsetTable()
std::vector< SecHdrTableEntry > SecHdrTable
bool collectFuncsFromModule() override
Collect functions with definitions in Module M.
uint64_t getSectionSize(SecType Type)
Get the total size of all Type sections.
std::error_code readCSNameTableSec()
virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry)=0
std::vector< std::pair< SampleContext, uint64_t > > FuncOffsetList
The list version of FuncOffsetTable.
DenseSet< StringRef > FuncsToUse
The set containing the functions to use when compiling a module.
std::error_code readFuncMetadata(bool ProfileHasAttribute, SampleProfileMap &Profiles)
std::unique_ptr< ProfileSymbolList > ProfSymList
std::error_code readSecHdrTable()
std::error_code readFuncProfiles()
bool useFuncOffsetList() const
Determine which container readFuncOffsetTable() should populate, the list FuncOffsetList or the map F...
std::error_code readNameTableSec(bool IsMD5, bool FixedLengthMD5)
std::error_code readSecHdrTableEntry(uint64_t Idx)
std::error_code readImpl() override
Read sample profiles in extensible format from the associated file.
virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry)
bool dumpSectionInfo(raw_ostream &OS=dbgs()) override
DenseMap< hash_code, uint64_t > FuncOffsetTable
The table mapping from a function context's MD5 to the offset of its FunctionSample towards file star...
std::error_code readHeader() override
Read and validate the file header.
uint64_t getFileSize()
Get the total size of header and all sections.
std::error_code readProfileSymbolList()
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
ErrorOr< T > readNumber()
GCOVBuffer GcovBuffer
GCOV buffer containing the profile.
std::vector< std::string > Names
Function names in this profile.
std::error_code readImpl() override
Read sample profiles from the associated file.
std::error_code readNameTable()
std::error_code readHeader() override
Read and validate the file header.
ErrorOr< StringRef > readString()
static const uint32_t GCOVTagAFDOFunction
std::error_code readOneFunctionProfile(const InlineCallStack &InlineStack, bool Update, uint32_t Offset)
std::error_code readFunctionProfiles()
static const uint32_t GCOVTagAFDOFileNames
GCOV tags used to separate sections in the profile file.
std::error_code skipNextWord()
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
std::error_code readSectionTag(uint32_t Expected)
Read the section tag and check that it's the same as Expected.
static ErrorOr< std::unique_ptr< SampleProfileReaderItaniumRemapper > > create(StringRef Filename, vfs::FileSystem &FS, SampleProfileReader &Reader, LLVMContext &C)
Create a remapper from the given remapping file.
void applyRemapping(LLVMContext &Ctx)
Apply remappings to the profile read by Reader.
std::optional< StringRef > lookUpNameInProfile(StringRef FunctionName)
Return the equivalent name in the profile for FunctionName if it exists.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
std::error_code readImpl() override
Read sample profiles from the associated file.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
Sample-based profile reader.
std::pair< const uint8_t *, const uint8_t * > ProfileSecRange
bool ProfileIsPreInlined
Whether function profile contains ShouldBeInlined contexts.
std::unordered_map< uint64_t, std::pair< const uint8_t *, const uint8_t * > > FuncMetadataIndex
SampleProfileMap & getProfiles()
Return all the profiles.
uint32_t CSProfileCount
Number of context-sensitive profiles.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(StringRef Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, StringRef RemapFilename="")
Create a sample profile reader appropriate to the file format.
void dump(raw_ostream &OS=dbgs())
Print all the profiles on stream OS.
bool useMD5() const
Return whether names in the profile are all MD5 numbers.
const Module * M
The current module being compiled if SampleProfileReader is used by compiler.
std::unique_ptr< MemoryBuffer > Buffer
Memory buffer holding the profile file.
std::unique_ptr< SampleProfileReaderItaniumRemapper > Remapper
bool ProfileHasAttribute
Whether the profile has attribute metadata.
bool SkipFlatProf
If SkipFlatProf is true, skip functions marked with !Flat in text mode or sections with SecFlagFlat f...
std::error_code read()
The interface to read sample profiles from the associated file.
MemoryBuffer * getBuffer() const
bool ProfileIsCS
Whether function profiles are context-sensitive flat profiles.
bool ProfileIsMD5
Whether the profile uses MD5 for Sample Contexts and function names.
std::unique_ptr< ProfileSummary > Summary
Profile summary information.
void computeSummary()
Compute summary for this profile.
uint32_t getDiscriminatorMask() const
Get the bitmask the discriminators: For FS profiles, return the bit mask for this pass.
bool ProfileIsFS
Whether the function profiles use FS discriminators.
void dumpJson(raw_ostream &OS=dbgs())
Print all the profiles on stream OS in the JSON format.
SampleProfileMap Profiles
Map every function to its associated profile.
void dumpFunctionProfile(const FunctionSamples &FS, raw_ostream &OS=dbgs())
Print the profile for FunctionSamples on stream OS.
bool ProfileIsProbeBased
Whether samples are collected based on pseudo probes.
void reportError(int64_t LineNumber, const Twine &Msg) const
Report a parse error message.
LLVMContext & Ctx
LLVM context used to emit diagnostics.
Representation of a single sample record.
uint64_t getSamples() const
const SortedCallTargetSet getSortedCallTargets() const
The virtual file system interface.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)
static uint64_t SPMagic(SampleProfileFormat Format=SPF_Binary)
static bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag)
std::map< LineLocation, FunctionSamplesMap > CallsiteSampleMap
@ HIST_TYPE_INDIR_CALL_TOPN
uint64_t MD5Hash(const FunctionId &Obj)
std::map< LineLocation, SampleRecord > BodySampleMap
@ SecFlagIsPreInlined
SecFlagIsPreInlined means this profile contains ShouldBeInlined contexts thus this is CS preinliner c...
@ SecFlagPartial
SecFlagPartial means the profile is for common/shared code.
@ SecFlagFSDiscriminator
SecFlagFSDiscriminator means this profile uses flow-sensitive discriminators.
@ SecFlagFullContext
SecFlagContext means this is context-sensitive flat profile for CSSPGO.
SmallVector< SampleContextFrame, 1 > SampleContextFrameVector
static std::string getSecName(SecType Type)
static uint64_t SPVersion()
uint64_t read64le(const void *P)
void write64le(void *P, uint64_t V)
This is an optimization pass for GlobalISel generic memory operations.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
sampleprof_error mergeSampleProfErrors(sampleprof_error &Accumulator, sampleprof_error Result)
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Represents the relative location of an instruction.
