Movatterモバイル変換

Go to the documentation of this file.

1//===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//

2//

3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

4// See https://llvm.org/LICENSE.txt for license information.

5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

6//

7//===----------------------------------------------------------------------===//

8//

9// This file implements the pruning of a directory based on least recently used.

10//

11//===----------------------------------------------------------------------===//

13#include "llvm/Support/CachePruning.h"

14#include "llvm/ADT/StringRef.h"

15#include "llvm/Support/Debug.h"

16#include "llvm/Support/Errc.h"

17#include "llvm/Support/Error.h"

18#include "llvm/Support/FileSystem.h"

19#include "llvm/Support/Path.h"

20#include "llvm/Support/WithColor.h"

21#include "llvm/Support/raw_ostream.h"

23#define DEBUG_TYPE "cache-pruning"

25#include <set>

26#include <system_error>

28using namespacellvm;

30namespace{

31structFileInfo {

32sys::TimePoint<> Time;

33uint64_t Size;

34 std::string Path;

36 /// Used to determine which files to prune first. Also used to determine

37 /// set membership, so must take into account all fields.

38booloperator<(const FileInfo &Other) const{

39return std::tie(Time,Other.Size, Path) <

40 std::tie(Other.Time, Size,Other.Path);

41 }

42};

43}// anonymous namespace

45/// Write a new timestamp file with the given path. This is used for the pruning

46/// interval option.

47staticvoidwriteTimestampFile(StringRef TimestampFile) {

48 std::error_code EC;

49raw_fd_ostream Out(TimestampFile.str(), EC,sys::fs::OF_None);

50}

52staticExpected<std::chrono::seconds>parseDuration(StringRef Duration) {

53if (Duration.empty())

54return make_error<StringError>("Duration must not be empty",

55inconvertibleErrorCode());

57StringRef NumStr =Duration.slice(0,Duration.size()-1);

58uint64_t Num;

59if (NumStr.getAsInteger(0, Num))

60return make_error<StringError>("'" + NumStr +"' not an integer",

61inconvertibleErrorCode());

63switch (Duration.back()) {

64case's':

65return std::chrono::seconds(Num);

66case'm':

67return std::chrono::minutes(Num);

68case'h':

69return std::chrono::hours(Num);

70default:

71return make_error<StringError>("'" +Duration +

72"' must end with one of 's', 'm' or 'h'",

73inconvertibleErrorCode());

74 }

75}

77Expected<CachePruningPolicy>

78llvm::parseCachePruningPolicy(StringRef PolicyStr) {

79CachePruningPolicy Policy;

80 std::pair<StringRef, StringRef>P = {"", PolicyStr};

81while (!P.second.empty()) {

82P =P.second.split(':');

84StringRef Key,Value;

85 std::tie(Key,Value) =P.first.split('=');

86if (Key =="prune_interval") {

87auto DurationOrErr =parseDuration(Value);

88if (!DurationOrErr)

89return DurationOrErr.takeError();

90 Policy.Interval = *DurationOrErr;

91 }elseif (Key =="prune_after") {

92auto DurationOrErr =parseDuration(Value);

93if (!DurationOrErr)

94return DurationOrErr.takeError();

95 Policy.Expiration = *DurationOrErr;

96 }elseif (Key =="cache_size") {

97if (Value.back() !='%')

98return make_error<StringError>("'" +Value +"' must be a percentage",

99inconvertibleErrorCode());

100StringRef SizeStr =Value.drop_back();

101uint64_t Size;

102if (SizeStr.getAsInteger(0,Size))

103return make_error<StringError>("'" + SizeStr +"' not an integer",

104inconvertibleErrorCode());

105if (Size > 100)

106return make_error<StringError>("'" + SizeStr +

107"' must be between 0 and 100",

108inconvertibleErrorCode());

109 Policy.MaxSizePercentageOfAvailableSpace =Size;

110 }elseif (Key =="cache_size_bytes") {

111uint64_t Mult = 1;

112switch (tolower(Value.back())) {

113case'k':

114 Mult = 1024;

115Value =Value.drop_back();

116break;

117case'm':

118 Mult = 1024 * 1024;

119Value =Value.drop_back();

120break;

121case'g':

122 Mult = 1024 * 1024 * 1024;

123Value =Value.drop_back();

124break;

125 }

126uint64_t Size;

127if (Value.getAsInteger(0,Size))

128return make_error<StringError>("'" +Value +"' not an integer",

129inconvertibleErrorCode());

130 Policy.MaxSizeBytes =Size * Mult;

131 }elseif (Key =="cache_size_files") {

132if (Value.getAsInteger(0, Policy.MaxSizeFiles))

133return make_error<StringError>("'" +Value +"' not an integer",

134inconvertibleErrorCode());

135 }else {

136return make_error<StringError>("Unknown key: '" + Key +"'",

137inconvertibleErrorCode());

138 }

139 }

140

141return Policy;

142}

143

144/// Prune the cache of files that haven't been accessed in a long time.

145boolllvm::pruneCache(StringRef Path,CachePruningPolicy Policy,

146const std::vector<std::unique_ptr<MemoryBuffer>> &Files) {

147using namespacestd::chrono;

148

149if (Path.empty())

150returnfalse;

151

152bool isPathDir;

153if (sys::fs::is_directory(Path, isPathDir))

154returnfalse;

155

156if (!isPathDir)

157returnfalse;

158

159 Policy.MaxSizePercentageOfAvailableSpace =

160 std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);

161

162if (Policy.Expiration == seconds(0) &&

163 Policy.MaxSizePercentageOfAvailableSpace == 0 &&

164 Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {

165LLVM_DEBUG(dbgs() <<"No pruning settings set, exit early\n");

166// Nothing will be pruned, early exit

167returnfalse;

168 }

169

170// Try to stat() the timestamp file.

171SmallString<128> TimestampFile(Path);

172sys::path::append(TimestampFile,"llvmcache.timestamp");

173sys::fs::file_status FileStatus;

174constauto CurrentTime = system_clock::now();

175if (auto EC =sys::fs::status(TimestampFile, FileStatus)) {

176if (EC == errc::no_such_file_or_directory) {

177// If the timestamp file wasn't there, create one now.

178writeTimestampFile(TimestampFile);

179 }else {

180// Unknown error?

181returnfalse;

182 }

183 }else {

184if (!Policy.Interval)

185returnfalse;

186if (Policy.Interval != seconds(0)) {

187// Check whether the time stamp is older than our pruning interval.

188// If not, do nothing.

189constauto TimeStampModTime = FileStatus.getLastModificationTime();

190auto TimeStampAge = CurrentTime - TimeStampModTime;

191if (TimeStampAge <= *Policy.Interval) {

192LLVM_DEBUG(dbgs() <<"Timestamp file too recent ("

193 << duration_cast<seconds>(TimeStampAge).count()

194 <<"s old), do not prune.\n");

195returnfalse;

196 }

197 }

198// Write a new timestamp file so that nobody else attempts to prune.

199// There is a benign race condition here, if two processes happen to

200// notice at the same time that the timestamp is out-of-date.

201writeTimestampFile(TimestampFile);

202 }

203

204// Keep track of files to delete to get below the size limit.

205// Order by time of last use so that recently used files are preserved.

206 std::set<FileInfo> FileInfos;

207uint64_t TotalSize = 0;

208

209// Walk the entire directory cache, looking for unused files.

210 std::error_code EC;

211SmallString<128> CachePathNative;

212sys::path::native(Path, CachePathNative);

213// Walk all of the files within this directory.

214for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;

215 File != FileEnd && !EC; File.increment(EC)) {

216// Ignore filenames not beginning with "llvmcache-" or "Thin-". This

217// includes the timestamp file as well as any files created by the user.

218// This acts as a safeguard against data loss if the user specifies the

219// wrong directory as their cache directory.

220StringRef filename =sys::path::filename(File->path());

221if (!filename.starts_with("llvmcache-") && !filename.starts_with("Thin-"))

222continue;

223

224// Look at this file. If we can't stat it, there's nothing interesting

225// there.

226ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();

227if (!StatusOrErr) {

228LLVM_DEBUG(dbgs() <<"Ignore " << File->path() <<" (can't stat)\n");

229continue;

230 }

231

232// If the file hasn't been used recently enough, delete it

233constauto FileAccessTime = StatusOrErr->getLastAccessedTime();

234auto FileAge = CurrentTime - FileAccessTime;

235if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {

236LLVM_DEBUG(dbgs() <<"Remove " << File->path() <<" ("

237 << duration_cast<seconds>(FileAge).count()

238 <<"s old)\n");

239sys::fs::remove(File->path());

240continue;

241 }

242

243// Leave it here for now, but add it to the list of size-based pruning.

244 TotalSize += StatusOrErr->getSize();

245 FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});

246 }

247

248auto FileInfo = FileInfos.begin();

249size_t NumFiles = FileInfos.size();

250

251auto RemoveCacheFile = [&]() {

252// Remove the file.

253sys::fs::remove(FileInfo->Path);

254// Update size

255 TotalSize -= FileInfo->Size;

256 NumFiles--;

257LLVM_DEBUG(dbgs() <<" - Remove " << FileInfo->Path <<" (size "

258 << FileInfo->Size <<"), new occupancy is " << TotalSize

259 <<"%\n");

260 ++FileInfo;

261 };

262

263// files.size() is greater the number of inputs by one. However, a timestamp

264// file is created and stored in the cache directory if --thinlto-cache-policy

265// option is used. Therefore, files.size() is used as ActualNums.

266constsize_t ActualNums = Files.size();

267if (Policy.MaxSizeFiles && ActualNums > Policy.MaxSizeFiles)

268WithColor::warning()

269 <<"ThinLTO cache pruning happens since the number of created files ("

270 << ActualNums <<") exceeds the maximum number of files ("

271 << Policy.MaxSizeFiles

272 <<"); consider adjusting --thinlto-cache-policy\n";

273

274// Prune for number of files.

275if (Policy.MaxSizeFiles)

276while (NumFiles > Policy.MaxSizeFiles)

277 RemoveCacheFile();

278

279// Prune for size now if needed

280if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {

281auto ErrOrSpaceInfo =sys::fs::disk_space(Path);

282if (!ErrOrSpaceInfo) {

283report_fatal_error("Can't get available size");

284 }

285sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();

286auto AvailableSpace = TotalSize + SpaceInfo.free;

287

288if (Policy.MaxSizePercentageOfAvailableSpace == 0)

289 Policy.MaxSizePercentageOfAvailableSpace = 100;

290if (Policy.MaxSizeBytes == 0)

291 Policy.MaxSizeBytes = AvailableSpace;

292auto TotalSizeTarget = std::min<uint64_t>(

293 AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,

294 Policy.MaxSizeBytes);

295

296LLVM_DEBUG(dbgs() <<"Occupancy: " << ((100 * TotalSize) / AvailableSpace)

297 <<"% target is: "

298 << Policy.MaxSizePercentageOfAvailableSpace <<"%, "

299 << Policy.MaxSizeBytes <<" bytes\n");

300

301size_t ActualSizes = 0;

302for (constauto &File : Files)

303if (File)

304 ActualSizes += File->getBufferSize();

305

306if (ActualSizes > TotalSizeTarget)

307WithColor::warning()

308 <<"ThinLTO cache pruning happens since the total size of the cache "

309"files consumed by the current link job ("

310 << ActualSizes <<" bytes) exceeds maximum cache size ("

311 << TotalSizeTarget

312 <<" bytes); consider adjusting --thinlto-cache-policy\n";

313

314// Remove the oldest accessed files first, till we get below the threshold.

315while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())

316 RemoveCacheFile();

317 }

318returntrue;

319}

parseDuration

static Expected< std::chrono::seconds > parseDuration(StringRef Duration)

Definition:CachePruning.cpp:52

writeTimestampFile

static void writeTimestampFile(StringRef TimestampFile)

Write a new timestamp file with the given path.

Definition:CachePruning.cpp:47

CachePruning.h

Debug.h

LLVM_DEBUG

#define LLVM_DEBUG(...)

Definition:Debug.h:106

Size

uint64_t Size

Definition:ELFObjHandler.cpp:81

Other

std::optional< std::vector< StOtherPiece > > Other

Definition:ELFYAML.cpp:1315

#define P(N)

Definition:Duration.h:20

llvm::ErrorOr

Represents either an error or a value T.

Definition:ErrorOr.h:56

llvm::Expected

Tagged union holding either a T or a Error.

Definition:Error.h:481

llvm::SmallString

SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...

Definition:SmallString.h:26

llvm::StringRef

StringRef - Represent a constant reference to a string, i.e.

Definition:StringRef.h:51

llvm::StringRef::getAsInteger

bool getAsInteger(unsigned Radix, T &Result) const

Parse the current string as an integer of the specified radix.

Definition:StringRef.h:470

llvm::StringRef::str

std::string str() const

str - Get the contents as an std::string.

Definition:StringRef.h:229

llvm::StringRef::starts_with

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

Definition:StringRef.h:265

llvm::Value

LLVM Value Representation.

Definition:Value.h:74

llvm::WithColor::warning

static raw_ostream & warning()

Convenience method for printing "warning: " to stderr.

Definition:WithColor.cpp:85

llvm::raw_fd_ostream

A raw_ostream that writes to a file descriptor.

Definition:raw_ostream.h:460

llvm::sys::fs::basic_file_status::getLastModificationTime

TimePoint getLastModificationTime() const

The file modification time as reported from the underlying file system.

llvm::sys::fs::directory_iterator

directory_iterator - Iterates through the entries in path.

Definition:FileSystem.h:1416

llvm::sys::fs::file_status

Represents the result of a call to sys::fs::status().

Definition:FileSystem.h:221

uint64_t

Error.h

llvm::sys::fs::status

std::error_code status(const Twine &path, file_status &result, bool follow=true)

Get file status as if by POSIX stat().

llvm::sys::fs::OF_None

@ OF_None

Definition:FileSystem.h:750

llvm::sys::fs::remove

std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)

Remove path.

llvm::sys::fs::disk_space

ErrorOr< space_info > disk_space(const Twine &Path)

Get disk space usage information.

llvm::sys::fs::is_directory

bool is_directory(const basic_file_status &status)

Does status represent a directory?

Definition:Path.cpp:1092

llvm::sys::path::Style::native

@ native

llvm::sys::path::filename

StringRef filename(StringRef path LLVM_LIFETIME_BOUND, Style style=Style::native)

Get filename.

Definition:Path.cpp:577

llvm::sys::path::append

void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")

Append to path.

Definition:Path.cpp:456

llvm::sys::TimePoint

std::chrono::time_point< std::chrono::system_clock, D > TimePoint

A time point on the system clock.

Definition:Chrono.h:34

llvm

This is an optimization pass for GlobalISel generic memory operations.

Definition:AddressRanges.h:18

llvm::operator<

bool operator<(int64_t V1, const APSInt &V2)

Definition:APSInt.h:361

llvm::inconvertibleErrorCode

std::error_code inconvertibleErrorCode()

The value returned by this function can be returned from convertToErrorCode for Error values where no...

Definition:Error.cpp:98

llvm::parseCachePruningPolicy

Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)

Parse the given string as a cache pruning policy.

Definition:CachePruning.cpp:78

llvm::dbgs

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

Definition:Debug.cpp:163

llvm::report_fatal_error

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

Definition:Error.cpp:167

llvm::pruneCache

bool pruneCache(StringRef Path, CachePruningPolicy Policy, const std::vector< std::unique_ptr< MemoryBuffer > > &Files={})

Peform pruning using the supplied policy, returns true if pruning occurred, i.e.

Definition:CachePruning.cpp:145

llvm::count

auto count(R &&Range, const E &Element)

Wrapper function around std::count to count the number of times an element Element occurs in the give...

Definition:STLExtras.h:1938

raw_ostream.h

llvm::CachePruningPolicy

Policy for the pruneCache() function.

Definition:CachePruning.h:28

llvm::CachePruningPolicy::MaxSizeFiles

uint64_t MaxSizeFiles

The maximum number of files in the cache directory.

Definition:CachePruning.h:61

llvm::CachePruningPolicy::Interval

std::optional< std::chrono::seconds > Interval

The pruning interval.

Definition:CachePruning.h:33

llvm::CachePruningPolicy::Expiration

std::chrono::seconds Expiration

The expiration for a file.

Definition:CachePruning.h:38

llvm::CachePruningPolicy::MaxSizeBytes

uint64_t MaxSizeBytes

The maximum size for the cache directory in bytes.

Definition:CachePruning.h:50

llvm::CachePruningPolicy::MaxSizePercentageOfAvailableSpace

unsigned MaxSizePercentageOfAvailableSpace

The maximum size for the cache directory, in terms of percentage of the available space on the disk.

Definition:CachePruning.h:45

llvm::sys::fs::space_info

space_info - Self explanatory.

Definition:FileSystem.h:75

llvm::sys::fs::space_info::free

uint64_t free

Definition:FileSystem.h:77