Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitfbf55c2

Browse files
feature#33461 [Cache] Improve RedisTagAwareAdapter invalidation logic & requirements (andrerom)
This PR was merged into the 4.4 branch.Discussion----------[Cache] Improve RedisTagAwareAdapter invalidation logic & requirements| Q | A| ------------- | ---| Branch? | 4.4| Bug fix? | yes, _and improvment_| New feature? | no| BC breaks? | no| Deprecations? | no| Tests pass? | yes| License | MIT| Doc PR |Changes logic of invalidation in RedisTagAwareAdapter in order to:- Delete the tag key on invalidation => _avoiding possible left behind empty tag keys that Redis is not allowed to evict, gradually consuming more and more memory_Positive side effects of no longer using sPOP:- Lowered requirements to Redis 2.8, and no specific version constraint for phpredis- Lift limitation of 2 billion keys per tag _(Now only limited by Redis Set datatype: 4 billion)_Commits-------3d38c58 [Cache] Improve RedisTagAwareAdapter invalidation logic & requirements
2 parentse3b513b +3d38c58 commitfbf55c2

File tree

4 files changed

+92
-97
lines changed

4 files changed

+92
-97
lines changed

‎src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php‎

Lines changed: 67 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,32 @@
1111

1212
namespaceSymfony\Component\Cache\Adapter;
1313

14-
usePredis;
1514
usePredis\Connection\Aggregate\ClusterInterface;
15+
usePredis\Connection\Aggregate\PredisCluster;
1616
usePredis\Response\Status;
17-
useSymfony\Component\Cache\CacheItem;
18-
useSymfony\Component\Cache\Exception\LogicException;
17+
useSymfony\Component\Cache\Exception\InvalidArgumentException;
1918
useSymfony\Component\Cache\Marshaller\MarshallerInterface;
2019
useSymfony\Component\Cache\Traits\RedisTrait;
2120

2221
/**
23-
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation usingsPOP.
22+
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation usingRENAME+SMEMBERS.
2423
*
2524
* Set (tag relation info) is stored without expiry (non-volatile), while cache always gets an expiry (volatile) even
2625
* if not set by caller. Thus if you configure redis with the right eviction policy you can be safe this tag <> cache
2726
* relationship survives eviction (cache cleanup when Redis runs out of memory).
2827
*
2928
* Requirements:
30-
* - Server: Redis 3.2+
31-
* - Client: PHP Redis 3.1.3+ OR Predis
32-
* - Redis Server(s) configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
29+
* - Client: PHP Redis or Predis
30+
* Note: Due to lack of RENAME support it is NOT recommended to use Cluster on Predis, instead use phpredis.
31+
* - Server: Redis 2.8+
32+
* Configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
3333
*
3434
* Design limitations:
35-
* - Max2 billion cache keys per cache tag
36-
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to2 billion cache itemsas well
35+
* - Max4 billion cache keys per cache tag as limited by Redis Set datatype.
36+
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to4 billion cache itemsalso.
3737
*
3838
* @see https://redis.io/topics/lru-cache#eviction-policies Documentation for Redis eviction policies.
3939
* @see https://redis.io/topics/data-types#sets Documentation for Redis Set datatype.
40-
* @see https://redis.io/commands/spop Documentation for sPOP operation, capable of retriving AND emptying a Set at once.
4140
*
4241
* @author Nicolas Grekas <p@tchwork.com>
4342
* @author André Rømcke <andre.romcke+symfony@gmail.com>
@@ -46,11 +45,6 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
4645
{
4746
use RedisTrait;
4847

49-
/**
50-
* Redis "Set" can hold more than 4 billion members, here we limit ourselves to PHP's > 2 billion max int (32Bit).
51-
*/
52-
privateconstPOP_MAX_LIMIT =2147483647 -1;
53-
5448
/**
5549
* Limits for how many keys are deleted in batch.
5650
*/
@@ -62,26 +56,18 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
6256
*/
6357
privateconstDEFAULT_CACHE_TTL =8640000;
6458

65-
/**
66-
* @var bool|null
67-
*/
68-
private$redisServerSupportSPOP =null;
69-
7059
/**
7160
* @param \Redis|\RedisArray|\RedisCluster|\Predis\ClientInterface $redisClient The redis client
7261
* @param string $namespace The default namespace
7362
* @param int $defaultLifetime The default lifetime
74-
*
75-
* @throws \Symfony\Component\Cache\Exception\LogicException If phpredis with version lower than 3.1.3.
7663
*/
7764
publicfunction__construct($redisClient,string$namespace ='',int$defaultLifetime =0,MarshallerInterface$marshaller =null)
7865
{
79-
$this->init($redisClient,$namespace,$defaultLifetime,$marshaller);
80-
81-
// Make sure php-redis is 3.1.3 or higher configured for Redis classes
82-
if (!$this->redisinstanceof \Predis\ClientInterface &&version_compare(phpversion('redis'),'3.1.3','<')) {
83-
thrownewLogicException('RedisTagAwareAdapter requires php-redis 3.1.3 or higher, alternatively use predis/predis');
66+
if ($redisClientinstanceof \Predis\ClientInterface &&$redisClient->getConnection()instanceof ClusterInterface && !$redisClient->getConnection()instanceof PredisCluster) {
67+
thrownewInvalidArgumentException(sprintf('Unsupported Predis cluster connection: only "%s" is, "%s" given.', PredisCluster::class,\get_class($redisClient->getConnection())));
8468
}
69+
70+
$this->init($redisClient,$namespace,$defaultLifetime,$marshaller);
8571
}
8672

8773
/**
@@ -121,7 +107,7 @@ protected function doSave(array $values, ?int $lifetime, array $addTagData = [],
121107
continue;
122108
}
123109
// setEx results
124-
if (true !==$result && (!$resultinstanceof Status ||$result !==Status::get('OK'))) {
110+
if (true !==$result && (!$resultinstanceof Status || Status::get('OK') !==$result)) {
125111
$failed[] =$id;
126112
}
127113
}
@@ -138,9 +124,10 @@ protected function doDelete(array $ids, array $tagData = []): bool
138124
returntrue;
139125
}
140126

141-
$predisCluster =$this->redisinstanceof \Predis\ClientInterface &&$this->redis->getConnection()instanceofClusterInterface;
127+
$predisCluster =$this->redisinstanceof \Predis\ClientInterface &&$this->redis->getConnection()instanceofPredisCluster;
142128
$this->pipeline(staticfunction ()use ($ids,$tagData,$predisCluster) {
143129
if ($predisCluster) {
130+
// Unlike phpredis, Predis does not handle bulk calls for us against cluster
144131
foreach ($idsas$id) {
145132
yield'del' => [$id];
146133
}
@@ -161,46 +148,76 @@ protected function doDelete(array $ids, array $tagData = []): bool
161148
*/
162149
protectedfunctiondoInvalidate(array$tagIds):bool
163150
{
164-
if (!$this->redisServerSupportSPOP()) {
151+
if (!$this->redisinstanceof \Predis\ClientInterface || !$this->redis->getConnection()instanceof PredisCluster) {
152+
$movedTagSetIds =$this->renameKeys($this->redis,$tagIds);
153+
}else {
154+
$clusterConnection =$this->redis->getConnection();
155+
$tagIdsByConnection =new \SplObjectStorage();
156+
$movedTagSetIds = [];
157+
158+
foreach ($tagIdsas$id) {
159+
$connection =$clusterConnection->getConnectionByKey($id);
160+
$slot =$tagIdsByConnection[$connection] ??$tagIdsByConnection[$connection] =new \ArrayObject();
161+
$slot[] =$id;
162+
}
163+
164+
foreach ($tagIdsByConnectionas$connection) {
165+
$slot =$tagIdsByConnection[$connection];
166+
$movedTagSetIds =array_merge($movedTagSetIds,$this->renameKeys(new$this->redis($connection,$this->redis->getOptions()),$slot->getArrayCopy()));
167+
}
168+
}
169+
170+
// No Sets found
171+
if (!$movedTagSetIds) {
165172
returnfalse;
166173
}
167174

168-
// Pop all tag info at once to avoid race conditions
169-
$tagIdSets =$this->pipeline(staticfunction ()use ($tagIds) {
170-
foreach ($tagIdsas$tagId) {
171-
// Client: Predis or PHP Redis 3.1.3+ (https://github.com/phpredis/phpredis/commit/d2e203a6)
172-
// Server: Redis 3.2 or higher (https://redis.io/commands/spop)
173-
yield'sPop' => [$tagId,self::POP_MAX_LIMIT];
175+
// Now safely take the time to read the keys in each set and collect ids we need to delete
176+
$tagIdSets =$this->pipeline(staticfunction ()use ($movedTagSetIds) {
177+
foreach ($movedTagSetIdsas$movedTagId) {
178+
yield'sMembers' => [$movedTagId];
174179
}
175180
});
176181

177-
//Flatten generator result from pipeline, ignore keys (tag ids)
178-
$ids =array_unique(array_merge(...iterator_to_array($tagIdSets,false)));
182+
//Return combination of the temporary Tag Set ids and their values (cache ids)
183+
$ids =array_merge($movedTagSetIds,...iterator_to_array($tagIdSets,false));
179184

180185
// Delete cache in chunks to avoid overloading the connection
181-
foreach (array_chunk($ids,self::BULK_DELETE_LIMIT)as$chunkIds) {
186+
foreach (array_chunk(array_unique($ids),self::BULK_DELETE_LIMIT)as$chunkIds) {
182187
$this->doDelete($chunkIds);
183188
}
184189

185190
returntrue;
186191
}
187192

188-
privatefunctionredisServerSupportSPOP():bool
193+
/**
194+
* Renames several keys in order to be able to operate on them without risk of race conditions.
195+
*
196+
* Filters out keys that do not exist before returning new keys.
197+
*
198+
* @see https://redis.io/commands/rename
199+
* @see https://redis.io/topics/cluster-spec#keys-hash-tags
200+
*
201+
* @return array Filtered list of the valid moved keys (only those that existed)
202+
*/
203+
privatefunctionrenameKeys($redis,array$ids):array
189204
{
190-
if (null !==$this->redisServerSupportSPOP) {
191-
return$this->redisServerSupportSPOP;
192-
}
205+
$newIds = [];
206+
$uniqueToken =bin2hex(random_bytes(10));
193207

194-
foreach ($this->getHosts()as$host) {
195-
$info =$host->info('Server');
196-
$info =isset($info['Server']) ?$info['Server'] :$info;
197-
if (version_compare($info['redis_version'],'3.2','<')) {
198-
CacheItem::log($this->logger,'Redis server needs to be version 3.2 or higher, your Redis server was detected as'.$info['redis_version']);
208+
$results =$this->pipeline(staticfunction ()use ($ids,$uniqueToken) {
209+
foreach ($idsas$id) {
210+
yield'rename' => [$id,'{'.$id.'}'.$uniqueToken];
211+
}
212+
},$redis);
199213

200-
return$this->redisServerSupportSPOP =false;
214+
foreach ($resultsas$id =>$result) {
215+
if (true ===$result || ($resultinstanceof Status && Status::get('OK') ===$result)) {
216+
// Only take into account if ok (key existed), will be false on phpredis if it did not exist
217+
$newIds[] ='{'.$id.'}'.$uniqueToken;
201218
}
202219
}
203220

204-
return$this->redisServerSupportSPOP =true;
221+
return$newIds;
205222
}
206223
}

‎src/Symfony/Component/Cache/CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ CHANGELOG
66

77
* added support for connecting to Redis Sentinel clusters
88
* added argument`$prefix` to`AdapterInterface::clear()`
9+
* improved`RedisTagAwareAdapter` to support Redis server >= 2.8 and up to 4B items per tag
10+
*[BC BREAK]`RedisTagAwareAdapter` is not compatible with`RedisCluster` from`Predis` anymore, use`phpredis` instead
911

1012
4.3.0
1113
-----

‎src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎src/Symfony/Component/Cache/Traits/RedisTrait.php‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,17 @@ private function init($redisClient, string $namespace, int $defaultLifetime, ?Ma
5555
if (preg_match('#[^-+_.A-Za-z0-9]#',$namespace,$match)) {
5656
thrownewInvalidArgumentException(sprintf('RedisAdapter namespace contains "%s" but only characters in [-+_.A-Za-z0-9] are allowed.',$match[0]));
5757
}
58+
5859
if (!$redisClientinstanceof \Redis && !$redisClientinstanceof \RedisArray && !$redisClientinstanceof \RedisCluster && !$redisClientinstanceof \Predis\ClientInterface && !$redisClientinstanceof RedisProxy && !$redisClientinstanceof RedisClusterProxy) {
5960
thrownewInvalidArgumentException(sprintf('%s() expects parameter 1 to be Redis, RedisArray, RedisCluster or Predis\ClientInterface, %s given.',__METHOD__,\is_object($redisClient) ?\get_class($redisClient) :\gettype($redisClient)));
6061
}
62+
63+
if ($redisClientinstanceof \Predis\ClientInterface &&$redisClient->getOptions()->exceptions) {
64+
$options =clone$redisClient->getOptions();
65+
\Closure::bind(function () {$this->options['exceptions'] =false; },$options,$options)();
66+
$redisClient =new$redisClient($redisClient->getConnection(),$options);
67+
}
68+
6169
$this->redis =$redisClient;
6270
$this->marshaller =$marshaller ??newDefaultMarshaller();
6371
}
@@ -277,6 +285,7 @@ public static function createConnection($dsn, array $options = [])
277285
$params['replication'] =true;
278286
$hosts[0] += ['alias' =>'master'];
279287
}
288+
$params['exceptions'] =false;
280289

281290
$redis =new$class($hosts,array_diff_key($params,self::$defaultConnectionOptions));
282291
if (isset($params['redis_sentinel'])) {
@@ -414,40 +423,42 @@ protected function doSave(array $values, $lifetime)
414423
}
415424
}
416425
});
426+
417427
foreach ($resultsas$id =>$result) {
418-
if (true !==$result && (!$resultinstanceof Status ||$result !==Status::get('OK'))) {
428+
if (true !==$result && (!$resultinstanceof Status || Status::get('OK') !==$result)) {
419429
$failed[] =$id;
420430
}
421431
}
422432

423433
return$failed;
424434
}
425435

426-
privatefunctionpipeline(\Closure$generator):\Generator
436+
privatefunctionpipeline(\Closure$generator,$redis =null):\Generator
427437
{
428438
$ids = [];
439+
$redis =$redis ??$this->redis;
429440

430-
if ($this->redisinstanceof RedisClusterProxy ||$this->redisinstanceof \RedisCluster || ($this->redisinstanceof \Predis\ClientInterface &&$this->redis->getConnection()instanceof RedisCluster)) {
441+
if ($redisinstanceof RedisClusterProxy ||$redisinstanceof \RedisCluster || ($redisinstanceof \Predis\ClientInterface &&$redis->getConnection()instanceof RedisCluster)) {
431442
// phpredis & predis don't support pipelining with RedisCluster
432443
// see https://github.com/phpredis/phpredis/blob/develop/cluster.markdown#pipelining
433444
// see https://github.com/nrk/predis/issues/267#issuecomment-123781423
434445
$results = [];
435446
foreach ($generator()as$command =>$args) {
436-
$results[] =$this->redis->{$command}(...$args);
447+
$results[] =$redis->{$command}(...$args);
437448
$ids[] =$args[0];
438449
}
439-
}elseif ($this->redisinstanceof \Predis\ClientInterface) {
440-
$results =$this->redis->pipeline(function ($redis)use ($generator, &$ids) {
450+
}elseif ($redisinstanceof \Predis\ClientInterface) {
451+
$results =$redis->pipeline(staticfunction ($redis)use ($generator, &$ids) {
441452
foreach ($generator()as$command =>$args) {
442453
$redis->{$command}(...$args);
443454
$ids[] =$args[0];
444455
}
445456
});
446-
}elseif ($this->redisinstanceof \RedisArray) {
457+
}elseif ($redisinstanceof \RedisArray) {
447458
$connections =$results =$ids = [];
448459
foreach ($generator()as$command =>$args) {
449-
if (!isset($connections[$h =$this->redis->_target($args[0])])) {
450-
$connections[$h] = [$this->redis->_instance($h), -1];
460+
if (!isset($connections[$h =$redis->_target($args[0])])) {
461+
$connections[$h] = [$redis->_instance($h), -1];
451462
$connections[$h][0]->multi(\Redis::PIPELINE);
452463
}
453464
$connections[$h][0]->{$command}(...$args);
@@ -461,12 +472,12 @@ private function pipeline(\Closure $generator): \Generator
461472
$results[$k] =$connections[$h][$c];
462473
}
463474
}else {
464-
$this->redis->multi(\Redis::PIPELINE);
475+
$redis->multi(\Redis::PIPELINE);
465476
foreach ($generator()as$command =>$args) {
466-
$this->redis->{$command}(...$args);
477+
$redis->{$command}(...$args);
467478
$ids[] =$args[0];
468479
}
469-
$results =$this->redis->exec();
480+
$results =$redis->exec();
470481
}
471482

472483
foreach ($idsas$k =>$id) {

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp