Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit67d569d

Browse files
André Rnicolas-grekas
André R
authored andcommitted
[Cache] Improve RedisTagAwareAdapter invalidation logic & requirements
1 parenta0bbae7 commit67d569d

File tree

4 files changed

+86
-95
lines changed

4 files changed

+86
-95
lines changed

‎src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php‎

Lines changed: 72 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,32 @@
1111

1212
namespaceSymfony\Component\Cache\Adapter;
1313

14-
usePredis;
1514
usePredis\Connection\Aggregate\ClusterInterface;
15+
usePredis\Connection\Aggregate\PredisCluster;
1616
usePredis\Response\Status;
17-
useSymfony\Component\Cache\CacheItem;
18-
useSymfony\Component\Cache\Exception\LogicException;
17+
useSymfony\Component\Cache\Exception\InvalidArgumentException;
1918
useSymfony\Component\Cache\Marshaller\MarshallerInterface;
2019
useSymfony\Component\Cache\Traits\RedisTrait;
2120

2221
/**
23-
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation usingsPOP.
22+
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation usingRENAME+SMEMBERS.
2423
*
2524
* Set (tag relation info) is stored without expiry (non-volatile), while cache always gets an expiry (volatile) even
2625
* if not set by caller. Thus if you configure redis with the right eviction policy you can be safe this tag <> cache
2726
* relationship survives eviction (cache cleanup when Redis runs out of memory).
2827
*
2928
* Requirements:
30-
* - Server: Redis 3.2+
31-
* - Client: PHP Redis 3.1.3+ OR Predis
32-
* - Redis Server(s) configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
29+
* - Client: PHP Redis or Predis
30+
* Note: Due to lack of RENAME support it is NOT recommended to use Cluster on Predis, instead use phpredis.
31+
* - Server: Redis 2.8+
32+
* Configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
3333
*
3434
* Design limitations:
35-
* - Max2 billion cache keys per cache tag
36-
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to2 billion cache itemsas well
35+
* - Max4 billion cache keys per cache tag as limited by Redis Set datatype.
36+
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to4 billion cache itemsalso.
3737
*
3838
* @see https://redis.io/topics/lru-cache#eviction-policies Documentation for Redis eviction policies.
3939
* @see https://redis.io/topics/data-types#sets Documentation for Redis Set datatype.
40-
* @see https://redis.io/commands/spop Documentation for sPOP operation, capable of retriving AND emptying a Set at once.
4140
*
4241
* @author Nicolas Grekas <p@tchwork.com>
4342
* @author André Rømcke <andre.romcke+symfony@gmail.com>
@@ -46,11 +45,6 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
4645
{
4746
use RedisTrait;
4847

49-
/**
50-
* Redis "Set" can hold more than 4 billion members, here we limit ourselves to PHP's > 2 billion max int (32Bit).
51-
*/
52-
privateconstPOP_MAX_LIMIT =2147483647 -1;
53-
5448
/**
5549
* Limits for how many keys are deleted in batch.
5650
*/
@@ -62,26 +56,18 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
6256
*/
6357
privateconstDEFAULT_CACHE_TTL =8640000;
6458

65-
/**
66-
* @var bool|null
67-
*/
68-
private$redisServerSupportSPOP =null;
69-
7059
/**
7160
* @param \Redis|\RedisArray|\RedisCluster|\Predis\ClientInterface $redisClient The redis client
7261
* @param string $namespace The default namespace
7362
* @param int $defaultLifetime The default lifetime
74-
*
75-
* @throws \Symfony\Component\Cache\Exception\LogicException If phpredis with version lower than 3.1.3.
7663
*/
7764
publicfunction__construct($redisClient,string$namespace ='',int$defaultLifetime =0,MarshallerInterface$marshaller =null)
7865
{
79-
$this->init($redisClient,$namespace,$defaultLifetime,$marshaller);
80-
81-
// Make sure php-redis is 3.1.3 or higher configured for Redis classes
82-
if (!$this->redisinstanceof \Predis\ClientInterface &&version_compare(phpversion('redis'),'3.1.3','<')) {
83-
thrownewLogicException('RedisTagAwareAdapter requires php-redis 3.1.3 or higher, alternatively use predis/predis');
66+
if ($redisClientinstanceof \Predis\ClientInterface &&$redisClient->getConnection()instanceof ClusterInterface && !$redisClient->getConnection()instanceof PredisCluster) {
67+
thrownewInvalidArgumentException(sprintf('Unsupported Predis cluster connection: only "%s" is, "%s" given.', PredisCluster::class,\get_class($redisClient->getConnection())));
8468
}
69+
70+
$this->init($redisClient,$namespace,$defaultLifetime,$marshaller);
8571
}
8672

8773
/**
@@ -138,9 +124,10 @@ protected function doDelete(array $ids, array $tagData = []): bool
138124
returntrue;
139125
}
140126

141-
$predisCluster =$this->redisinstanceof \Predis\ClientInterface &&$this->redis->getConnection()instanceofClusterInterface;
127+
$predisCluster =$this->redisinstanceof \Predis\ClientInterface &&$this->redis->getConnection()instanceofPredisCluster;
142128
$this->pipeline(staticfunction ()use ($ids,$tagData,$predisCluster) {
143129
if ($predisCluster) {
130+
// Unlike phpredis, Predis does not handle bulk calls for us against cluster
144131
foreach ($idsas$id) {
145132
yield'del' => [$id];
146133
}
@@ -161,46 +148,82 @@ protected function doDelete(array $ids, array $tagData = []): bool
161148
*/
162149
protectedfunctiondoInvalidate(array$tagIds):bool
163150
{
164-
if (!$this->redisServerSupportSPOP()) {
151+
if (($redis =$this->redis)instanceof \Predis\ClientInterface) {
152+
$options =clone$redis->getOptions();
153+
\Closure::bind(function () {$this->options['exceptions'] =false; },$options,$options)();
154+
$redis =new$redis($redis->getConnection(),$options);
155+
}
156+
157+
if (!$redisinstanceof \Predis\ClientInterface || !$redis->getConnection()instanceof PredisCluster) {
158+
$movedTagSetIds =$this->renameKeys($redis,$tagIds);
159+
}else {
160+
$clusterConnection =$redis->getConnection();
161+
$tagIdsByConnection =new \SplObjectStorage();
162+
$movedTagSetIds = [];
163+
164+
foreach ($tagIdsas$id) {
165+
$connection =$clusterConnection->getConnectionByKey($id);
166+
$slot =$tagIdsByConnection[$connection] ??$tagIdsByConnection[$connection] =new \ArrayObject();
167+
$slot[] =$id;
168+
}
169+
170+
foreach ($tagIdsByConnectionas$connection) {
171+
$slot =$tagIdsByConnection[$connection];
172+
$movedTagSetIds =array_merge($movedTagSetIds,$this->renameKeys(new$redis($connection,$options),$slot->getArrayCopy()));
173+
}
174+
}
175+
176+
// No Sets found
177+
if (!$movedTagSetIds) {
165178
returnfalse;
166179
}
167180

168-
// Pop all tag info at once to avoid race conditions
169-
$tagIdSets =$this->pipeline(staticfunction ()use ($tagIds) {
170-
foreach ($tagIdsas$tagId) {
171-
// Client: Predis or PHP Redis 3.1.3+ (https://github.com/phpredis/phpredis/commit/d2e203a6)
172-
// Server: Redis 3.2 or higher (https://redis.io/commands/spop)
173-
yield'sPop' => [$tagId,self::POP_MAX_LIMIT];
181+
// Now safely take the time to read the keys in each set and collect ids we need to delete
182+
$tagIdSets =$this->pipeline(staticfunction ()use ($movedTagSetIds) {
183+
foreach ($movedTagSetIdsas$movedTagId) {
184+
yield'sMembers' => [$movedTagId];
174185
}
175186
});
176187

177-
//Flatten generator result from pipeline, ignore keys (tag ids)
178-
$ids =array_unique(array_merge(...iterator_to_array($tagIdSets,false)));
188+
//Return combination of the temporary Tag Set ids and their values (cache ids)
189+
$ids =array_merge($movedTagSetIds,...iterator_to_array($tagIdSets,false));
179190

180191
// Delete cache in chunks to avoid overloading the connection
181-
foreach (array_chunk($ids,self::BULK_DELETE_LIMIT)as$chunkIds) {
192+
foreach (array_chunk(array_unique($ids),self::BULK_DELETE_LIMIT)as$chunkIds) {
182193
$this->doDelete($chunkIds);
183194
}
184195

185196
returntrue;
186197
}
187198

188-
privatefunctionredisServerSupportSPOP():bool
199+
/**
200+
* Renames several keys in order to be able to operate on them without risk of race conditions.
201+
*
202+
* Filters out keys that do not exist before returning new keys.
203+
*
204+
* @see https://redis.io/commands/rename
205+
* @see https://redis.io/topics/cluster-spec#keys-hash-tags
206+
*
207+
* @return array Filtered list of the valid moved keys (only those that existed)
208+
*/
209+
privatefunctionrenameKeys($redis,array$ids):array
189210
{
190-
if (null !==$this->redisServerSupportSPOP) {
191-
return$this->redisServerSupportSPOP;
192-
}
211+
$newIds = [];
212+
$uniqueToken =bin2hex(random_bytes(10));
193213

194-
foreach ($this->getHosts()as$host) {
195-
$info =$host->info('Server');
196-
$info =isset($info['Server']) ?$info['Server'] :$info;
197-
if (version_compare($info['redis_version'],'3.2','<')) {
198-
CacheItem::log($this->logger,'Redis server needs to be version 3.2 or higher, your Redis server was detected as'.$info['redis_version']);
214+
$results =$this->pipeline(staticfunction ()use ($ids,$uniqueToken) {
215+
foreach ($idsas$id) {
216+
yield'rename' => [$id,'{'.$id.'}'.$uniqueToken];
217+
}
218+
},$redis);
199219

200-
return$this->redisServerSupportSPOP =false;
220+
foreach ($resultsas$id =>$ok) {
221+
if (true ===$ok || ($okinstanceof Status && Status::get('OK') ===$ok)) {
222+
// Only take into account if ok (key existed), will be false on phpredis if it did not exist
223+
$newIds[] ='{'.$id.'}'.$uniqueToken;
201224
}
202225
}
203226

204-
return$this->redisServerSupportSPOP =true;
227+
return$newIds;
205228
}
206229
}

‎src/Symfony/Component/Cache/CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ CHANGELOG
66

77
* added support for connecting to Redis Sentinel clusters
88
* added argument`$prefix` to`AdapterInterface::clear()`
9+
* improved`RedisTagAwareAdapter` to support Redis server >= 2.8 and up to 4B items per tag
10+
*[BC BREAK]`RedisTagAwareAdapter` is not compatible with`RedisCluster` from`Predis` anymore, use`phpredis` instead
911

1012
4.3.0
1113
-----

‎src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎src/Symfony/Component/Cache/Traits/RedisTrait.php‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -423,31 +423,32 @@ protected function doSave(array $values, $lifetime)
423423
return$failed;
424424
}
425425

426-
privatefunctionpipeline(\Closure$generator):\Generator
426+
privatefunctionpipeline(\Closure$generator,$redis =null):\Generator
427427
{
428428
$ids = [];
429+
$redis =$redis ??$this->redis;
429430

430-
if ($this->redisinstanceof RedisClusterProxy ||$this->redisinstanceof \RedisCluster || ($this->redisinstanceof \Predis\ClientInterface &&$this->redis->getConnection()instanceof RedisCluster)) {
431+
if ($redisinstanceof RedisClusterProxy ||$redisinstanceof \RedisCluster || ($redisinstanceof \Predis\ClientInterface &&$redis->getConnection()instanceof RedisCluster)) {
431432
// phpredis & predis don't support pipelining with RedisCluster
432433
// see https://github.com/phpredis/phpredis/blob/develop/cluster.markdown#pipelining
433434
// see https://github.com/nrk/predis/issues/267#issuecomment-123781423
434435
$results = [];
435436
foreach ($generator()as$command =>$args) {
436-
$results[] =$this->redis->{$command}(...$args);
437+
$results[] =$redis->{$command}(...$args);
437438
$ids[] =$args[0];
438439
}
439-
}elseif ($this->redisinstanceof \Predis\ClientInterface) {
440-
$results =$this->redis->pipeline(function ($redis)use ($generator, &$ids) {
440+
}elseif ($redisinstanceof \Predis\ClientInterface) {
441+
$results =$redis->pipeline(staticfunction ($redis)use ($generator, &$ids) {
441442
foreach ($generator()as$command =>$args) {
442443
$redis->{$command}(...$args);
443444
$ids[] =$args[0];
444445
}
445446
});
446-
}elseif ($this->redisinstanceof \RedisArray) {
447+
}elseif ($redisinstanceof \RedisArray) {
447448
$connections =$results =$ids = [];
448449
foreach ($generator()as$command =>$args) {
449-
if (!isset($connections[$h =$this->redis->_target($args[0])])) {
450-
$connections[$h] = [$this->redis->_instance($h), -1];
450+
if (!isset($connections[$h =$redis->_target($args[0])])) {
451+
$connections[$h] = [$redis->_instance($h), -1];
451452
$connections[$h][0]->multi(\Redis::PIPELINE);
452453
}
453454
$connections[$h][0]->{$command}(...$args);
@@ -461,12 +462,12 @@ private function pipeline(\Closure $generator): \Generator
461462
$results[$k] =$connections[$h][$c];
462463
}
463464
}else {
464-
$this->redis->multi(\Redis::PIPELINE);
465+
$redis->multi(\Redis::PIPELINE);
465466
foreach ($generator()as$command =>$args) {
466-
$this->redis->{$command}(...$args);
467+
$redis->{$command}(...$args);
467468
$ids[] =$args[0];
468469
}
469-
$results =$this->redis->exec();
470+
$results =$redis->exec();
470471
}
471472

472473
foreach ($idsas$k =>$id) {

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp