summaryrefslogtreecommitdiff
path: root/includes/cache/bloom
diff options
context:
space:
mode:
authorLuke Shumaker <lukeshu@sbcglobal.net>2016-05-01 15:12:12 -0400
committerLuke Shumaker <lukeshu@sbcglobal.net>2016-05-01 15:12:12 -0400
commitc9aa36da061816dee256a979c2ff8d2ee41824d9 (patch)
tree29f7002b80ee984b488bd047dbbd80b36bf892e9 /includes/cache/bloom
parentb4274e0e33eafb5e9ead9d949ebf031a9fb8363b (diff)
parentd1ba966140d7a60cd5ae4e8667ceb27c1a138592 (diff)
Merge branch 'archwiki'
# Conflicts: # skins/ArchLinux.php # skins/ArchLinux/archlogo.gif
Diffstat (limited to 'includes/cache/bloom')
-rw-r--r--includes/cache/bloom/BloomCache.php323
-rw-r--r--includes/cache/bloom/BloomCacheRedis.php370
-rw-r--r--includes/cache/bloom/BloomFilters.php79
3 files changed, 772 insertions, 0 deletions
diff --git a/includes/cache/bloom/BloomCache.php b/includes/cache/bloom/BloomCache.php
new file mode 100644
index 00000000..236db954
--- /dev/null
+++ b/includes/cache/bloom/BloomCache.php
@@ -0,0 +1,323 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+/**
+ * Persistent bloom filter used to avoid expensive lookups
+ *
+ * @since 1.24
+ */
+abstract class BloomCache {
+ /** @var string Unique ID for key namespacing */
+ protected $cacheID;
+
+ /** @var array Map of (id => BloomCache) */
+ protected static $instances = array();
+
+ /**
+ * @param string $id
+ * @return BloomCache
+ */
+ final public static function get( $id ) {
+ global $wgBloomFilterStores;
+
+ if ( !isset( self::$instances[$id] ) ) {
+ if ( isset( $wgBloomFilterStores[$id] ) ) {
+ $class = $wgBloomFilterStores[$id]['class'];
+ self::$instances[$id] = new $class( $wgBloomFilterStores[$id] );
+ } else {
+ wfDebug( "No bloom filter store '$id'; using EmptyBloomCache." );
+ return new EmptyBloomCache( array() );
+ }
+ }
+
+ return self::$instances[$id];
+ }
+
+ /**
+ * Create a new bloom cache instance from configuration.
+ * This should only be called from within BloomCache.
+ *
+ * @param array $config Parameters include:
+ * - cacheID : Prefix to all bloom filter names that is unique to this cache.
+ * It should only consist of alphanumberic, '-', and '_' characters.
+ * This ID is what avoids collisions if multiple logical caches
+ * use the same storage system, so this should be set carefully.
+ */
+ public function __construct( array $config ) {
+ $this->cacheID = $config['cacheId'];
+ if ( !preg_match( '!^[a-zA-Z0-9-_]{1,32}$!', $this->cacheID ) ) {
+ throw new MWException( "Cache ID '{$this->cacheID}' is invalid." );
+ }
+ }
+
+ /**
+ * Check if a member is set in the bloom filter
+ *
+ * A member being set means that it *might* have been added.
+ * A member not being set means it *could not* have been added.
+ *
+ * This abstracts over isHit() to deal with filter updates and readiness.
+ * A class must exist with the name BloomFilter<type> and a static public
+ * mergeAndCheck() method. The later takes the following arguments:
+ * (BloomCache $bcache, $domain, $virtualKey, array $status)
+ * The method should return a bool indicating whether to use the filter.
+ *
+ * The 'shared' bloom key must be used for any updates and will be used
+ * for the membership check if the method returns true. Since the key is shared,
+ * the method should never use delete(). The filter cannot be used in cases where
+ * membership in the filter needs to be taken away. In such cases, code *cannot*
+ * use this method - instead, it can directly use the other BloomCache methods
+ * to manage custom filters with their own keys (e.g. not 'shared').
+ *
+ * @param string $domain
+ * @param string $type
+ * @param string $member
+ * @return bool True if set, false if not (also returns true on error)
+ */
+ final public function check( $domain, $type, $member ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) {
+ try {
+ $virtualKey = "$domain:$type";
+
+ $status = $this->getStatus( $virtualKey );
+ if ( $status == false ) {
+ wfDebug( "Could not query virtual bloom filter '$virtualKey'." );
+ return null;
+ }
+
+ $useFilter = call_user_func_array(
+ array( "BloomFilter{$type}", 'mergeAndCheck' ),
+ array( $this, $domain, $virtualKey, $status )
+ );
+
+ if ( $useFilter ) {
+ return ( $this->isHit( 'shared', "$virtualKey:$member" ) !== false );
+ }
+ } catch ( MWException $e ) {
+ MWExceptionHandler::logException( $e );
+ return true;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Inform the bloom filter of a new member in order to keep it up to date
+ *
+ * @param string $domain
+ * @param string $type
+ * @param string|array $members
+ * @return bool Success
+ */
+ final public function insert( $domain, $type, $members ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) {
+ try {
+ $virtualKey = "$domain:$type";
+ $prefixedMembers = array();
+ foreach ( (array)$members as $member ) {
+ $prefixedMembers[] = "$virtualKey:$member";
+ }
+
+ return $this->add( 'shared', $prefixedMembers );
+ } catch ( MWException $e ) {
+ MWExceptionHandler::logException( $e );
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Create a new bloom filter at $key (if one does not exist yet)
+ *
+ * @param string $key
+ * @param integer $size Bit length [default: 1000000]
+ * @param float $precision [default: .001]
+ * @return bool Success
+ */
+ final public function init( $key, $size = 1000000, $precision = .001 ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doInit( "{$this->cacheID}:$key", $size, min( .1, $precision ) );
+ }
+
+ /**
+ * Add a member to the bloom filter at $key
+ *
+ * @param string $key
+ * @param string|array $members
+ * @return bool Success
+ */
+ final public function add( $key, $members ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doAdd( "{$this->cacheID}:$key", (array)$members );
+ }
+
+ /**
+ * Check if a member is set in the bloom filter.
+ *
+ * A member being set means that it *might* have been added.
+ * A member not being set means it *could not* have been added.
+ *
+ * If this returns true, then the caller usually should do the
+ * expensive check (whatever that may be). It can be avoided otherwise.
+ *
+ * @param string $key
+ * @param string $member
+ * @return bool|null True if set, false if not, null on error
+ */
+ final public function isHit( $key, $member ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doIsHit( "{$this->cacheID}:$key", $member );
+ }
+
+ /**
+ * Destroy a bloom filter at $key
+ *
+ * @param string $key
+ * @return bool Success
+ */
+ final public function delete( $key ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doDelete( "{$this->cacheID}:$key" );
+ }
+
+ /**
+ * Set the status map of the virtual bloom filter at $key
+ *
+ * @param string $virtualKey
+ * @param array $values Map including some of (lastID, asOfTime, epoch)
+ * @return bool Success
+ */
+ final public function setStatus( $virtualKey, array $values ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doSetStatus( "{$this->cacheID}:$virtualKey", $values );
+ }
+
+ /**
+ * Get the status map of the virtual bloom filter at $key
+ *
+ * The map includes:
+ * - lastID : the highest ID of the items merged in
+ * - asOfTime : UNIX timestamp that the filter is up-to-date as of
+ * - epoch : UNIX timestamp that filter started being populated
+ * Unset fields will have a null value.
+ *
+ * @param string $virtualKey
+ * @return array|bool False on failure
+ */
+ final public function getStatus( $virtualKey ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doGetStatus( "{$this->cacheID}:$virtualKey" );
+ }
+
+ /**
+ * Get an exclusive lock on a filter for updates
+ *
+ * @param string $virtualKey
+ * @return ScopedCallback|ScopedLock|null Returns null if acquisition failed
+ */
+ public function getScopedLock( $virtualKey ) {
+ return null;
+ }
+
+ /**
+ * @param string $key
+ * @param integer $size Bit length
+ * @param float $precision
+ * @return bool Success
+ */
+ abstract protected function doInit( $key, $size, $precision );
+
+ /**
+ * @param string $key
+ * @param array $members
+ * @return bool Success
+ */
+ abstract protected function doAdd( $key, array $members );
+
+ /**
+ * @param string $key
+ * @param string $member
+ * @return bool|null
+ */
+ abstract protected function doIsHit( $key, $member );
+
+ /**
+ * @param string $key
+ * @return bool Success
+ */
+ abstract protected function doDelete( $key );
+
+ /**
+ * @param string $virtualKey
+ * @param array $values
+ * @return bool Success
+ */
+ abstract protected function doSetStatus( $virtualKey, array $values );
+
+ /**
+ * @param string $key
+ * @return array|bool
+ */
+ abstract protected function doGetStatus( $key );
+}
+
+class EmptyBloomCache extends BloomCache {
+ public function __construct( array $config ) {
+ parent::__construct( array( 'cacheId' => 'none' ) );
+ }
+
+ protected function doInit( $key, $size, $precision ) {
+ return true;
+ }
+
+ protected function doAdd( $key, array $members ) {
+ return true;
+ }
+
+ protected function doIsHit( $key, $member ) {
+ return true;
+ }
+
+ protected function doDelete( $key ) {
+ return true;
+ }
+
+ protected function doSetStatus( $virtualKey, array $values ) {
+ return true;
+ }
+
+ protected function doGetStatus( $virtualKey ) {
+ return array( 'lastID' => null, 'asOfTime' => null, 'epoch' => null ) ;
+ }
+}
diff --git a/includes/cache/bloom/BloomCacheRedis.php b/includes/cache/bloom/BloomCacheRedis.php
new file mode 100644
index 00000000..212e5e8b
--- /dev/null
+++ b/includes/cache/bloom/BloomCacheRedis.php
@@ -0,0 +1,370 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+/**
+ * Bloom filter implemented using Redis
+ *
+ * The Redis server must be >= 2.6 and should have volatile-lru or volatile-ttl
+ * if there is any eviction policy. It should not be allkeys-* in any case. Also,
+ * this can be used in a simple master/slave setup or with Redis Sentinel preferably.
+ *
+ * Some bits are based on https://github.com/ErikDubbelboer/redis-lua-scaling-bloom-filter
+ * but are simplified to use a single filter instead of up to 32 filters.
+ *
+ * @since 1.24
+ */
+class BloomCacheRedis extends BloomCache {
+ /** @var RedisConnectionPool */
+ protected $redisPool;
+ /** @var RedisLockManager */
+ protected $lockMgr;
+ /** @var array */
+ protected $servers;
+ /** @var integer Federate each filter into this many redis bitfield objects */
+ protected $segments = 128;
+
+ /**
+ * @params include:
+ * - redisServers : list of servers (address:<port>) (the first is the master)
+ * - redisConf : additional redis configuration
+ *
+ * @param array $config
+ */
+ public function __construct( array $config ) {
+ parent::__construct( $config );
+
+ $redisConf = $config['redisConfig'];
+ $redisConf['serializer'] = 'none'; // manage that in this class
+ $this->redisPool = RedisConnectionPool::singleton( $redisConf );
+ $this->servers = $config['redisServers'];
+ $this->lockMgr = new RedisLockManager( array(
+ 'lockServers' => array( 'srv1' => $this->servers[0] ),
+ 'srvsByBucket' => array( 0 => array( 'srv1' ) ),
+ 'redisConfig' => $config['redisConfig']
+ ) );
+ }
+
+ protected function doInit( $key, $size, $precision ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ // 80000000 items at p = .001 take up 500MB and fit into one value.
+ // Do not hit the 512MB redis value limit by reducing the demands.
+ $size = min( $size, 80000000 * $this->segments );
+ $precision = max( round( $precision, 3 ), .001 );
+ $epoch = microtime( true );
+
+ static $script =
+<<<LUA
+ local kMetadata, kData = unpack(KEYS)
+ local aEntries, aPrec, aEpoch = unpack(ARGV)
+ if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then
+ redis.call('DEL',kMetadata)
+ redis.call('HSET',kMetadata,'entries',aEntries)
+ redis.call('HSET',kMetadata,'precision',aPrec)
+ redis.call('HSET',kMetadata,'epoch',aEpoch)
+ redis.call('SET',kData,'')
+ return 1
+ end
+ return 0
+LUA;
+
+ $res = false;
+ try {
+ $conn->script( 'load', $script );
+ $conn->multi( Redis::MULTI );
+ for ( $i = 0; $i < $this->segments; ++$i ) {
+ $res = $conn->luaEval( $script,
+ array(
+ "$key:$i:bloom-metadata", # KEYS[1]
+ "$key:$i:bloom-data", # KEYS[2]
+ ceil( $size / $this->segments ), # ARGV[1]
+ $precision, # ARGV[2]
+ $epoch # ARGV[3]
+ ),
+ 2 # number of first argument(s) that are keys
+ );
+ }
+ $results = $conn->exec();
+ $res = $results && !in_array( false, $results, true );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return ( $res !== false );
+ }
+
+ protected function doAdd( $key, array $members ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ static $script =
+<<<LUA
+ local kMetadata, kData = unpack(KEYS)
+ local aMember = unpack(ARGV)
+
+ -- Check if the filter was initialized
+ if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then
+ return false
+ end
+
+ -- Initial expected entries and desired precision
+ local entries = 1*redis.call('HGET',kMetadata,'entries')
+ local precision = 1*redis.call('HGET',kMetadata,'precision')
+ local hash = redis.sha1hex(aMember)
+
+ -- Based on the math from: http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives
+ -- 0.480453013 = ln(2)^2
+ local bits = math.ceil((entries * math.log(precision)) / -0.480453013)
+
+ -- 0.693147180 = ln(2)
+ local k = math.floor(0.693147180 * bits / entries)
+
+ -- This uses a variation on:
+ -- 'Less Hashing, Same Performance: Building a Better Bloom Filter'
+ -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf
+ local h = { }
+ h[0] = tonumber(string.sub(hash, 1, 8 ), 16)
+ h[1] = tonumber(string.sub(hash, 9, 16), 16)
+ h[2] = tonumber(string.sub(hash, 17, 24), 16)
+ h[3] = tonumber(string.sub(hash, 25, 32), 16)
+
+ for i=1, k do
+ local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits
+ redis.call('SETBIT', kData, pos, 1)
+ end
+
+ return 1
+LUA;
+
+ $res = false;
+ try {
+ $conn->script( 'load', $script );
+ $conn->multi( Redis::PIPELINE );
+ foreach ( $members as $member ) {
+ $i = $this->getSegment( $member );
+ $conn->luaEval( $script,
+ array(
+ "$key:$i:bloom-metadata", # KEYS[1],
+ "$key:$i:bloom-data", # KEYS[2]
+ $member # ARGV[1]
+ ),
+ 2 # number of first argument(s) that are keys
+ );
+ }
+ $results = $conn->exec();
+ $res = $results && !in_array( false, $results, true );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ if ( $res === false ) {
+ wfDebug( "Could not add to the '$key' bloom filter; it may be missing." );
+ }
+
+ return ( $res !== false );
+ }
+
+ protected function doSetStatus( $virtualKey, array $values ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return null;
+ }
+
+ $res = false;
+ try {
+ $res = $conn->hMSet( "$virtualKey:filter-metadata", $values );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return ( $res !== false );
+ }
+
+ protected function doGetStatus( $virtualKey ) {
+ $conn = $this->getConnection( 'slave' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ $res = false;
+ try {
+ $res = $conn->hGetAll( "$virtualKey:filter-metadata" );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ if ( is_array( $res ) ) {
+ $res['lastID'] = isset( $res['lastID'] ) ? $res['lastID'] : null;
+ $res['asOfTime'] = isset( $res['asOfTime'] ) ? $res['asOfTime'] : null;
+ $res['epoch'] = isset( $res['epoch'] ) ? $res['epoch'] : null;
+ }
+
+ return $res;
+ }
+
+ protected function doIsHit( $key, $member ) {
+ $conn = $this->getConnection( 'slave' );
+ if ( !$conn ) {
+ return null;
+ }
+
+ static $script =
+<<<LUA
+ local kMetadata, kData = unpack(KEYS)
+ local aMember = unpack(ARGV)
+
+ -- Check if the filter was initialized
+ if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then
+ return false
+ end
+
+ -- Initial expected entries and desired precision.
+ -- This determines the size of the first and subsequent filters.
+ local entries = redis.call('HGET',kMetadata,'entries')
+ local precision = redis.call('HGET',kMetadata,'precision')
+ local hash = redis.sha1hex(aMember)
+
+ -- This uses a variation on:
+ -- 'Less Hashing, Same Performance: Building a Better Bloom Filter'
+ -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf
+ local h = { }
+ h[0] = tonumber(string.sub(hash, 1, 8 ), 16)
+ h[1] = tonumber(string.sub(hash, 9, 16), 16)
+ h[2] = tonumber(string.sub(hash, 17, 24), 16)
+ h[3] = tonumber(string.sub(hash, 25, 32), 16)
+
+ -- 0.480453013 = ln(2)^2
+ local bits = math.ceil((entries * math.log(precision)) / -0.480453013)
+
+ -- 0.693147180 = ln(2)
+ local k = math.floor(0.693147180 * bits / entries)
+
+ local found = 1
+ for i=1, k do
+ local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits
+ if redis.call('GETBIT', kData, pos) == 0 then
+ found = 0
+ break
+ end
+ end
+
+ return found
+LUA;
+
+ $res = null;
+ try {
+ $i = $this->getSegment( $member );
+ $res = $conn->luaEval( $script,
+ array(
+ "$key:$i:bloom-metadata", # KEYS[1],
+ "$key:$i:bloom-data", # KEYS[2]
+ $member # ARGV[1]
+ ),
+ 2 # number of first argument(s) that are keys
+ );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return is_int( $res ) ? (bool)$res : null;
+ }
+
+ protected function doDelete( $key ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ $res = false;
+ try {
+ $keys = array();
+ for ( $i = 0; $i < $this->segments; ++$i ) {
+ $keys[] = "$key:$i:bloom-metadata";
+ $keys[] = "$key:$i:bloom-data";
+ }
+ $res = $conn->delete( $keys );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return ( $res !== false );
+ }
+
+ public function getScopedLock( $virtualKey ) {
+ $status = Status::newGood();
+ return ScopedLock::factory( $this->lockMgr,
+ array( $virtualKey ), LockManager::LOCK_EX, $status );
+ }
+
+ /**
+ * @param string $member
+ * @return integer
+ */
+ protected function getSegment( $member ) {
+ return hexdec( substr( md5( $member ), 0, 2 ) ) % $this->segments;
+ }
+
+ /**
+ * $param string $to (master/slave)
+ * @return RedisConnRef|bool Returns false on failure
+ */
+ protected function getConnection( $to ) {
+ if ( $to === 'master' ) {
+ $conn = $this->redisPool->getConnection( $this->servers[0] );
+ } else {
+ static $lastServer = null;
+
+ $conn = false;
+ if ( $lastServer ) {
+ $conn = $this->redisPool->getConnection( $lastServer );
+ if ( $conn ) {
+ return $conn; // reuse connection
+ }
+ }
+ $servers = $this->servers;
+ $attempts = min( 3, count( $servers ) );
+ for ( $i = 1; $i <= $attempts; ++$i ) {
+ $index = mt_rand( 0, count( $servers ) - 1 );
+ $conn = $this->redisPool->getConnection( $servers[$index] );
+ if ( $conn ) {
+ $lastServer = $servers[$index];
+ return $conn;
+ }
+ unset( $servers[$index] ); // skip next time
+ }
+ }
+
+ return $conn;
+ }
+
+ /**
+ * @param RedisConnRef $conn
+ * @param Exception $e
+ */
+ protected function handleException( RedisConnRef $conn, $e ) {
+ $this->redisPool->handleError( $conn, $e );
+ }
+}
diff --git a/includes/cache/bloom/BloomFilters.php b/includes/cache/bloom/BloomFilters.php
new file mode 100644
index 00000000..9b710d79
--- /dev/null
+++ b/includes/cache/bloom/BloomFilters.php
@@ -0,0 +1,79 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+/**
+ * @since 1.24
+ */
+class BloomFilterTitleHasLogs {
+ public static function mergeAndCheck(
+ BloomCache $bcache, $domain, $virtualKey, array $status
+ ) {
+ $age = microtime( true ) - $status['asOfTime']; // seconds
+ $scopedLock = ( mt_rand( 1, (int)pow( 3, max( 0, 5 - $age ) ) ) == 1 )
+ ? $bcache->getScopedLock( $virtualKey )
+ : false;
+
+ if ( $scopedLock ) {
+ $updates = self::merge( $bcache, $domain, $virtualKey, $status );
+ if ( isset( $updates['asOfTime'] ) ) {
+ $age = ( microtime( true ) - $updates['asOfTime'] );
+ }
+ }
+
+ return ( $age < 30 );
+ }
+
+ public static function merge(
+ BloomCache $bcache, $domain, $virtualKey, array $status
+ ) {
+ $limit = 1000;
+ $dbr = wfGetDB( DB_SLAVE, array(), $domain );
+ $res = $dbr->select( 'logging',
+ array( 'log_namespace', 'log_title', 'log_id', 'log_timestamp' ),
+ array( 'log_id > ' . $dbr->addQuotes( (int)$status['lastID'] ) ),
+ __METHOD__,
+ array( 'ORDER BY' => 'log_id', 'LIMIT' => $limit )
+ );
+
+ $updates = array();
+ if ( $res->numRows() > 0 ) {
+ $members = array();
+ foreach ( $res as $row ) {
+ $members[] = "$virtualKey:{$row->log_namespace}:{$row->log_title}";
+ }
+ $lastID = $row->log_id;
+ $lastTime = $row->log_timestamp;
+ if ( !$bcache->add( 'shared', $members ) ) {
+ return false;
+ }
+ $updates['lastID'] = $lastID;
+ $updates['asOfTime'] = wfTimestamp( TS_UNIX, $lastTime );
+ } else {
+ $updates['asOfTime'] = microtime( true );
+ }
+
+ $updates['epoch'] = $status['epoch'] ?: microtime( true );
+
+ $bcache->setStatus( $virtualKey, $updates );
+
+ return $updates;
+ }
+}