diff options
author | Luke Shumaker <lukeshu@sbcglobal.net> | 2016-05-01 15:12:12 -0400 |
---|---|---|
committer | Luke Shumaker <lukeshu@sbcglobal.net> | 2016-05-01 15:12:12 -0400 |
commit | c9aa36da061816dee256a979c2ff8d2ee41824d9 (patch) | |
tree | 29f7002b80ee984b488bd047dbbd80b36bf892e9 /includes/cache/bloom | |
parent | b4274e0e33eafb5e9ead9d949ebf031a9fb8363b (diff) | |
parent | d1ba966140d7a60cd5ae4e8667ceb27c1a138592 (diff) |
Merge branch 'archwiki'
# Conflicts:
# skins/ArchLinux.php
# skins/ArchLinux/archlogo.gif
Diffstat (limited to 'includes/cache/bloom')
-rw-r--r-- | includes/cache/bloom/BloomCache.php | 323 | ||||
-rw-r--r-- | includes/cache/bloom/BloomCacheRedis.php | 370 | ||||
-rw-r--r-- | includes/cache/bloom/BloomFilters.php | 79 |
3 files changed, 772 insertions, 0 deletions
diff --git a/includes/cache/bloom/BloomCache.php b/includes/cache/bloom/BloomCache.php new file mode 100644 index 00000000..236db954 --- /dev/null +++ b/includes/cache/bloom/BloomCache.php @@ -0,0 +1,323 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * Persistent bloom filter used to avoid expensive lookups + * + * @since 1.24 + */ +abstract class BloomCache { + /** @var string Unique ID for key namespacing */ + protected $cacheID; + + /** @var array Map of (id => BloomCache) */ + protected static $instances = array(); + + /** + * @param string $id + * @return BloomCache + */ + final public static function get( $id ) { + global $wgBloomFilterStores; + + if ( !isset( self::$instances[$id] ) ) { + if ( isset( $wgBloomFilterStores[$id] ) ) { + $class = $wgBloomFilterStores[$id]['class']; + self::$instances[$id] = new $class( $wgBloomFilterStores[$id] ); + } else { + wfDebug( "No bloom filter store '$id'; using EmptyBloomCache." ); + return new EmptyBloomCache( array() ); + } + } + + return self::$instances[$id]; + } + + /** + * Create a new bloom cache instance from configuration. + * This should only be called from within BloomCache. + * + * @param array $config Parameters include: + * - cacheID : Prefix to all bloom filter names that is unique to this cache. + * It should only consist of alphanumberic, '-', and '_' characters. + * This ID is what avoids collisions if multiple logical caches + * use the same storage system, so this should be set carefully. + */ + public function __construct( array $config ) { + $this->cacheID = $config['cacheId']; + if ( !preg_match( '!^[a-zA-Z0-9-_]{1,32}$!', $this->cacheID ) ) { + throw new MWException( "Cache ID '{$this->cacheID}' is invalid." ); + } + } + + /** + * Check if a member is set in the bloom filter + * + * A member being set means that it *might* have been added. + * A member not being set means it *could not* have been added. + * + * This abstracts over isHit() to deal with filter updates and readiness. + * A class must exist with the name BloomFilter<type> and a static public + * mergeAndCheck() method. The later takes the following arguments: + * (BloomCache $bcache, $domain, $virtualKey, array $status) + * The method should return a bool indicating whether to use the filter. + * + * The 'shared' bloom key must be used for any updates and will be used + * for the membership check if the method returns true. Since the key is shared, + * the method should never use delete(). The filter cannot be used in cases where + * membership in the filter needs to be taken away. In such cases, code *cannot* + * use this method - instead, it can directly use the other BloomCache methods + * to manage custom filters with their own keys (e.g. not 'shared'). + * + * @param string $domain + * @param string $type + * @param string $member + * @return bool True if set, false if not (also returns true on error) + */ + final public function check( $domain, $type, $member ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) { + try { + $virtualKey = "$domain:$type"; + + $status = $this->getStatus( $virtualKey ); + if ( $status == false ) { + wfDebug( "Could not query virtual bloom filter '$virtualKey'." ); + return null; + } + + $useFilter = call_user_func_array( + array( "BloomFilter{$type}", 'mergeAndCheck' ), + array( $this, $domain, $virtualKey, $status ) + ); + + if ( $useFilter ) { + return ( $this->isHit( 'shared', "$virtualKey:$member" ) !== false ); + } + } catch ( MWException $e ) { + MWExceptionHandler::logException( $e ); + return true; + } + } + + return true; + } + + /** + * Inform the bloom filter of a new member in order to keep it up to date + * + * @param string $domain + * @param string $type + * @param string|array $members + * @return bool Success + */ + final public function insert( $domain, $type, $members ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) { + try { + $virtualKey = "$domain:$type"; + $prefixedMembers = array(); + foreach ( (array)$members as $member ) { + $prefixedMembers[] = "$virtualKey:$member"; + } + + return $this->add( 'shared', $prefixedMembers ); + } catch ( MWException $e ) { + MWExceptionHandler::logException( $e ); + return false; + } + } + + return true; + } + + /** + * Create a new bloom filter at $key (if one does not exist yet) + * + * @param string $key + * @param integer $size Bit length [default: 1000000] + * @param float $precision [default: .001] + * @return bool Success + */ + final public function init( $key, $size = 1000000, $precision = .001 ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doInit( "{$this->cacheID}:$key", $size, min( .1, $precision ) ); + } + + /** + * Add a member to the bloom filter at $key + * + * @param string $key + * @param string|array $members + * @return bool Success + */ + final public function add( $key, $members ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doAdd( "{$this->cacheID}:$key", (array)$members ); + } + + /** + * Check if a member is set in the bloom filter. + * + * A member being set means that it *might* have been added. + * A member not being set means it *could not* have been added. + * + * If this returns true, then the caller usually should do the + * expensive check (whatever that may be). It can be avoided otherwise. + * + * @param string $key + * @param string $member + * @return bool|null True if set, false if not, null on error + */ + final public function isHit( $key, $member ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doIsHit( "{$this->cacheID}:$key", $member ); + } + + /** + * Destroy a bloom filter at $key + * + * @param string $key + * @return bool Success + */ + final public function delete( $key ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doDelete( "{$this->cacheID}:$key" ); + } + + /** + * Set the status map of the virtual bloom filter at $key + * + * @param string $virtualKey + * @param array $values Map including some of (lastID, asOfTime, epoch) + * @return bool Success + */ + final public function setStatus( $virtualKey, array $values ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doSetStatus( "{$this->cacheID}:$virtualKey", $values ); + } + + /** + * Get the status map of the virtual bloom filter at $key + * + * The map includes: + * - lastID : the highest ID of the items merged in + * - asOfTime : UNIX timestamp that the filter is up-to-date as of + * - epoch : UNIX timestamp that filter started being populated + * Unset fields will have a null value. + * + * @param string $virtualKey + * @return array|bool False on failure + */ + final public function getStatus( $virtualKey ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doGetStatus( "{$this->cacheID}:$virtualKey" ); + } + + /** + * Get an exclusive lock on a filter for updates + * + * @param string $virtualKey + * @return ScopedCallback|ScopedLock|null Returns null if acquisition failed + */ + public function getScopedLock( $virtualKey ) { + return null; + } + + /** + * @param string $key + * @param integer $size Bit length + * @param float $precision + * @return bool Success + */ + abstract protected function doInit( $key, $size, $precision ); + + /** + * @param string $key + * @param array $members + * @return bool Success + */ + abstract protected function doAdd( $key, array $members ); + + /** + * @param string $key + * @param string $member + * @return bool|null + */ + abstract protected function doIsHit( $key, $member ); + + /** + * @param string $key + * @return bool Success + */ + abstract protected function doDelete( $key ); + + /** + * @param string $virtualKey + * @param array $values + * @return bool Success + */ + abstract protected function doSetStatus( $virtualKey, array $values ); + + /** + * @param string $key + * @return array|bool + */ + abstract protected function doGetStatus( $key ); +} + +class EmptyBloomCache extends BloomCache { + public function __construct( array $config ) { + parent::__construct( array( 'cacheId' => 'none' ) ); + } + + protected function doInit( $key, $size, $precision ) { + return true; + } + + protected function doAdd( $key, array $members ) { + return true; + } + + protected function doIsHit( $key, $member ) { + return true; + } + + protected function doDelete( $key ) { + return true; + } + + protected function doSetStatus( $virtualKey, array $values ) { + return true; + } + + protected function doGetStatus( $virtualKey ) { + return array( 'lastID' => null, 'asOfTime' => null, 'epoch' => null ) ; + } +} diff --git a/includes/cache/bloom/BloomCacheRedis.php b/includes/cache/bloom/BloomCacheRedis.php new file mode 100644 index 00000000..212e5e8b --- /dev/null +++ b/includes/cache/bloom/BloomCacheRedis.php @@ -0,0 +1,370 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * Bloom filter implemented using Redis + * + * The Redis server must be >= 2.6 and should have volatile-lru or volatile-ttl + * if there is any eviction policy. It should not be allkeys-* in any case. Also, + * this can be used in a simple master/slave setup or with Redis Sentinel preferably. + * + * Some bits are based on https://github.com/ErikDubbelboer/redis-lua-scaling-bloom-filter + * but are simplified to use a single filter instead of up to 32 filters. + * + * @since 1.24 + */ +class BloomCacheRedis extends BloomCache { + /** @var RedisConnectionPool */ + protected $redisPool; + /** @var RedisLockManager */ + protected $lockMgr; + /** @var array */ + protected $servers; + /** @var integer Federate each filter into this many redis bitfield objects */ + protected $segments = 128; + + /** + * @params include: + * - redisServers : list of servers (address:<port>) (the first is the master) + * - redisConf : additional redis configuration + * + * @param array $config + */ + public function __construct( array $config ) { + parent::__construct( $config ); + + $redisConf = $config['redisConfig']; + $redisConf['serializer'] = 'none'; // manage that in this class + $this->redisPool = RedisConnectionPool::singleton( $redisConf ); + $this->servers = $config['redisServers']; + $this->lockMgr = new RedisLockManager( array( + 'lockServers' => array( 'srv1' => $this->servers[0] ), + 'srvsByBucket' => array( 0 => array( 'srv1' ) ), + 'redisConfig' => $config['redisConfig'] + ) ); + } + + protected function doInit( $key, $size, $precision ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return false; + } + + // 80000000 items at p = .001 take up 500MB and fit into one value. + // Do not hit the 512MB redis value limit by reducing the demands. + $size = min( $size, 80000000 * $this->segments ); + $precision = max( round( $precision, 3 ), .001 ); + $epoch = microtime( true ); + + static $script = +<<<LUA + local kMetadata, kData = unpack(KEYS) + local aEntries, aPrec, aEpoch = unpack(ARGV) + if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then + redis.call('DEL',kMetadata) + redis.call('HSET',kMetadata,'entries',aEntries) + redis.call('HSET',kMetadata,'precision',aPrec) + redis.call('HSET',kMetadata,'epoch',aEpoch) + redis.call('SET',kData,'') + return 1 + end + return 0 +LUA; + + $res = false; + try { + $conn->script( 'load', $script ); + $conn->multi( Redis::MULTI ); + for ( $i = 0; $i < $this->segments; ++$i ) { + $res = $conn->luaEval( $script, + array( + "$key:$i:bloom-metadata", # KEYS[1] + "$key:$i:bloom-data", # KEYS[2] + ceil( $size / $this->segments ), # ARGV[1] + $precision, # ARGV[2] + $epoch # ARGV[3] + ), + 2 # number of first argument(s) that are keys + ); + } + $results = $conn->exec(); + $res = $results && !in_array( false, $results, true ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return ( $res !== false ); + } + + protected function doAdd( $key, array $members ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return false; + } + + static $script = +<<<LUA + local kMetadata, kData = unpack(KEYS) + local aMember = unpack(ARGV) + + -- Check if the filter was initialized + if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then + return false + end + + -- Initial expected entries and desired precision + local entries = 1*redis.call('HGET',kMetadata,'entries') + local precision = 1*redis.call('HGET',kMetadata,'precision') + local hash = redis.sha1hex(aMember) + + -- Based on the math from: http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives + -- 0.480453013 = ln(2)^2 + local bits = math.ceil((entries * math.log(precision)) / -0.480453013) + + -- 0.693147180 = ln(2) + local k = math.floor(0.693147180 * bits / entries) + + -- This uses a variation on: + -- 'Less Hashing, Same Performance: Building a Better Bloom Filter' + -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf + local h = { } + h[0] = tonumber(string.sub(hash, 1, 8 ), 16) + h[1] = tonumber(string.sub(hash, 9, 16), 16) + h[2] = tonumber(string.sub(hash, 17, 24), 16) + h[3] = tonumber(string.sub(hash, 25, 32), 16) + + for i=1, k do + local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits + redis.call('SETBIT', kData, pos, 1) + end + + return 1 +LUA; + + $res = false; + try { + $conn->script( 'load', $script ); + $conn->multi( Redis::PIPELINE ); + foreach ( $members as $member ) { + $i = $this->getSegment( $member ); + $conn->luaEval( $script, + array( + "$key:$i:bloom-metadata", # KEYS[1], + "$key:$i:bloom-data", # KEYS[2] + $member # ARGV[1] + ), + 2 # number of first argument(s) that are keys + ); + } + $results = $conn->exec(); + $res = $results && !in_array( false, $results, true ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + if ( $res === false ) { + wfDebug( "Could not add to the '$key' bloom filter; it may be missing." ); + } + + return ( $res !== false ); + } + + protected function doSetStatus( $virtualKey, array $values ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return null; + } + + $res = false; + try { + $res = $conn->hMSet( "$virtualKey:filter-metadata", $values ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return ( $res !== false ); + } + + protected function doGetStatus( $virtualKey ) { + $conn = $this->getConnection( 'slave' ); + if ( !$conn ) { + return false; + } + + $res = false; + try { + $res = $conn->hGetAll( "$virtualKey:filter-metadata" ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + if ( is_array( $res ) ) { + $res['lastID'] = isset( $res['lastID'] ) ? $res['lastID'] : null; + $res['asOfTime'] = isset( $res['asOfTime'] ) ? $res['asOfTime'] : null; + $res['epoch'] = isset( $res['epoch'] ) ? $res['epoch'] : null; + } + + return $res; + } + + protected function doIsHit( $key, $member ) { + $conn = $this->getConnection( 'slave' ); + if ( !$conn ) { + return null; + } + + static $script = +<<<LUA + local kMetadata, kData = unpack(KEYS) + local aMember = unpack(ARGV) + + -- Check if the filter was initialized + if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then + return false + end + + -- Initial expected entries and desired precision. + -- This determines the size of the first and subsequent filters. + local entries = redis.call('HGET',kMetadata,'entries') + local precision = redis.call('HGET',kMetadata,'precision') + local hash = redis.sha1hex(aMember) + + -- This uses a variation on: + -- 'Less Hashing, Same Performance: Building a Better Bloom Filter' + -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf + local h = { } + h[0] = tonumber(string.sub(hash, 1, 8 ), 16) + h[1] = tonumber(string.sub(hash, 9, 16), 16) + h[2] = tonumber(string.sub(hash, 17, 24), 16) + h[3] = tonumber(string.sub(hash, 25, 32), 16) + + -- 0.480453013 = ln(2)^2 + local bits = math.ceil((entries * math.log(precision)) / -0.480453013) + + -- 0.693147180 = ln(2) + local k = math.floor(0.693147180 * bits / entries) + + local found = 1 + for i=1, k do + local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits + if redis.call('GETBIT', kData, pos) == 0 then + found = 0 + break + end + end + + return found +LUA; + + $res = null; + try { + $i = $this->getSegment( $member ); + $res = $conn->luaEval( $script, + array( + "$key:$i:bloom-metadata", # KEYS[1], + "$key:$i:bloom-data", # KEYS[2] + $member # ARGV[1] + ), + 2 # number of first argument(s) that are keys + ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return is_int( $res ) ? (bool)$res : null; + } + + protected function doDelete( $key ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return false; + } + + $res = false; + try { + $keys = array(); + for ( $i = 0; $i < $this->segments; ++$i ) { + $keys[] = "$key:$i:bloom-metadata"; + $keys[] = "$key:$i:bloom-data"; + } + $res = $conn->delete( $keys ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return ( $res !== false ); + } + + public function getScopedLock( $virtualKey ) { + $status = Status::newGood(); + return ScopedLock::factory( $this->lockMgr, + array( $virtualKey ), LockManager::LOCK_EX, $status ); + } + + /** + * @param string $member + * @return integer + */ + protected function getSegment( $member ) { + return hexdec( substr( md5( $member ), 0, 2 ) ) % $this->segments; + } + + /** + * $param string $to (master/slave) + * @return RedisConnRef|bool Returns false on failure + */ + protected function getConnection( $to ) { + if ( $to === 'master' ) { + $conn = $this->redisPool->getConnection( $this->servers[0] ); + } else { + static $lastServer = null; + + $conn = false; + if ( $lastServer ) { + $conn = $this->redisPool->getConnection( $lastServer ); + if ( $conn ) { + return $conn; // reuse connection + } + } + $servers = $this->servers; + $attempts = min( 3, count( $servers ) ); + for ( $i = 1; $i <= $attempts; ++$i ) { + $index = mt_rand( 0, count( $servers ) - 1 ); + $conn = $this->redisPool->getConnection( $servers[$index] ); + if ( $conn ) { + $lastServer = $servers[$index]; + return $conn; + } + unset( $servers[$index] ); // skip next time + } + } + + return $conn; + } + + /** + * @param RedisConnRef $conn + * @param Exception $e + */ + protected function handleException( RedisConnRef $conn, $e ) { + $this->redisPool->handleError( $conn, $e ); + } +} diff --git a/includes/cache/bloom/BloomFilters.php b/includes/cache/bloom/BloomFilters.php new file mode 100644 index 00000000..9b710d79 --- /dev/null +++ b/includes/cache/bloom/BloomFilters.php @@ -0,0 +1,79 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * @since 1.24 + */ +class BloomFilterTitleHasLogs { + public static function mergeAndCheck( + BloomCache $bcache, $domain, $virtualKey, array $status + ) { + $age = microtime( true ) - $status['asOfTime']; // seconds + $scopedLock = ( mt_rand( 1, (int)pow( 3, max( 0, 5 - $age ) ) ) == 1 ) + ? $bcache->getScopedLock( $virtualKey ) + : false; + + if ( $scopedLock ) { + $updates = self::merge( $bcache, $domain, $virtualKey, $status ); + if ( isset( $updates['asOfTime'] ) ) { + $age = ( microtime( true ) - $updates['asOfTime'] ); + } + } + + return ( $age < 30 ); + } + + public static function merge( + BloomCache $bcache, $domain, $virtualKey, array $status + ) { + $limit = 1000; + $dbr = wfGetDB( DB_SLAVE, array(), $domain ); + $res = $dbr->select( 'logging', + array( 'log_namespace', 'log_title', 'log_id', 'log_timestamp' ), + array( 'log_id > ' . $dbr->addQuotes( (int)$status['lastID'] ) ), + __METHOD__, + array( 'ORDER BY' => 'log_id', 'LIMIT' => $limit ) + ); + + $updates = array(); + if ( $res->numRows() > 0 ) { + $members = array(); + foreach ( $res as $row ) { + $members[] = "$virtualKey:{$row->log_namespace}:{$row->log_title}"; + } + $lastID = $row->log_id; + $lastTime = $row->log_timestamp; + if ( !$bcache->add( 'shared', $members ) ) { + return false; + } + $updates['lastID'] = $lastID; + $updates['asOfTime'] = wfTimestamp( TS_UNIX, $lastTime ); + } else { + $updates['asOfTime'] = microtime( true ); + } + + $updates['epoch'] = $status['epoch'] ?: microtime( true ); + + $bcache->setStatus( $virtualKey, $updates ); + + return $updates; + } +} |