diff options
Diffstat (limited to 'includes/cache')
| -rw-r--r-- | includes/cache/BacklinkCache.php | 125 | ||||
| -rw-r--r-- | includes/cache/CacheDependency.php | 230 | ||||
| -rw-r--r-- | includes/cache/CacheHelper.php | 386 | ||||
| -rw-r--r-- | includes/cache/FileCacheBase.php | 15 | ||||
| -rw-r--r-- | includes/cache/GenderCache.php | 28 | ||||
| -rw-r--r-- | includes/cache/HTMLCacheUpdate.php | 73 | ||||
| -rw-r--r-- | includes/cache/HTMLFileCache.php | 47 | ||||
| -rw-r--r-- | includes/cache/LinkBatch.php | 47 | ||||
| -rw-r--r-- | includes/cache/LinkCache.php | 75 | ||||
| -rw-r--r-- | includes/cache/LocalisationCache.php | 571 | ||||
| -rw-r--r-- | includes/cache/MapCacheLRU.php (renamed from includes/cache/ProcessCacheLRU.php) | 64 | ||||
| -rw-r--r-- | includes/cache/MessageCache.php | 83 | ||||
| -rw-r--r-- | includes/cache/ObjectFileCache.php | 4 | ||||
| -rw-r--r-- | includes/cache/ResourceFileCache.php | 7 | ||||
| -rw-r--r-- | includes/cache/SquidUpdate.php | 300 | ||||
| -rw-r--r-- | includes/cache/UserCache.php | 16 | ||||
| -rw-r--r-- | includes/cache/bloom/BloomCache.php | 323 | ||||
| -rw-r--r-- | includes/cache/bloom/BloomCacheRedis.php | 370 | ||||
| -rw-r--r-- | includes/cache/bloom/BloomFilters.php | 79 |
19 files changed, 1819 insertions, 1024 deletions
diff --git a/includes/cache/BacklinkCache.php b/includes/cache/BacklinkCache.php index 193f20fe..ed62bba0 100644 --- a/includes/cache/BacklinkCache.php +++ b/includes/cache/BacklinkCache.php @@ -48,6 +48,7 @@ class BacklinkCache { /** * Multi dimensions array representing batches. Keys are: * > (string) links table name + * > (int) batch size * > 'numRows' : Number of rows for this link table * > 'batches' : array( $start, $end ) * @@ -96,7 +97,7 @@ class BacklinkCache { * Currently, only one cache instance can exist; callers that * need multiple backlink cache objects should keep them in scope. * - * @param Title $title : Title object to get a backlink cache for + * @param Title $title Title object to get a backlink cache for * @return BacklinkCache */ public static function get( Title $title ) { @@ -107,6 +108,7 @@ class BacklinkCache { if ( !self::$cache->has( $dbKey, 'obj', 3600 ) ) { self::$cache->set( $dbKey, 'obj', new self( $title ) ); } + return self::$cache->get( $dbKey, 'obj' ); } @@ -133,7 +135,7 @@ class BacklinkCache { /** * Set the Database object to use * - * @param $db DatabaseBase + * @param DatabaseBase $db */ public function setDB( $db ) { $this->db = $db; @@ -142,21 +144,22 @@ class BacklinkCache { /** * Get the slave connection to the database * When non existing, will initialize the connection. - * @return DatabaseBase object + * @return DatabaseBase */ protected function getDB() { if ( !isset( $this->db ) ) { $this->db = wfGetDB( DB_SLAVE ); } + return $this->db; } /** * Get the backlinks for a given table. Cached in process memory only. - * @param $table String - * @param $startId Integer|false - * @param $endId Integer|false - * @param $max Integer|INF + * @param string $table + * @param int|bool $startId + * @param int|bool $endId + * @param int|INF $max * @return TitleArrayFromResult */ public function getLinks( $table, $startId = false, $endId = false, $max = INF ) { @@ -165,20 +168,21 @@ class BacklinkCache { /** * Get the backlinks for a given table. Cached in process memory only. - * @param $table String - * @param $startId Integer|false - * @param $endId Integer|false - * @param $max Integer|INF + * @param string $table + * @param int|bool $startId + * @param int|bool $endId + * @param int|INF $max + * @param string $select 'all' or 'ids' * @return ResultWrapper */ - protected function queryLinks( $table, $startId, $endId, $max ) { + protected function queryLinks( $table, $startId, $endId, $max, $select = 'all' ) { wfProfileIn( __METHOD__ ); $fromField = $this->getPrefix( $table ) . '_from'; if ( !$startId && !$endId && is_infinite( $max ) - && isset( $this->fullResultCache[$table] ) ) - { + && isset( $this->fullResultCache[$table] ) + ) { wfDebug( __METHOD__ . ": got results from cache\n" ); $res = $this->fullResultCache[$table]; } else { @@ -192,20 +196,34 @@ class BacklinkCache { if ( $endId ) { $conds[] = "$fromField <= " . intval( $endId ); } - $options = array( 'STRAIGHT_JOIN', 'ORDER BY' => $fromField ); + $options = array( 'ORDER BY' => $fromField ); if ( is_finite( $max ) && $max > 0 ) { $options['LIMIT'] = $max; } - $res = $this->getDB()->select( - array( $table, 'page' ), - array( 'page_namespace', 'page_title', 'page_id' ), - $conds, - __METHOD__, - $options - ); + if ( $select === 'ids' ) { + // Just select from the backlink table and ignore the page JOIN + $res = $this->getDB()->select( + $table, + array( $this->getPrefix( $table ) . '_from AS page_id' ), + array_filter( $conds, function ( $clause ) { // kind of janky + return !preg_match( '/(\b|=)page_id(\b|=)/', $clause ); + } ), + __METHOD__, + $options + ); + } else { + // Select from the backlink table and JOIN with page title information + $res = $this->getDB()->select( + array( $table, 'page' ), + array( 'page_namespace', 'page_title', 'page_id' ), + $conds, + __METHOD__, + array_merge( array( 'STRAIGHT_JOIN' ), $options ) + ); + } - if ( !$startId && !$endId && $res->numRows() < $max ) { + if ( $select === 'all' && !$startId && !$endId && $res->numRows() < $max ) { // The full results fit within the limit, so cache them $this->fullResultCache[$table] = $res; } else { @@ -214,12 +232,13 @@ class BacklinkCache { } wfProfileOut( __METHOD__ ); + return $res; } /** * Get the field name prefix for a given table - * @param $table String + * @param string $table * @throws MWException * @return null|string */ @@ -248,15 +267,13 @@ class BacklinkCache { /** * Get the SQL condition array for selecting backlinks, with a join * on the page table. - * @param $table String + * @param string $table * @throws MWException * @return array|null */ protected function getConditions( $table ) { $prefix = $this->getPrefix( $table ); - // @todo FIXME: imagelinks and categorylinks do not rely on getNamespace, - // they could be moved up for nicer case statements switch ( $table ) { case 'pagelinks': case 'templatelinks': @@ -278,15 +295,10 @@ class BacklinkCache { ); break; case 'imagelinks': - $conds = array( - 'il_to' => $this->title->getDBkey(), - 'page_id=il_from' - ); - break; case 'categorylinks': $conds = array( - 'cl_to' => $this->title->getDBkey(), - 'page_id=cl_from', + "{$prefix}_to" => $this->title->getDBkey(), + "page_id={$prefix}_from" ); break; default: @@ -302,7 +314,7 @@ class BacklinkCache { /** * Check if there are any backlinks - * @param $table String + * @param string $table * @return bool */ public function hasLinks( $table ) { @@ -311,16 +323,17 @@ class BacklinkCache { /** * Get the approximate number of backlinks - * @param $table String - * @param $max integer|INF Only count up to this many backlinks - * @return integer + * @param string $table + * @param int|INF $max Only count up to this many backlinks + * @return int */ public function getNumLinks( $table, $max = INF ) { - global $wgMemc; + global $wgMemc, $wgUpdateRowsPerJob; // 1) try partition cache ... if ( isset( $this->partitionCache[$table] ) ) { $entry = reset( $this->partitionCache[$table] ); + return min( $max, $entry['numRows'] ); } @@ -338,9 +351,17 @@ class BacklinkCache { } // 4) fetch from the database ... - $count = $this->getLinks( $table, false, false, $max )->count(); - if ( $count < $max ) { // full count - $wgMemc->set( $memcKey, $count, self::CACHE_EXPIRY ); + if ( is_infinite( $max ) ) { // no limit at all + // Use partition() since it will batch the query and skip the JOIN. + // Use $wgUpdateRowsPerJob just to encourage cache reuse for jobs. + $this->partition( $table, $wgUpdateRowsPerJob ); // updates $this->partitionCache + return $this->partitionCache[$table][$wgUpdateRowsPerJob]['numRows']; + } else { // probably some sane limit + // Fetch the full title info, since the caller will likely need it next + $count = $this->getLinks( $table, false, false, $max )->count(); + if ( $count < $max ) { // full count + $wgMemc->set( $memcKey, $count, self::CACHE_EXPIRY ); + } } return min( $max, $count ); @@ -351,9 +372,9 @@ class BacklinkCache { * Returns an array giving the start and end of each range. The first * batch has a start of false, and the last batch has an end of false. * - * @param string $table the links table name - * @param $batchSize Integer - * @return Array + * @param string $table The links table name + * @param int $batchSize + * @return array */ public function partition( $table, $batchSize ) { global $wgMemc; @@ -361,6 +382,7 @@ class BacklinkCache { // 1) try partition cache ... if ( isset( $this->partitionCache[$table][$batchSize] ) ) { wfDebug( __METHOD__ . ": got from partition cache\n" ); + return $this->partitionCache[$table][$batchSize]['batches']; } @@ -371,6 +393,7 @@ class BacklinkCache { if ( isset( $this->fullResultCache[$table] ) ) { $cacheEntry = $this->partitionResult( $this->fullResultCache[$table], $batchSize ); wfDebug( __METHOD__ . ": got from full result cache\n" ); + return $cacheEntry['batches']; } @@ -386,6 +409,7 @@ class BacklinkCache { if ( is_array( $memcValue ) ) { $cacheEntry = $memcValue; wfDebug( __METHOD__ . ": got from memcached $memcKey\n" ); + return $cacheEntry['batches']; } @@ -396,13 +420,13 @@ class BacklinkCache { $selectSize = max( $batchSize, 200000 - ( 200000 % $batchSize ) ); $start = false; do { - $res = $this->queryLinks( $table, $start, false, $selectSize ); + $res = $this->queryLinks( $table, $start, false, $selectSize, 'ids' ); $partitions = $this->partitionResult( $res, $batchSize, false ); // Merge the link count and range partitions for this chunk $cacheEntry['numRows'] += $partitions['numRows']; $cacheEntry['batches'] = array_merge( $cacheEntry['batches'], $partitions['batches'] ); if ( count( $partitions['batches'] ) ) { - list( $lStart, $lEnd ) = end( $partitions['batches'] ); + list( , $lEnd ) = end( $partitions['batches'] ); $start = $lEnd + 1; // pick up after this inclusive range } } while ( $partitions['numRows'] >= $selectSize ); @@ -420,14 +444,15 @@ class BacklinkCache { $wgMemc->set( $memcKey, $cacheEntry['numRows'], self::CACHE_EXPIRY ); wfDebug( __METHOD__ . ": got from database\n" ); + return $cacheEntry['batches']; } /** * Partition a DB result with backlinks in it into batches - * @param $res ResultWrapper database result - * @param $batchSize integer - * @param $isComplete bool Whether $res includes all the backlinks + * @param ResultWrapper $res Database result + * @param int $batchSize + * @param bool $isComplete Whether $res includes all the backlinks * @throws MWException * @return array */ diff --git a/includes/cache/CacheDependency.php b/includes/cache/CacheDependency.php index 32bcdf7f..9b48ecb7 100644 --- a/includes/cache/CacheDependency.php +++ b/includes/cache/CacheDependency.php @@ -28,14 +28,15 @@ * @ingroup Cache */ class DependencyWrapper { - var $value; - var $deps; + private $value; + /** @var CacheDependency[] */ + private $deps; /** * Create an instance. - * @param $value Mixed: the user-supplied value - * @param $deps Mixed: a dependency or dependency array. All dependencies - * must be objects implementing CacheDependency. + * @param mixed $value The user-supplied value + * @param CacheDependency|CacheDependency[] $deps A dependency or dependency + * array. All dependencies must be objects implementing CacheDependency. */ function __construct( $value = false, $deps = array() ) { $this->value = $value; @@ -74,7 +75,7 @@ class DependencyWrapper { /** * Get the user-defined value - * @return bool|Mixed + * @return bool|mixed */ function getValue() { return $this->value; @@ -83,9 +84,9 @@ class DependencyWrapper { /** * Store the wrapper to a cache * - * @param $cache BagOStuff - * @param $key - * @param $expiry + * @param BagOStuff $cache + * @param string $key + * @param int $expiry */ function storeToCache( $cache, $key, $expiry = 0 ) { $this->initialiseDeps(); @@ -97,12 +98,12 @@ class DependencyWrapper { * it will be generated with the callback function (if present), and the newly * calculated value will be stored to the cache in a wrapper. * - * @param $cache BagOStuff a cache object such as $wgMemc - * @param string $key the cache key - * @param $expiry Integer: the expiry timestamp or interval in seconds - * @param $callback Mixed: the callback for generating the value, or false - * @param array $callbackParams the function parameters for the callback - * @param array $deps the dependencies to store on a cache miss. Note: these + * @param BagOStuff $cache A cache object such as $wgMemc + * @param string $key The cache key + * @param int $expiry The expiry timestamp or interval in seconds + * @param bool|callable $callback The callback for generating the value, or false + * @param array $callbackParams The function parameters for the callback + * @param array $deps The dependencies to store on a cache miss. Note: these * are not the dependencies used on a cache hit! Cache hits use the stored * dependency array. * @@ -110,8 +111,8 @@ class DependencyWrapper { * callback was defined. */ static function getValueFromCache( $cache, $key, $expiry = 0, $callback = false, - $callbackParams = array(), $deps = array() ) - { + $callbackParams = array(), $deps = array() + ) { $obj = $cache->get( $key ); if ( is_object( $obj ) && $obj instanceof DependencyWrapper && !$obj->isExpired() ) { @@ -141,20 +142,22 @@ abstract class CacheDependency { /** * Hook to perform any expensive pre-serialize loading of dependency values. */ - function loadDependencyValues() { } + function loadDependencyValues() { + } } /** * @ingroup Cache */ class FileDependency extends CacheDependency { - var $filename, $timestamp; + private $filename; + private $timestamp; /** * Create a file dependency * - * @param string $filename the name of the file, preferably fully qualified - * @param $timestamp Mixed: the unix last modified timestamp, or false if the + * @param string $filename The name of the file, preferably fully qualified + * @param null|bool|int $timestamp The unix last modified timestamp, or false if the * file does not exist. If omitted, the timestamp will be loaded from * the file. * @@ -172,6 +175,7 @@ class FileDependency extends CacheDependency { */ function __sleep() { $this->loadDependencyValues(); + return array( 'filename', 'timestamp' ); } @@ -198,6 +202,7 @@ class FileDependency extends CacheDependency { } else { # Deleted wfDebug( "Dependency triggered: {$this->filename} deleted.\n" ); + return true; } } else { @@ -205,6 +210,7 @@ class FileDependency extends CacheDependency { if ( $lastmod > $this->timestamp ) { # Modified or created wfDebug( "Dependency triggered: {$this->filename} changed.\n" ); + return true; } else { # Not modified @@ -217,183 +223,9 @@ class FileDependency extends CacheDependency { /** * @ingroup Cache */ -class TitleDependency extends CacheDependency { - var $titleObj; - var $ns, $dbk; - var $touched; - - /** - * Construct a title dependency - * @param $title Title - */ - function __construct( Title $title ) { - $this->titleObj = $title; - $this->ns = $title->getNamespace(); - $this->dbk = $title->getDBkey(); - } - - function loadDependencyValues() { - $this->touched = $this->getTitle()->getTouched(); - } - - /** - * Get rid of bulky Title object for sleep - * - * @return array - */ - function __sleep() { - return array( 'ns', 'dbk', 'touched' ); - } - - /** - * @return Title - */ - function getTitle() { - if ( !isset( $this->titleObj ) ) { - $this->titleObj = Title::makeTitle( $this->ns, $this->dbk ); - } - - return $this->titleObj; - } - - /** - * @return bool - */ - function isExpired() { - $touched = $this->getTitle()->getTouched(); - - if ( $this->touched === false ) { - if ( $touched === false ) { - # Still missing - return false; - } else { - # Created - return true; - } - } elseif ( $touched === false ) { - # Deleted - return true; - } elseif ( $touched > $this->touched ) { - # Updated - return true; - } else { - # Unmodified - return false; - } - } -} - -/** - * @ingroup Cache - */ -class TitleListDependency extends CacheDependency { - var $linkBatch; - var $timestamps; - - /** - * Construct a dependency on a list of titles - * @param $linkBatch LinkBatch - */ - function __construct( LinkBatch $linkBatch ) { - $this->linkBatch = $linkBatch; - } - - /** - * @return array - */ - function calculateTimestamps() { - # Initialise values to false - $timestamps = array(); - - foreach ( $this->getLinkBatch()->data as $ns => $dbks ) { - if ( count( $dbks ) > 0 ) { - $timestamps[$ns] = array(); - - foreach ( $dbks as $dbk => $value ) { - $timestamps[$ns][$dbk] = false; - } - } - } - - # Do the query - if ( count( $timestamps ) ) { - $dbr = wfGetDB( DB_SLAVE ); - $where = $this->getLinkBatch()->constructSet( 'page', $dbr ); - $res = $dbr->select( - 'page', - array( 'page_namespace', 'page_title', 'page_touched' ), - $where, - __METHOD__ - ); - - foreach ( $res as $row ) { - $timestamps[$row->page_namespace][$row->page_title] = $row->page_touched; - } - } - - return $timestamps; - } - - function loadDependencyValues() { - $this->timestamps = $this->calculateTimestamps(); - } - - /** - * @return array - */ - function __sleep() { - return array( 'timestamps' ); - } - - /** - * @return LinkBatch - */ - function getLinkBatch() { - if ( !isset( $this->linkBatch ) ) { - $this->linkBatch = new LinkBatch; - $this->linkBatch->setArray( $this->timestamps ); - } - return $this->linkBatch; - } - - /** - * @return bool - */ - function isExpired() { - $newTimestamps = $this->calculateTimestamps(); - - foreach ( $this->timestamps as $ns => $dbks ) { - foreach ( $dbks as $dbk => $oldTimestamp ) { - $newTimestamp = $newTimestamps[$ns][$dbk]; - - if ( $oldTimestamp === false ) { - if ( $newTimestamp === false ) { - # Still missing - } else { - # Created - return true; - } - } elseif ( $newTimestamp === false ) { - # Deleted - return true; - } elseif ( $newTimestamp > $oldTimestamp ) { - # Updated - return true; - } else { - # Unmodified - } - } - } - - return false; - } -} - -/** - * @ingroup Cache - */ class GlobalDependency extends CacheDependency { - var $name, $value; + private $name; + private $value; function __construct( $name ) { $this->name = $name; @@ -407,6 +239,7 @@ class GlobalDependency extends CacheDependency { if ( !isset( $GLOBALS[$this->name] ) ) { return true; } + return $GLOBALS[$this->name] != $this->value; } } @@ -415,7 +248,8 @@ class GlobalDependency extends CacheDependency { * @ingroup Cache */ class ConstantDependency extends CacheDependency { - var $name, $value; + private $name; + private $value; function __construct( $name ) { $this->name = $name; diff --git a/includes/cache/CacheHelper.php b/includes/cache/CacheHelper.php new file mode 100644 index 00000000..401c2b96 --- /dev/null +++ b/includes/cache/CacheHelper.php @@ -0,0 +1,386 @@ +<?php +/** + * Cache of various elements in a single cache entry. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @license GNU GPL v2 or later + * @author Jeroen De Dauw < jeroendedauw@gmail.com > + */ + +/** + * Interface for all classes implementing CacheHelper functionality. + * + * @since 1.20 + */ +interface ICacheHelper { + /** + * Sets if the cache should be enabled or not. + * + * @since 1.20 + * @param bool $cacheEnabled + */ + function setCacheEnabled( $cacheEnabled ); + + /** + * Initializes the caching. + * Should be called before the first time anything is added via addCachedHTML. + * + * @since 1.20 + * + * @param int|null $cacheExpiry Sets the cache expiry, either ttl in seconds or unix timestamp. + * @param bool|null $cacheEnabled Sets if the cache should be enabled or not. + */ + function startCache( $cacheExpiry = null, $cacheEnabled = null ); + + /** + * Get a cached value if available or compute it if not and then cache it if possible. + * The provided $computeFunction is only called when the computation needs to happen + * and should return a result value. $args are arguments that will be passed to the + * compute function when called. + * + * @since 1.20 + * + * @param callable $computeFunction + * @param array|mixed $args + * @param string|null $key + * + * @return mixed + */ + function getCachedValue( $computeFunction, $args = array(), $key = null ); + + /** + * Saves the HTML to the cache in case it got recomputed. + * Should be called after the last time anything is added via addCachedHTML. + * + * @since 1.20 + */ + function saveCache(); + + /** + * Sets the time to live for the cache, in seconds or a unix timestamp + * indicating the point of expiry... + * + * @since 1.20 + * + * @param int $cacheExpiry + */ + function setExpiry( $cacheExpiry ); +} + +/** + * Helper class for caching various elements in a single cache entry. + * + * To get a cached value or compute it, use getCachedValue like this: + * $this->getCachedValue( $callback ); + * + * To add HTML that should be cached, use addCachedHTML like this: + * $this->addCachedHTML( $callback ); + * + * The callback function is only called when needed, so do all your expensive + * computations here. This function should returns the HTML to be cached. + * It should not add anything to the PageOutput object! + * + * Before the first addCachedHTML call, you should call $this->startCache(); + * After adding the last HTML that should be cached, call $this->saveCache(); + * + * @since 1.20 + */ +class CacheHelper implements ICacheHelper { + /** + * The time to live for the cache, in seconds or a unix timestamp indicating the point of expiry. + * + * @since 1.20 + * @var int + */ + protected $cacheExpiry = 3600; + + /** + * List of HTML chunks to be cached (if !hasCached) or that where cached (of hasCached). + * If not cached already, then the newly computed chunks are added here, + * if it as cached already, chunks are removed from this list as they are needed. + * + * @since 1.20 + * @var array + */ + protected $cachedChunks; + + /** + * Indicates if the to be cached content was already cached. + * Null if this information is not available yet. + * + * @since 1.20 + * @var bool|null + */ + protected $hasCached = null; + + /** + * If the cache is enabled or not. + * + * @since 1.20 + * @var bool + */ + protected $cacheEnabled = true; + + /** + * Function that gets called when initialization is done. + * + * @since 1.20 + * @var callable + */ + protected $onInitHandler = false; + + /** + * Elements to build a cache key with. + * + * @since 1.20 + * @var array + */ + protected $cacheKey = array(); + + /** + * Sets if the cache should be enabled or not. + * + * @since 1.20 + * @param bool $cacheEnabled + */ + public function setCacheEnabled( $cacheEnabled ) { + $this->cacheEnabled = $cacheEnabled; + } + + /** + * Initializes the caching. + * Should be called before the first time anything is added via addCachedHTML. + * + * @since 1.20 + * + * @param int|null $cacheExpiry Sets the cache expiry, either ttl in seconds or unix timestamp. + * @param bool|null $cacheEnabled Sets if the cache should be enabled or not. + */ + public function startCache( $cacheExpiry = null, $cacheEnabled = null ) { + if ( is_null( $this->hasCached ) ) { + if ( !is_null( $cacheExpiry ) ) { + $this->cacheExpiry = $cacheExpiry; + } + + if ( !is_null( $cacheEnabled ) ) { + $this->setCacheEnabled( $cacheEnabled ); + } + + $this->initCaching(); + } + } + + /** + * Returns a message that notifies the user he/she is looking at + * a cached version of the page, including a refresh link. + * + * @since 1.20 + * + * @param IContextSource $context + * @param bool $includePurgeLink + * + * @return string + */ + public function getCachedNotice( IContextSource $context, $includePurgeLink = true ) { + if ( $this->cacheExpiry < 86400 * 3650 ) { + $message = $context->msg( + 'cachedspecial-viewing-cached-ttl', + $context->getLanguage()->formatDuration( $this->cacheExpiry ) + )->escaped(); + } else { + $message = $context->msg( + 'cachedspecial-viewing-cached-ts' + )->escaped(); + } + + if ( $includePurgeLink ) { + $refreshArgs = $context->getRequest()->getQueryValues(); + unset( $refreshArgs['title'] ); + $refreshArgs['action'] = 'purge'; + + $subPage = $context->getTitle()->getFullText(); + $subPage = explode( '/', $subPage, 2 ); + $subPage = count( $subPage ) > 1 ? $subPage[1] : false; + + $message .= ' ' . Linker::link( + $context->getTitle( $subPage ), + $context->msg( 'cachedspecial-refresh-now' )->escaped(), + array(), + $refreshArgs + ); + } + + return $message; + } + + /** + * Initializes the caching if not already done so. + * Should be called before any of the caching functionality is used. + * + * @since 1.20 + */ + protected function initCaching() { + if ( $this->cacheEnabled && is_null( $this->hasCached ) ) { + $cachedChunks = wfGetCache( CACHE_ANYTHING )->get( $this->getCacheKeyString() ); + + $this->hasCached = is_array( $cachedChunks ); + $this->cachedChunks = $this->hasCached ? $cachedChunks : array(); + + if ( $this->onInitHandler !== false ) { + call_user_func( $this->onInitHandler, $this->hasCached ); + } + } + } + + /** + * Get a cached value if available or compute it if not and then cache it if possible. + * The provided $computeFunction is only called when the computation needs to happen + * and should return a result value. $args are arguments that will be passed to the + * compute function when called. + * + * @since 1.20 + * + * @param callable $computeFunction + * @param array|mixed $args + * @param string|null $key + * + * @return mixed + */ + public function getCachedValue( $computeFunction, $args = array(), $key = null ) { + $this->initCaching(); + + if ( $this->cacheEnabled && $this->hasCached ) { + $value = null; + + if ( is_null( $key ) ) { + $itemKey = array_keys( array_slice( $this->cachedChunks, 0, 1 ) ); + $itemKey = array_shift( $itemKey ); + + if ( !is_integer( $itemKey ) ) { + wfWarn( "Attempted to get item with non-numeric key while " . + "the next item in the queue has a key ($itemKey) in " . __METHOD__ ); + } elseif ( is_null( $itemKey ) ) { + wfWarn( "Attempted to get an item while the queue is empty in " . __METHOD__ ); + } else { + $value = array_shift( $this->cachedChunks ); + } + } else { + if ( array_key_exists( $key, $this->cachedChunks ) ) { + $value = $this->cachedChunks[$key]; + unset( $this->cachedChunks[$key] ); + } else { + wfWarn( "There is no item with key '$key' in this->cachedChunks in " . __METHOD__ ); + } + } + } else { + if ( !is_array( $args ) ) { + $args = array( $args ); + } + + $value = call_user_func_array( $computeFunction, $args ); + + if ( $this->cacheEnabled ) { + if ( is_null( $key ) ) { + $this->cachedChunks[] = $value; + } else { + $this->cachedChunks[$key] = $value; + } + } + } + + return $value; + } + + /** + * Saves the HTML to the cache in case it got recomputed. + * Should be called after the last time anything is added via addCachedHTML. + * + * @since 1.20 + */ + public function saveCache() { + if ( $this->cacheEnabled && $this->hasCached === false && !empty( $this->cachedChunks ) ) { + wfGetCache( CACHE_ANYTHING )->set( + $this->getCacheKeyString(), + $this->cachedChunks, + $this->cacheExpiry + ); + } + } + + /** + * Sets the time to live for the cache, in seconds or a unix timestamp + * indicating the point of expiry... + * + * @since 1.20 + * + * @param int $cacheExpiry + */ + public function setExpiry( $cacheExpiry ) { + $this->cacheExpiry = $cacheExpiry; + } + + /** + * Returns the cache key to use to cache this page's HTML output. + * Is constructed from the special page name and language code. + * + * @since 1.20 + * + * @return string + * @throws MWException + */ + protected function getCacheKeyString() { + if ( $this->cacheKey === array() ) { + throw new MWException( 'No cache key set, so cannot obtain or save the CacheHelper values.' ); + } + + return call_user_func_array( 'wfMemcKey', $this->cacheKey ); + } + + /** + * Sets the cache key that should be used. + * + * @since 1.20 + * + * @param array $cacheKey + */ + public function setCacheKey( array $cacheKey ) { + $this->cacheKey = $cacheKey; + } + + /** + * Rebuild the content, even if it's already cached. + * This effectively has the same effect as purging the cache, + * since it will be overridden with the new value on the next request. + * + * @since 1.20 + */ + public function rebuildOnDemand() { + $this->hasCached = false; + } + + /** + * Sets a function that gets called when initialization of the cache is done. + * + * @since 1.20 + * + * @param callable $handlerFunction + */ + public function setOnInitializedHandler( $handlerFunction ) { + $this->onInitHandler = $handlerFunction; + } +} diff --git a/includes/cache/FileCacheBase.php b/includes/cache/FileCacheBase.php index d4bf5ee6..4bf36114 100644 --- a/includes/cache/FileCacheBase.php +++ b/includes/cache/FileCacheBase.php @@ -51,6 +51,7 @@ abstract class FileCacheBase { */ final protected function baseCacheDirectory() { global $wgFileCacheDirectory; + return $wgFileCacheDirectory; } @@ -91,6 +92,7 @@ abstract class FileCacheBase { if ( $this->mCached === null ) { $this->mCached = file_exists( $this->cachePath() ); } + return $this->mCached; } @@ -100,6 +102,7 @@ abstract class FileCacheBase { */ public function cacheTimestamp() { $timestamp = filemtime( $this->cachePath() ); + return ( $timestamp !== false ) ? wfTimestamp( TS_MW, $timestamp ) : false; @@ -120,7 +123,8 @@ abstract class FileCacheBase { $cachetime = $this->cacheTimestamp(); $good = ( $timestamp <= $cachetime && $wgCacheEpoch <= $cachetime ); - wfDebug( __METHOD__ . ": cachetime $cachetime, touched '{$timestamp}' epoch {$wgCacheEpoch}, good $good\n" ); + wfDebug( __METHOD__ . + ": cachetime $cachetime, touched '{$timestamp}' epoch {$wgCacheEpoch}, good $good\n" ); return $good; } @@ -140,6 +144,7 @@ abstract class FileCacheBase { public function fetchText() { if ( $this->useGzip() ) { $fh = gzopen( $this->cachePath(), 'rb' ); + return stream_get_contents( $fh ); } else { return file_get_contents( $this->cachePath() ); @@ -148,7 +153,8 @@ abstract class FileCacheBase { /** * Save and compress text to the cache - * @return string compressed text + * @param string $text + * @return string Compressed text */ public function saveText( $text ) { global $wgUseFileCache; @@ -165,10 +171,12 @@ abstract class FileCacheBase { if ( !file_put_contents( $this->cachePath(), $text, LOCK_EX ) ) { wfDebug( __METHOD__ . "() failed saving " . $this->cachePath() . "\n" ); $this->mCached = null; + return false; } $this->mCached = true; + return $text; } @@ -223,7 +231,7 @@ abstract class FileCacheBase { /** * Roughly increments the cache misses in the last hour by unique visitors - * @param $request WebRequest + * @param WebRequest $request * @return void */ public function incrMissesRecent( WebRequest $request ) { @@ -262,6 +270,7 @@ abstract class FileCacheBase { */ public function getMissesRecent() { global $wgMemc; + return self::MISS_FACTOR * $wgMemc->get( $this->cacheMissKey() ); } diff --git a/includes/cache/GenderCache.php b/includes/cache/GenderCache.php index a933527a..63e7bfd7 100644 --- a/includes/cache/GenderCache.php +++ b/includes/cache/GenderCache.php @@ -41,27 +41,30 @@ class GenderCache { if ( $that === null ) { $that = new self(); } + return $that; } - protected function __construct() {} + protected function __construct() { + } /** * Returns the default gender option in this wiki. - * @return String + * @return string */ protected function getDefault() { if ( $this->default === null ) { $this->default = User::getDefaultOption( 'gender' ); } + return $this->default; } /** * Returns the gender for given username. - * @param string $username or User: username - * @param string $caller the calling method - * @return String + * @param string|User $username Username + * @param string $caller The calling method + * @return string */ public function getGenderOf( $username, $caller = '' ) { global $wgUser; @@ -77,8 +80,8 @@ class GenderCache { $this->misses++; wfDebug( __METHOD__ . ": too many misses, returning default onwards\n" ); } - return $this->getDefault(); + return $this->getDefault(); } else { $this->misses++; $this->doQuery( $username, $caller ); @@ -94,8 +97,8 @@ class GenderCache { /** * Wrapper for doQuery that processes raw LinkBatch data. * - * @param $data - * @param $caller + * @param array $data + * @param string $caller */ public function doLinkBatch( $data, $caller = '' ) { $users = array(); @@ -115,8 +118,8 @@ class GenderCache { * Wrapper for doQuery that processes a title or string array. * * @since 1.20 - * @param $titles List: array of Title objects or strings - * @param string $caller the calling method + * @param array $titles Array of Title objects or strings + * @param string $caller The calling method */ public function doTitlesArray( $titles, $caller = '' ) { $users = array(); @@ -136,8 +139,8 @@ class GenderCache { /** * Preloads genders for given list of users. - * @param $users List|String: usernames - * @param string $caller the calling method + * @param array|string $users Usernames + * @param string $caller The calling method */ public function doQuery( $users, $caller = '' ) { $default = $this->getDefault(); @@ -184,6 +187,7 @@ class GenderCache { if ( $indexSlash !== false ) { $username = substr( $username, 0, $indexSlash ); } + // normalize underscore/spaces return strtr( $username, '_', ' ' ); } diff --git a/includes/cache/HTMLCacheUpdate.php b/includes/cache/HTMLCacheUpdate.php deleted file mode 100644 index 992809ef..00000000 --- a/includes/cache/HTMLCacheUpdate.php +++ /dev/null @@ -1,73 +0,0 @@ -<?php -/** - * HTML cache invalidation of all pages linking to a given title. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * http://www.gnu.org/copyleft/gpl.html - * - * @file - * @ingroup Cache - */ - -/** - * Class to invalidate the HTML cache of all the pages linking to a given title. - * - * @ingroup Cache - */ -class HTMLCacheUpdate implements DeferrableUpdate { - /** - * @var Title - */ - public $mTitle; - - public $mTable; - - /** - * @param $titleTo - * @param $table - * @param $start bool - * @param $end bool - */ - function __construct( Title $titleTo, $table ) { - $this->mTitle = $titleTo; - $this->mTable = $table; - } - - public function doUpdate() { - wfProfileIn( __METHOD__ ); - - $job = new HTMLCacheUpdateJob( - $this->mTitle, - array( - 'table' => $this->mTable, - ) + Job::newRootJobParams( // "overall" refresh links job info - "htmlCacheUpdate:{$this->mTable}:{$this->mTitle->getPrefixedText()}" - ) - ); - - $count = $this->mTitle->getBacklinkCache()->getNumLinks( $this->mTable, 200 ); - if ( $count >= 200 ) { // many backlinks - JobQueueGroup::singleton()->push( $job ); - JobQueueGroup::singleton()->deduplicateRootJob( $job ); - } else { // few backlinks ($count might be off even if 0) - $dbw = wfGetDB( DB_MASTER ); - $dbw->onTransactionIdle( function() use ( $job ) { - $job->run(); // just do the purge query now - } ); - } - - wfProfileOut( __METHOD__ ); - } -} diff --git a/includes/cache/HTMLFileCache.php b/includes/cache/HTMLFileCache.php index ab379116..58ca2dcd 100644 --- a/includes/cache/HTMLFileCache.php +++ b/includes/cache/HTMLFileCache.php @@ -31,25 +31,32 @@ class HTMLFileCache extends FileCacheBase { /** * Construct an ObjectFileCache from a Title and an action - * @param $title Title|string Title object or prefixed DB key string - * @param $action string + * @param Title|string $title Title object or prefixed DB key string + * @param string $action * @throws MWException * @return HTMLFileCache + * + * @deprecated Since 1.24, instantiate this class directly */ public static function newFromTitle( $title, $action ) { - $cache = new self(); + return new self( $title, $action ); + } + /** + * @param Title|string $title Title object or prefixed DB key string + * @param string $action + * @throws MWException + */ + public function __construct( $title, $action ) { $allowedTypes = self::cacheablePageActions(); if ( !in_array( $action, $allowedTypes ) ) { - throw new MWException( "Invalid filecache type given." ); + throw new MWException( 'Invalid file cache type given.' ); } - $cache->mKey = ( $title instanceof Title ) + $this->mKey = ( $title instanceof Title ) ? $title->getPrefixedDBkey() : (string)$title; - $cache->mType = (string)$action; - $cache->mExt = 'html'; - - return $cache; + $this->mType = (string)$action; + $this->mExt = 'html'; } /** @@ -84,7 +91,7 @@ class HTMLFileCache extends FileCacheBase { /** * Check if pages can be cached for this request/user - * @param $context IContextSource + * @param IContextSource $context * @return bool */ public static function useFileCache( IContextSource $context ) { @@ -94,6 +101,7 @@ class HTMLFileCache extends FileCacheBase { } if ( $wgShowIPinHeader || $wgDebugToolbar ) { wfDebug( "HTML file cache skipped. Either \$wgShowIPinHeader and/or \$wgDebugToolbar on\n" ); + return false; } @@ -109,6 +117,7 @@ class HTMLFileCache extends FileCacheBase { } elseif ( $query === 'maxage' || $query === 'smaxage' ) { continue; } + return false; } $user = $context->getUser(); @@ -116,13 +125,18 @@ class HTMLFileCache extends FileCacheBase { // and extensions for auto-detecting user language. $ulang = $context->getLanguage()->getCode(); $clang = $wgContLang->getCode(); + // Check that there are no other sources of variation - return !$user->getId() && !$user->getNewtalk() && $ulang == $clang; + if ( $user->getId() || $user->getNewtalk() || $ulang != $clang ) { + return false; + } + // Allow extensions to disable caching + return wfRunHooks( 'HTMLFileCache::useFileCache', array( $context ) ); } /** * Read from cache to context output - * @param $context IContextSource + * @param IContextSource $context * @return void */ public function loadFromFileCache( IContextSource $context ) { @@ -152,7 +166,7 @@ class HTMLFileCache extends FileCacheBase { /** * Save this cache object with the given text. * Use this as an ob_start() handler. - * @param $text string + * @param string $text * @return bool Whether $wgUseFileCache is enabled */ public function saveToFileCache( $text ) { @@ -163,7 +177,7 @@ class HTMLFileCache extends FileCacheBase { return $text; } - wfDebug( __METHOD__ . "()\n", false ); + wfDebug( __METHOD__ . "()\n", 'log' ); $now = wfTimestampNow(); if ( $this->useGzip() ) { @@ -185,6 +199,7 @@ class HTMLFileCache extends FileCacheBase { // @todo Ugly wfClientAcceptsGzip() function - use context! if ( wfClientAcceptsGzip() ) { header( 'Content-Encoding: gzip' ); + return $compressed; } else { return $text; @@ -196,7 +211,7 @@ class HTMLFileCache extends FileCacheBase { /** * Clear the file caches for a page for all actions - * @param $title Title + * @param Title $title * @return bool Whether $wgUseFileCache is enabled */ public static function clearFileCache( Title $title ) { @@ -207,7 +222,7 @@ class HTMLFileCache extends FileCacheBase { } foreach ( self::cacheablePageActions() as $type ) { - $fc = self::newFromTitle( $title, $type ); + $fc = new self( $title, $type ); $fc->clearCache(); } diff --git a/includes/cache/LinkBatch.php b/includes/cache/LinkBatch.php index 48b60aa9..48c063f4 100644 --- a/includes/cache/LinkBatch.php +++ b/includes/cache/LinkBatch.php @@ -31,7 +31,7 @@ class LinkBatch { /** * 2-d array, first index namespace, second index dbkey, value arbitrary */ - var $data = array(); + public $data = array(); /** * For debugging which method is using this class. @@ -49,14 +49,14 @@ class LinkBatch { * class. Only used in debugging output. * @since 1.17 * - * @param $caller + * @param string $caller */ public function setCaller( $caller ) { $this->caller = $caller; } /** - * @param $title Title + * @param Title $title */ public function addObj( $title ) { if ( is_object( $title ) ) { @@ -67,9 +67,8 @@ class LinkBatch { } /** - * @param $ns int - * @param $dbkey string - * @return mixed + * @param int $ns + * @param string $dbkey */ public function add( $ns, $dbkey ) { if ( $ns < 0 ) { @@ -86,7 +85,7 @@ class LinkBatch { * Set the link list to a given 2-d array * First key is the namespace, second is the DB key, value arbitrary * - * @param $array array + * @param array $array */ public function setArray( $array ) { $this->data = $array; @@ -113,10 +112,11 @@ class LinkBatch { /** * Do the query and add the results to the LinkCache object * - * @return Array mapping PDBK to ID + * @return array Mapping PDBK to ID */ public function execute() { $linkCache = LinkCache::singleton(); + return $this->executeInto( $linkCache ); } @@ -124,8 +124,8 @@ class LinkBatch { * Do the query and add the results to a given LinkCache object * Return an array mapping PDBK to ID * - * @param $cache LinkCache - * @return Array remaining IDs + * @param LinkCache $cache + * @return array Remaining IDs */ protected function executeInto( &$cache ) { wfProfileIn( __METHOD__ ); @@ -133,6 +133,7 @@ class LinkBatch { $this->doGenderQuery(); $ids = $this->addResultToCache( $cache, $res ); wfProfileOut( __METHOD__ ); + return $ids; } @@ -142,9 +143,9 @@ class LinkBatch { * This function *also* stores extra fields of the title used for link * parsing to avoid extra DB queries. * - * @param $cache LinkCache - * @param $res - * @return Array of remaining titles + * @param LinkCache $cache + * @param ResultWrapper $res + * @return array Array of remaining titles */ public function addResultToCache( $cache, $res ) { if ( !$res ) { @@ -170,14 +171,17 @@ class LinkBatch { $ids[$title->getPrefixedDBkey()] = 0; } } + return $ids; } /** * Perform the existence test query, return a ResultWrapper with page_id fields - * @return Bool|ResultWrapper + * @return bool|ResultWrapper */ public function doQuery() { + global $wgContentHandlerUseDB; + if ( $this->isEmpty() ) { return false; } @@ -188,6 +192,11 @@ class LinkBatch { $table = 'page'; $fields = array( 'page_id', 'page_namespace', 'page_title', 'page_len', 'page_is_redirect', 'page_latest' ); + + if ( $wgContentHandlerUseDB ) { + $fields[] = 'page_content_model'; + } + $conds = $this->constructSet( 'page', $dbr ); // Do query @@ -197,13 +206,14 @@ class LinkBatch { } $res = $dbr->select( $table, $fields, $conds, $caller ); wfProfileOut( __METHOD__ ); + return $res; } /** * Do (and cache) {{GENDER:...}} information for userpages in this LinkBatch * - * @return bool whether the query was successful + * @return bool Whether the query was successful */ public function doGenderQuery() { if ( $this->isEmpty() ) { @@ -217,15 +227,16 @@ class LinkBatch { $genderCache = GenderCache::singleton(); $genderCache->doLinkBatch( $this->data, $this->caller ); + return true; } /** * Construct a WHERE clause which will match all the given titles. * - * @param string $prefix the appropriate table's field name prefix ('page', 'pl', etc) - * @param $db DatabaseBase object to use - * @return mixed string with SQL where clause fragment, or false if no items. + * @param string $prefix The appropriate table's field name prefix ('page', 'pl', etc) + * @param DatabaseBase $db DatabaseBase object to use + * @return string|bool String with SQL where clause fragment, or false if no items. */ public function constructSet( $prefix, $db ) { return $db->makeWhereFrom2d( $this->data, "{$prefix}_namespace", "{$prefix}_title" ); diff --git a/includes/cache/LinkCache.php b/includes/cache/LinkCache.php index 54de1989..6925df90 100644 --- a/includes/cache/LinkCache.php +++ b/includes/cache/LinkCache.php @@ -35,7 +35,6 @@ class LinkCache { private $mGoodLinkFields = array(); private $mBadLinks = array(); private $mForUpdate = false; - private $useDatabase = true; /** * @var LinkCache @@ -52,6 +51,7 @@ class LinkCache { return self::$instance; } self::$instance = new LinkCache; + return self::$instance; } @@ -78,6 +78,7 @@ class LinkCache { /** * General accessor to get/set whether SELECT FOR UPDATE should be used * + * @param bool $update * @return bool */ public function forUpdate( $update = null ) { @@ -85,8 +86,8 @@ class LinkCache { } /** - * @param $title - * @return array|int + * @param string $title + * @return int */ public function getGoodLinkID( $title ) { if ( array_key_exists( $title, $this->mGoodLinks ) ) { @@ -99,9 +100,9 @@ class LinkCache { /** * Get a field of a title object from cache. * If this link is not good, it will return NULL. - * @param $title Title + * @param Title $title * @param string $field ('length','redirect','revision','model') - * @return mixed + * @return string|null */ public function getGoodLinkFieldObj( $title, $field ) { $dbkey = $title->getPrefixedDBkey(); @@ -113,7 +114,7 @@ class LinkCache { } /** - * @param $title + * @param string $title * @return bool */ public function isBadLink( $title ) { @@ -123,28 +124,31 @@ class LinkCache { /** * Add a link for the title to the link cache * - * @param $id Integer: page's ID - * @param $title Title object - * @param $len Integer: text's length - * @param $redir Integer: whether the page is a redirect - * @param $revision Integer: latest revision's ID - * @param $model Integer: latest revision's content model ID + * @param int $id Page's ID + * @param Title $title + * @param int $len Text's length + * @param int $redir Whether the page is a redirect + * @param int $revision Latest revision's ID + * @param string|null $model Latest revision's content model ID */ - public function addGoodLinkObj( $id, $title, $len = -1, $redir = null, $revision = false, $model = false ) { + public function addGoodLinkObj( $id, $title, $len = -1, $redir = null, + $revision = 0, $model = null + ) { $dbkey = $title->getPrefixedDBkey(); - $this->mGoodLinks[$dbkey] = intval( $id ); + $this->mGoodLinks[$dbkey] = (int)$id; $this->mGoodLinkFields[$dbkey] = array( - 'length' => intval( $len ), - 'redirect' => intval( $redir ), - 'revision' => intval( $revision ), - 'model' => intval( $model ) ); + 'length' => (int)$len, + 'redirect' => (int)$redir, + 'revision' => (int)$revision, + 'model' => $model ? (string)$model : null, + ); } /** * Same as above with better interface. * @since 1.19 - * @param $title Title - * @param $row object which has the fields page_id, page_is_redirect, + * @param Title $title + * @param stdClass $row Object which has the fields page_id, page_is_redirect, * page_latest and page_content_model */ public function addGoodLinkObjFromRow( $title, $row ) { @@ -159,7 +163,7 @@ class LinkCache { } /** - * @param $title Title + * @param Title $title */ public function addBadLinkObj( $title ) { $dbkey = $title->getPrefixedDBkey(); @@ -173,7 +177,7 @@ class LinkCache { } /** - * @param $title Title + * @param Title $title */ public function clearLink( $title ) { $dbkey = $title->getPrefixedDBkey(); @@ -193,8 +197,8 @@ class LinkCache { /** * Add a title to the link cache, return the page_id or zero if non-existent * - * @param string $title title to add - * @return Integer + * @param string $title Title to add + * @return int */ public function addLink( $title ) { $nt = Title::newFromDBkey( $title ); @@ -206,23 +210,10 @@ class LinkCache { } /** - * Enable or disable database use. - * @since 1.22 - * @param $value Boolean - * @return Boolean - */ - public function useDatabase( $value = null ) { - if ( $value !== null ) { - $this->useDatabase = (bool)$value; - } - return $this->useDatabase; - } - - /** * Add a title to the link cache, return the page_id or zero if non-existent * - * @param $nt Title object to add - * @return Integer + * @param Title $nt Title object to add + * @return int */ public function addLinkObj( $nt ) { global $wgAntiLockFlags, $wgContentHandlerUseDB; @@ -232,20 +223,19 @@ class LinkCache { $key = $nt->getPrefixedDBkey(); if ( $this->isBadLink( $key ) || $nt->isExternal() ) { wfProfileOut( __METHOD__ ); + return 0; } $id = $this->getGoodLinkID( $key ); if ( $id != 0 ) { wfProfileOut( __METHOD__ ); + return $id; } if ( $key === '' ) { wfProfileOut( __METHOD__ ); - return 0; - } - if( !$this->useDatabase ) { return 0; } @@ -280,6 +270,7 @@ class LinkCache { } wfProfileOut( __METHOD__ ); + return $id; } diff --git a/includes/cache/LocalisationCache.php b/includes/cache/LocalisationCache.php index 25a1e196..ae27fba3 100644 --- a/includes/cache/LocalisationCache.php +++ b/includes/cache/LocalisationCache.php @@ -20,8 +20,6 @@ * @file */ -define( 'MW_LC_VERSION', 2 ); - /** * Class for caching the contents of localisation files, Messages*.php * and *.i18n.php. @@ -35,20 +33,22 @@ define( 'MW_LC_VERSION', 2 ); * as grammatical transformation, is done by the caller. */ class LocalisationCache { + const VERSION = 2; + /** Configuration associative array */ - var $conf; + private $conf; /** * True if recaching should only be done on an explicit call to recache(). * Setting this reduces the overhead of cache freshness checking, which * requires doing a stat() for every extension i18n file. */ - var $manualRecache = false; + private $manualRecache = false; /** * True to treat all files as expired until they are regenerated by this object. */ - var $forceRecache = false; + private $forceRecache = false; /** * The cache data. 3-d array, where the first key is the language code, @@ -56,14 +56,14 @@ class LocalisationCache { * an item specific subkey index. Some items are not arrays and so for those * items, there are no subkeys. */ - var $data = array(); + protected $data = array(); /** * The persistent store object. An instance of LCStore. * * @var LCStore */ - var $store; + private $store; /** * A 2-d associative array, code/key, where presence indicates that the item @@ -72,32 +72,32 @@ class LocalisationCache { * For split items, if set, this indicates that all of the subitems have been * loaded. */ - var $loadedItems = array(); + private $loadedItems = array(); /** * A 3-d associative array, code/key/subkey, where presence indicates that * the subitem is loaded. Only used for the split items, i.e. messages. */ - var $loadedSubitems = array(); + private $loadedSubitems = array(); /** * An array where presence of a key indicates that that language has been * initialised. Initialisation includes checking for cache expiry and doing * any necessary updates. */ - var $initialisedLangs = array(); + private $initialisedLangs = array(); /** * An array mapping non-existent pseudo-languages to fallback languages. This * is filled by initShallowFallback() when data is requested from a language * that lacks a Messages*.php file. */ - var $shallowFallbacks = array(); + private $shallowFallbacks = array(); /** * An array where the keys are codes that have been recached by this instance. */ - var $recachedLangs = array(); + private $recachedLangs = array(); /** * All item keys @@ -106,7 +106,7 @@ class LocalisationCache { 'fallback', 'namespaceNames', 'bookstoreList', 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns', 'digitTransformTable', 'separatorTransformTable', 'fallback8bitEncoding', 'linkPrefixExtension', - 'linkTrail', 'namespaceAliases', + 'linkTrail', 'linkPrefixCharset', 'namespaceAliases', 'dateFormats', 'datePreferences', 'datePreferenceMigrationMap', 'defaultDateFormat', 'extraUserToggles', 'specialPageAliases', 'imageFiles', 'preloadedMessages', 'namespaceGenderAliases', @@ -158,7 +158,7 @@ class LocalisationCache { * Associative array of cached plural rules. The key is the language code, * the value is an array of plural rules for that language. */ - var $pluralRules = null; + private $pluralRules = null; /** * Associative array of cached plural rule types. The key is the language @@ -172,16 +172,16 @@ class LocalisationCache { * example, {{plural:count|wordform1|wordform2|wordform3}}, rather than * {{plural:count|one=wordform1|two=wordform2|many=wordform3}}. */ - var $pluralRuleTypes = null; + private $pluralRuleTypes = null; - var $mergeableKeys = null; + private $mergeableKeys = null; /** * Constructor. * For constructor parameters, see the documentation in DefaultSettings.php * for $wgLocalisationCacheConf. * - * @param $conf Array + * @param array $conf * @throws MWException */ function __construct( $conf ) { @@ -195,16 +195,13 @@ class LocalisationCache { switch ( $conf['store'] ) { case 'files': case 'file': - $storeClass = 'LCStore_CDB'; + $storeClass = 'LCStoreCDB'; break; case 'db': - $storeClass = 'LCStore_DB'; - break; - case 'accel': - $storeClass = 'LCStore_Accel'; + $storeClass = 'LCStoreDB'; break; case 'detect': - $storeClass = $wgCacheDirectory ? 'LCStore_CDB' : 'LCStore_DB'; + $storeClass = $wgCacheDirectory ? 'LCStoreCDB' : 'LCStoreDB'; break; default: throw new MWException( @@ -212,7 +209,7 @@ class LocalisationCache { } } - wfDebug( get_class( $this ) . ": using store $storeClass\n" ); + wfDebugLog( 'caches', get_class( $this ) . ": using store $storeClass" ); if ( !empty( $conf['storeDirectory'] ) ) { $storeConf['directory'] = $conf['storeDirectory']; } @@ -228,7 +225,7 @@ class LocalisationCache { /** * Returns true if the given key is mergeable, that is, if it is an associative * array which can be merged through a fallback sequence. - * @param $key + * @param string $key * @return bool */ public function isMergeableKey( $key ) { @@ -241,6 +238,7 @@ class LocalisationCache { self::$magicWordKeys ) ); } + return isset( $this->mergeableKeys[$key] ); } @@ -249,8 +247,8 @@ class LocalisationCache { * * Warning: this may be slow for split items (messages), since it will * need to fetch all of the subitems from the cache individually. - * @param $code - * @param $key + * @param string $code + * @param string $key * @return mixed */ public function getItem( $code, $key ) { @@ -269,14 +267,15 @@ class LocalisationCache { /** * Get a subitem, for instance a single message for a given language. - * @param $code - * @param $key - * @param $subkey - * @return null + * @param string $code + * @param string $key + * @param string $subkey + * @return mixed|null */ public function getSubitem( $code, $key, $subkey ) { if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) && - !isset( $this->loadedItems[$code][$key] ) ) { + !isset( $this->loadedItems[$code][$key] ) + ) { wfProfileIn( __METHOD__ . '-load' ); $this->loadSubitem( $code, $key, $subkey ); wfProfileOut( __METHOD__ . '-load' ); @@ -297,8 +296,8 @@ class LocalisationCache { * * Will return null if the item is not found, or false if the item is not an * array. - * @param $code - * @param $key + * @param string $code + * @param string $key * @return bool|null|string */ public function getSubitemList( $code, $key ) { @@ -316,8 +315,8 @@ class LocalisationCache { /** * Load an item into the cache. - * @param $code - * @param $key + * @param string $code + * @param string $key */ protected function loadItem( $code, $key ) { if ( !isset( $this->initialisedLangs[$code] ) ) { @@ -331,6 +330,7 @@ class LocalisationCache { if ( isset( $this->shallowFallbacks[$code] ) ) { $this->loadItem( $this->shallowFallbacks[$code], $key ); + return; } @@ -351,13 +351,14 @@ class LocalisationCache { /** * Load a subitem into the cache - * @param $code - * @param $key - * @param $subkey + * @param string $code + * @param string $key + * @param string $subkey */ protected function loadSubitem( $code, $key, $subkey ) { if ( !in_array( $key, self::$splitKeys ) ) { $this->loadItem( $code, $key ); + return; } @@ -367,12 +368,14 @@ class LocalisationCache { // Check to see if initLanguage() loaded it for us if ( isset( $this->loadedItems[$code][$key] ) || - isset( $this->loadedSubitems[$code][$key][$subkey] ) ) { + isset( $this->loadedSubitems[$code][$key][$subkey] ) + ) { return; } if ( isset( $this->shallowFallbacks[$code] ) ) { $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey ); + return; } @@ -391,15 +394,17 @@ class LocalisationCache { public function isExpired( $code ) { if ( $this->forceRecache && !isset( $this->recachedLangs[$code] ) ) { wfDebug( __METHOD__ . "($code): forced reload\n" ); + return true; } $deps = $this->store->get( $code, 'deps' ); $keys = $this->store->get( $code, 'list' ); $preload = $this->store->get( $code, 'preload' ); - // Different keys may expire separately, at least in LCStore_Accel + // Different keys may expire separately for some stores if ( $deps === null || $keys === null || $preload === null ) { wfDebug( __METHOD__ . "($code): cache missing, need to make one\n" ); + return true; } @@ -411,6 +416,7 @@ class LocalisationCache { if ( !$dep instanceof CacheDependency || $dep->isExpired() ) { wfDebug( __METHOD__ . "($code): cache for $code expired due to " . get_class( $dep ) . "\n" ); + return true; } } @@ -420,7 +426,7 @@ class LocalisationCache { /** * Initialise a language in this object. Rebuild the cache if necessary. - * @param $code + * @param string $code * @throws MWException */ protected function initLanguage( $code ) { @@ -433,18 +439,20 @@ class LocalisationCache { # If the code is of the wrong form for a Messages*.php file, do a shallow fallback if ( !Language::isValidBuiltInCode( $code ) ) { $this->initShallowFallback( $code, 'en' ); + return; } # Recache the data if necessary if ( !$this->manualRecache && $this->isExpired( $code ) ) { - if ( file_exists( Language::getMessagesFileName( $code ) ) ) { + if ( Language::isSupportedLanguage( $code ) ) { $this->recache( $code ); } elseif ( $code === 'en' ) { throw new MWException( 'MessagesEn.php is missing.' ); } else { $this->initShallowFallback( $code, 'en' ); } + return; } @@ -458,6 +466,7 @@ class LocalisationCache { 'Please run maintenance/rebuildLocalisationCache.php.' ); } $this->initShallowFallback( $code, 'en' ); + return; } else { throw new MWException( 'Invalid or missing localisation cache.' ); @@ -478,8 +487,8 @@ class LocalisationCache { /** * Create a fallback from one language to another, without creating a * complete persistent cache. - * @param $primaryCode - * @param $fallbackCode + * @param string $primaryCode + * @param string $fallbackCode */ public function initShallowFallback( $primaryCode, $fallbackCode ) { $this->data[$primaryCode] =& $this->data[$fallbackCode]; @@ -490,17 +499,23 @@ class LocalisationCache { /** * Read a PHP file containing localisation data. - * @param $_fileName - * @param $_fileType + * @param string $_fileName + * @param string $_fileType * @throws MWException * @return array */ protected function readPHPFile( $_fileName, $_fileType ) { wfProfileIn( __METHOD__ ); // Disable APC caching + wfSuppressWarnings(); $_apcEnabled = ini_set( 'apc.cache_by_default', '0' ); + wfRestoreWarnings(); + include $_fileName; + + wfSuppressWarnings(); ini_set( 'apc.cache_by_default', $_apcEnabled ); + wfRestoreWarnings(); if ( $_fileType == 'core' || $_fileType == 'extension' ) { $data = compact( self::$allKeys ); @@ -511,12 +526,57 @@ class LocalisationCache { throw new MWException( __METHOD__ . ": Invalid file type: $_fileType" ); } wfProfileOut( __METHOD__ ); + return $data; } /** + * Read a JSON file containing localisation messages. + * @param string $fileName Name of file to read + * @throws MWException If there is a syntax error in the JSON file + * @return array Array with a 'messages' key, or empty array if the file doesn't exist + */ + public function readJSONFile( $fileName ) { + wfProfileIn( __METHOD__ ); + + if ( !is_readable( $fileName ) ) { + wfProfileOut( __METHOD__ ); + + return array(); + } + + $json = file_get_contents( $fileName ); + if ( $json === false ) { + wfProfileOut( __METHOD__ ); + + return array(); + } + + $data = FormatJson::decode( $json, true ); + if ( $data === null ) { + wfProfileOut( __METHOD__ ); + + throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" ); + } + + // Remove keys starting with '@', they're reserved for metadata and non-message data + foreach ( $data as $key => $unused ) { + if ( $key === '' || $key[0] === '@' ) { + unset( $data[$key] ); + } + } + + wfProfileOut( __METHOD__ ); + + // The JSON format only supports messages, none of the other variables, so wrap the data + return array( 'messages' => $data ); + } + + /** * Get the compiled plural rules for a given language from the XML files. * @since 1.20 + * @param string $code + * @return array|null */ public function getCompiledPluralRules( $code ) { $rules = $this->getPluralRules( $code ); @@ -526,9 +586,11 @@ class LocalisationCache { try { $compiledRules = CLDRPluralRuleEvaluator::compile( $rules ); } catch ( CLDRPluralRuleError $e ) { - wfDebugLog( 'l10n', $e->getMessage() . "\n" ); + wfDebugLog( 'l10n', $e->getMessage() ); + return array(); } + return $compiledRules; } @@ -536,6 +598,8 @@ class LocalisationCache { * Get the plural rules for a given language from the XML files. * Cached. * @since 1.20 + * @param string $code + * @return array|null */ public function getPluralRules( $code ) { if ( $this->pluralRules === null ) { @@ -552,6 +616,8 @@ class LocalisationCache { * Get the plural rule types for a given language from the XML files. * Cached. * @since 1.22 + * @param string $code + * @return array|null */ public function getPluralRuleTypes( $code ) { if ( $this->pluralRuleTypes === null ) { @@ -582,6 +648,8 @@ class LocalisationCache { /** * Load a plural XML file with the given filename, compile the relevant * rules, and save the compiled rules in a process-local cache. + * + * @param string $fileName */ protected function loadPluralFile( $fileName ) { $doc = new DOMDocument; @@ -612,20 +680,24 @@ class LocalisationCache { * Read the data from the source files for a given language, and register * the relevant dependencies in the $deps array. If the localisation * exists, the data array is returned, otherwise false is returned. + * + * @param string $code + * @param array $deps + * @return array */ protected function readSourceFilesAndRegisterDeps( $code, &$deps ) { global $IP; wfProfileIn( __METHOD__ ); + // This reads in the PHP i18n file with non-messages l10n data $fileName = Language::getMessagesFileName( $code ); if ( !file_exists( $fileName ) ) { - wfProfileOut( __METHOD__ ); - return false; + $data = array(); + } else { + $deps[] = new FileDependency( $fileName ); + $data = $this->readPHPFile( $fileName, 'core' ); } - $deps[] = new FileDependency( $fileName ); - $data = $this->readPHPFile( $fileName, 'core' ); - # Load CLDR plural rules for JavaScript $data['pluralRules'] = $this->getPluralRules( $code ); # And for PHP @@ -637,15 +709,16 @@ class LocalisationCache { $deps['plurals-mw'] = new FileDependency( "$IP/languages/data/plurals-mediawiki.xml" ); wfProfileOut( __METHOD__ ); + return $data; } /** * Merge two localisation values, a primary and a fallback, overwriting the * primary value in place. - * @param $key - * @param $value - * @param $fallbackValue + * @param string $key + * @param mixed $value + * @param mixed $fallbackValue */ protected function mergeItem( $key, &$value, $fallbackValue ) { if ( !is_null( $value ) ) { @@ -674,8 +747,8 @@ class LocalisationCache { } /** - * @param $value - * @param $fallbackValue + * @param mixed $value + * @param mixed $fallbackValue */ protected function mergeMagicWords( &$value, $fallbackValue ) { foreach ( $fallbackValue as $magicName => $fallbackInfo ) { @@ -698,10 +771,10 @@ class LocalisationCache { * * Returns true if any data from the extension array was used, false * otherwise. - * @param $codeSequence - * @param $key - * @param $value - * @param $fallbackValue + * @param array $codeSequence + * @param string $key + * @param mixed $value + * @param mixed $fallbackValue * @return bool */ protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) { @@ -719,11 +792,11 @@ class LocalisationCache { /** * Load localisation data for a given language for both core and extensions * and save it to the persistent cache store and the process cache - * @param $code + * @param string $code * @throws MWException */ public function recache( $code ) { - global $wgExtensionMessagesFiles; + global $wgExtensionMessagesFiles, $wgMessagesDirs; wfProfileIn( __METHOD__ ); if ( !$code ) { @@ -751,7 +824,6 @@ class LocalisationCache { foreach ( $data as $key => $value ) { $this->mergeItem( $key, $coreData[$key], $value ); } - } # Fill in the fallback if it's not there already @@ -768,43 +840,31 @@ class LocalisationCache { if ( $coreData['fallbackSequence'][$len - 1] !== 'en' ) { $coreData['fallbackSequence'][] = 'en'; } + } - # Load the fallback localisation item by item and merge it - foreach ( $coreData['fallbackSequence'] as $fbCode ) { - # Load the secondary localisation from the source file to - # avoid infinite cycles on cyclic fallbacks - $fbData = $this->readSourceFilesAndRegisterDeps( $fbCode, $deps ); - if ( $fbData === false ) { - continue; - } + $codeSequence = array_merge( array( $code ), $coreData['fallbackSequence'] ); - foreach ( self::$allKeys as $key ) { - if ( !isset( $fbData[$key] ) ) { - continue; - } + wfProfileIn( __METHOD__ . '-fallbacks' ); - if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) { - $this->mergeItem( $key, $coreData[$key], $fbData[$key] ); - } - } + # Load non-JSON localisation data for extensions + $extensionData = array_combine( + $codeSequence, + array_fill( 0, count( $codeSequence ), $initialData ) ); + foreach ( $wgExtensionMessagesFiles as $extension => $fileName ) { + if ( isset( $wgMessagesDirs[$extension] ) ) { + # This extension has JSON message data; skip the PHP shim + continue; } - } - $codeSequence = array_merge( array( $code ), $coreData['fallbackSequence'] ); - - # Load the extension localisations - # This is done after the core because we know the fallback sequence now. - # But it has a higher precedence for merging so that we can support things - # like site-specific message overrides. - wfProfileIn( __METHOD__ . '-extensions' ); - $allData = $initialData; - foreach ( $wgExtensionMessagesFiles as $fileName ) { $data = $this->readPHPFile( $fileName, 'extension' ); $used = false; foreach ( $data as $key => $item ) { - if ( $this->mergeExtensionItem( $codeSequence, $key, $allData[$key], $item ) ) { - $used = true; + foreach ( $codeSequence as $csCode ) { + if ( isset( $item[$csCode] ) ) { + $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] ); + $used = true; + } } } @@ -813,15 +873,81 @@ class LocalisationCache { } } - # Merge core data into extension data - foreach ( $coreData as $key => $item ) { - $this->mergeItem( $key, $allData[$key], $item ); + # Load the localisation data for each fallback, then merge it into the full array + $allData = $initialData; + foreach ( $codeSequence as $csCode ) { + $csData = $initialData; + + # Load core messages and the extension localisations. + foreach ( $wgMessagesDirs as $dirs ) { + foreach ( (array)$dirs as $dir ) { + $fileName = "$dir/$csCode.json"; + $data = $this->readJSONFile( $fileName ); + + foreach ( $data as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } + + $deps[] = new FileDependency( $fileName ); + } + } + + # Merge non-JSON extension data + if ( isset( $extensionData[$csCode] ) ) { + foreach ( $extensionData[$csCode] as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } + } + + if ( $csCode === $code ) { + # Merge core data into extension data + foreach ( $coreData as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } + } else { + # Load the secondary localisation from the source file to + # avoid infinite cycles on cyclic fallbacks + $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps ); + if ( $fbData !== false ) { + # Only merge the keys that make sense to merge + foreach ( self::$allKeys as $key ) { + if ( !isset( $fbData[$key] ) ) { + continue; + } + + if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) { + $this->mergeItem( $key, $csData[$key], $fbData[$key] ); + } + } + } + } + + # Allow extensions an opportunity to adjust the data for this + # fallback + wfRunHooks( 'LocalisationCacheRecacheFallback', array( $this, $csCode, &$csData ) ); + + # Merge the data for this fallback into the final array + if ( $csCode === $code ) { + $allData = $csData; + } else { + foreach ( self::$allKeys as $key ) { + if ( !isset( $csData[$key] ) ) { + continue; + } + + if ( is_null( $allData[$key] ) || $this->isMergeableKey( $key ) ) { + $this->mergeItem( $key, $allData[$key], $csData[$key] ); + } + } + } } - wfProfileOut( __METHOD__ . '-extensions' ); + + wfProfileOut( __METHOD__ . '-fallbacks' ); # Add cache dependencies for any referenced globals $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' ); - $deps['version'] = new ConstantDependency( 'MW_LC_VERSION' ); + $deps['wgMessagesDirs'] = new GlobalDependency( 'wgMessagesDirs' ); + $deps['version'] = new ConstantDependency( 'LocalisationCache::VERSION' ); # Add dependencies to the cache entry $allData['deps'] = $deps; @@ -854,7 +980,8 @@ class LocalisationCache { $allData['list'][$key] = array_keys( $allData[$key] ); } # Run hooks - wfRunHooks( 'LocalisationCacheRecache', array( $this, $code, &$allData ) ); + $purgeBlobs = true; + wfRunHooks( 'LocalisationCacheRecache', array( $this, $code, &$allData, &$purgeBlobs ) ); if ( is_null( $allData['namespaceNames'] ) ) { wfProfileOut( __METHOD__ ); @@ -889,8 +1016,8 @@ class LocalisationCache { # Clear out the MessageBlobStore # HACK: If using a null (i.e. disabled) storage backend, we # can't write to the MessageBlobStore either - if ( !$this->store instanceof LCStore_Null ) { - MessageBlobStore::clear(); + if ( $purgeBlobs && !$this->store instanceof LCStoreNull ) { + MessageBlobStore::getInstance()->clear(); } wfProfileOut( __METHOD__ ); @@ -901,7 +1028,7 @@ class LocalisationCache { * * The preload item will be loaded automatically, improving performance * for the commonly-requested items it contains. - * @param $data + * @param array $data * @return array */ protected function buildPreload( $data ) { @@ -925,7 +1052,7 @@ class LocalisationCache { /** * Unload the data for a given language from the object cache. * Reduces memory usage. - * @param $code + * @param string $code */ public function unload( $code ) { unset( $this->data[$code] ); @@ -954,28 +1081,9 @@ class LocalisationCache { * Disable the storage backend */ public function disableBackend() { - $this->store = new LCStore_Null; + $this->store = new LCStoreNull; $this->manualRecache = false; } - - /** - * Return an array with initialised languages. - * - * @return array - */ - public function getInitialisedLanguages() { - return $this->initialisedLangs; - } - - /** - * Set initialised languages. - * - * @param array $languages Optional array of initialised languages. - */ - public function setInitialisedLanguages( $languages = array() ) { - $this->initialisedLangs = $languages; - } - } /** @@ -1024,68 +1132,19 @@ interface LCStore { } /** - * LCStore implementation which uses PHP accelerator to store data. - * This will work if one of XCache, WinCache or APC cacher is configured. - * (See ObjectCache.php) - */ -class LCStore_Accel implements LCStore { - var $currentLang; - var $keys; - - public function __construct() { - $this->cache = wfGetCache( CACHE_ACCEL ); - } - - public function get( $code, $key ) { - $k = wfMemcKey( 'l10n', $code, 'k', $key ); - $r = $this->cache->get( $k ); - return $r === false ? null : $r; - } - - public function startWrite( $code ) { - $k = wfMemcKey( 'l10n', $code, 'l' ); - $keys = $this->cache->get( $k ); - if ( $keys ) { - foreach ( $keys as $k ) { - $this->cache->delete( $k ); - } - } - $this->currentLang = $code; - $this->keys = array(); - } - - public function finishWrite() { - if ( $this->currentLang ) { - $k = wfMemcKey( 'l10n', $this->currentLang, 'l' ); - $this->cache->set( $k, array_keys( $this->keys ) ); - } - $this->currentLang = null; - $this->keys = array(); - } - - public function set( $key, $value ) { - if ( $this->currentLang ) { - $k = wfMemcKey( 'l10n', $this->currentLang, 'k', $key ); - $this->keys[$k] = true; - $this->cache->set( $k, $value ); - } - } -} - -/** * LCStore implementation which uses the standard DB functions to store data. * This will work on any MediaWiki installation. */ -class LCStore_DB implements LCStore { - var $currentLang; - var $writesDone = false; +class LCStoreDB implements LCStore { + private $currentLang; + private $writesDone = false; - /** - * @var DatabaseBase - */ - var $dbw; - var $batch; - var $readOnly = false; + /** @var DatabaseBase */ + private $dbw; + /** @var array */ + private $batch = array(); + + private $readOnly = false; public function get( $code, $key ) { if ( $this->writesDone ) { @@ -1096,7 +1155,7 @@ class LCStore_DB implements LCStore { $row = $db->selectRow( 'l10n_cache', array( 'lc_value' ), array( 'lc_lang' => $code, 'lc_key' => $key ), __METHOD__ ); if ( $row ) { - return unserialize( $row->lc_value ); + return unserialize( $db->decodeBlob( $row->lc_value ) ); } else { return null; } @@ -1105,25 +1164,11 @@ class LCStore_DB implements LCStore { public function startWrite( $code ) { if ( $this->readOnly ) { return; - } - - if ( !$code ) { + } elseif ( !$code ) { throw new MWException( __METHOD__ . ": Invalid language \"$code\"" ); } $this->dbw = wfGetDB( DB_MASTER ); - try { - $this->dbw->begin( __METHOD__ ); - $this->dbw->delete( 'l10n_cache', array( 'lc_lang' => $code ), __METHOD__ ); - } catch ( DBQueryError $e ) { - if ( $this->dbw->wasReadOnlyError() ) { - $this->readOnly = true; - $this->dbw->rollback( __METHOD__ ); - return; - } else { - throw $e; - } - } $this->currentLang = $code; $this->batch = array(); @@ -1132,37 +1177,42 @@ class LCStore_DB implements LCStore { public function finishWrite() { if ( $this->readOnly ) { return; + } elseif ( is_null( $this->currentLang ) ) { + throw new MWException( __CLASS__ . ': must call startWrite() before finishWrite()' ); } - if ( $this->batch ) { - $this->dbw->insert( 'l10n_cache', $this->batch, __METHOD__ ); + $this->dbw->begin( __METHOD__ ); + try { + $this->dbw->delete( 'l10n_cache', + array( 'lc_lang' => $this->currentLang ), __METHOD__ ); + foreach ( array_chunk( $this->batch, 500 ) as $rows ) { + $this->dbw->insert( 'l10n_cache', $rows, __METHOD__ ); + } + $this->writesDone = true; + } catch ( DBQueryError $e ) { + if ( $this->dbw->wasReadOnlyError() ) { + $this->readOnly = true; // just avoid site down time + } else { + throw $e; + } } - $this->dbw->commit( __METHOD__ ); + $this->currentLang = null; - $this->dbw = null; $this->batch = array(); - $this->writesDone = true; } public function set( $key, $value ) { if ( $this->readOnly ) { return; - } - - if ( is_null( $this->currentLang ) ) { - throw new MWException( __CLASS__ . ': must call startWrite() before calling set()' ); + } elseif ( is_null( $this->currentLang ) ) { + throw new MWException( __CLASS__ . ': must call startWrite() before set()' ); } $this->batch[] = array( 'lc_lang' => $this->currentLang, 'lc_key' => $key, - 'lc_value' => serialize( $value ) ); - - if ( count( $this->batch ) >= 100 ) { - $this->dbw->insert( 'l10n_cache', $this->batch, __METHOD__ ); - $this->batch = array(); - } + 'lc_value' => $this->dbw->encodeBlob( serialize( $value ) ) ); } } @@ -1178,8 +1228,18 @@ class LCStore_DB implements LCStore { * * See Cdb.php and http://cr.yp.to/cdb.html */ -class LCStore_CDB implements LCStore { - var $readers, $writer, $currentLang, $directory; +class LCStoreCDB implements LCStore { + /** @var CdbReader[] */ + private $readers; + + /** @var CdbWriter */ + private $writer; + + /** @var string Current language code */ + private $currentLang; + + /** @var bool|string Cache directory. False if not set */ + private $directory; function __construct( $conf = array() ) { global $wgCacheDirectory; @@ -1195,21 +1255,30 @@ class LCStore_CDB implements LCStore { if ( !isset( $this->readers[$code] ) ) { $fileName = $this->getFileName( $code ); - if ( !file_exists( $fileName ) ) { - $this->readers[$code] = false; - } else { - $this->readers[$code] = CdbReader::open( $fileName ); + $this->readers[$code] = false; + if ( file_exists( $fileName ) ) { + try { + $this->readers[$code] = CdbReader::open( $fileName ); + } catch ( CdbException $e ) { + wfDebug( __METHOD__ . ": unable to open cdb file for reading\n" ); + } } } if ( !$this->readers[$code] ) { return null; } else { - $value = $this->readers[$code]->get( $key ); - + $value = false; + try { + $value = $this->readers[$code]->get( $key ); + } catch ( CdbException $e ) { + wfDebug( __METHOD__ . ": CdbException caught, error message was " + . $e->getMessage() . "\n" ); + } if ( $value === false ) { return null; } + return unserialize( $value ); } } @@ -1227,13 +1296,21 @@ class LCStore_CDB implements LCStore { $this->readers[$code]->close(); } - $this->writer = CdbWriter::open( $this->getFileName( $code ) ); + try { + $this->writer = CdbWriter::open( $this->getFileName( $code ) ); + } catch ( CdbException $e ) { + throw new MWException( $e->getMessage() ); + } $this->currentLang = $code; } public function finishWrite() { // Close the writer - $this->writer->close(); + try { + $this->writer->close(); + } catch ( CdbException $e ) { + throw new MWException( $e->getMessage() ); + } $this->writer = null; unset( $this->readers[$this->currentLang] ); $this->currentLang = null; @@ -1243,13 +1320,18 @@ class LCStore_CDB implements LCStore { if ( is_null( $this->writer ) ) { throw new MWException( __CLASS__ . ': must call startWrite() before calling set()' ); } - $this->writer->set( $key, serialize( $value ) ); + try { + $this->writer->set( $key, serialize( $value ) ); + } catch ( CdbException $e ) { + throw new MWException( $e->getMessage() ); + } } protected function getFileName( $code ) { if ( strval( $code ) === '' || strpos( $code, '/' ) !== false ) { throw new MWException( __METHOD__ . ": Invalid language \"$code\"" ); } + return "{$this->directory}/l10n_cache-$code.cdb"; } } @@ -1257,42 +1339,47 @@ class LCStore_CDB implements LCStore { /** * Null store backend, used to avoid DB errors during install */ -class LCStore_Null implements LCStore { +class LCStoreNull implements LCStore { public function get( $code, $key ) { return null; } - public function startWrite( $code ) {} - public function finishWrite() {} - public function set( $key, $value ) {} + public function startWrite( $code ) { + } + + public function finishWrite() { + } + + public function set( $key, $value ) { + } } /** * A localisation cache optimised for loading large amounts of data for many * languages. Used by rebuildLocalisationCache.php. */ -class LocalisationCache_BulkLoad extends LocalisationCache { +class LocalisationCacheBulkLoad extends LocalisationCache { /** * A cache of the contents of data files. * Core files are serialized to avoid using ~1GB of RAM during a recache. */ - var $fileCache = array(); + private $fileCache = array(); /** * Most recently used languages. Uses the linked-list aspect of PHP hashtables * to keep the most recently used language codes at the end of the array, and * the language codes that are ready to be deleted at the beginning. */ - var $mruLangs = array(); + private $mruLangs = array(); /** * Maximum number of languages that may be loaded into $this->data */ - var $maxLoadedLangs = 10; + private $maxLoadedLangs = 10; /** - * @param $fileName - * @param $fileType + * @param string $fileName + * @param string $fileType * @return array|mixed */ protected function readPHPFile( $fileName, $fileType ) { @@ -1317,30 +1404,32 @@ class LocalisationCache_BulkLoad extends LocalisationCache { } /** - * @param $code - * @param $key + * @param string $code + * @param string $key * @return mixed */ public function getItem( $code, $key ) { unset( $this->mruLangs[$code] ); $this->mruLangs[$code] = true; + return parent::getItem( $code, $key ); } /** - * @param $code - * @param $key - * @param $subkey - * @return + * @param string $code + * @param string $key + * @param string $subkey + * @return mixed */ public function getSubitem( $code, $key, $subkey ) { unset( $this->mruLangs[$code] ); $this->mruLangs[$code] = true; + return parent::getSubitem( $code, $key, $subkey ); } /** - * @param $code + * @param string $code */ public function recache( $code ) { parent::recache( $code ); @@ -1350,7 +1439,7 @@ class LocalisationCache_BulkLoad extends LocalisationCache { } /** - * @param $code + * @param string $code */ public function unload( $code ) { unset( $this->mruLangs[$code] ); diff --git a/includes/cache/ProcessCacheLRU.php b/includes/cache/MapCacheLRU.php index 76c76f37..95e3af76 100644 --- a/includes/cache/ProcessCacheLRU.php +++ b/includes/cache/MapCacheLRU.php @@ -22,19 +22,22 @@ */ /** - * Handles per process caching of items + * Handles a simple LRU key/value map with a maximum number of entries + * + * Use ProcessCacheLRU if hierarchical purging is needed or objects can become stale + * + * @see ProcessCacheLRU * @ingroup Cache + * @since 1.23 */ -class ProcessCacheLRU { - /** @var Array */ - protected $cache = array(); // (key => prop => value) - /** @var Array */ - protected $cacheTimes = array(); // (key => prop => UNIX timestamp) +class MapCacheLRU { + /** @var array */ + protected $cache = array(); // (key => value) protected $maxCacheKeys; // integer; max entries /** - * @param $maxKeys integer Maximum number of entries allowed (min 1). + * @param int $maxKeys Maximum number of entries allowed (min 1). * @throws MWException When $maxCacheKeys is not an int or =< 0. */ public function __construct( $maxKeys ) { @@ -45,56 +48,47 @@ class ProcessCacheLRU { } /** - * Set a property field for a cache entry. + * Set a key/value pair. * This will prune the cache if it gets too large based on LRU. * If the item is already set, it will be pushed to the top of the cache. * - * @param $key string - * @param $prop string - * @param $value mixed + * @param string $key + * @param mixed $value * @return void */ - public function set( $key, $prop, $value ) { - if ( isset( $this->cache[$key] ) ) { + public function set( $key, $value ) { + if ( array_key_exists( $key, $this->cache ) ) { $this->ping( $key ); // push to top } elseif ( count( $this->cache ) >= $this->maxCacheKeys ) { reset( $this->cache ); $evictKey = key( $this->cache ); unset( $this->cache[$evictKey] ); - unset( $this->cacheTimes[$evictKey] ); } - $this->cache[$key][$prop] = $value; - $this->cacheTimes[$key][$prop] = time(); + $this->cache[$key] = $value; } /** - * Check if a property field exists for a cache entry. + * Check if a key exists * - * @param $key string - * @param $prop string - * @param $maxAge integer Ignore items older than this many seconds (since 1.21) + * @param string $key * @return bool */ - public function has( $key, $prop, $maxAge = 0 ) { - if ( isset( $this->cache[$key][$prop] ) ) { - return ( $maxAge <= 0 || ( time() - $this->cacheTimes[$key][$prop] ) <= $maxAge ); - } - return false; + public function has( $key ) { + return array_key_exists( $key, $this->cache ); } /** - * Get a property field for a cache entry. - * This returns null if the property is not set. + * Get the value for a key. + * This returns null if the key is not set. * If the item is already set, it will be pushed to the top of the cache. * - * @param $key string - * @param $prop string + * @param string $key * @return mixed */ - public function get( $key, $prop ) { - if ( isset( $this->cache[$key][$prop] ) ) { + public function get( $key ) { + if ( array_key_exists( $key, $this->cache ) ) { $this->ping( $key ); // push to top - return $this->cache[$key][$prop]; + return $this->cache[$key]; } else { return null; } @@ -103,17 +97,15 @@ class ProcessCacheLRU { /** * Clear one or several cache entries, or all cache entries * - * @param $keys string|Array + * @param string|array $keys * @return void */ public function clear( $keys = null ) { if ( $keys === null ) { $this->cache = array(); - $this->cacheTimes = array(); } else { foreach ( (array)$keys as $key ) { unset( $this->cache[$key] ); - unset( $this->cacheTimes[$key] ); } } } @@ -121,7 +113,7 @@ class ProcessCacheLRU { /** * Push an entry to the top of the cache * - * @param $key string + * @param string $key */ protected function ping( $key ) { $item = $this->cache[$key]; diff --git a/includes/cache/MessageCache.php b/includes/cache/MessageCache.php index a92c87f4..1ef7cc58 100644 --- a/includes/cache/MessageCache.php +++ b/includes/cache/MessageCache.php @@ -110,6 +110,7 @@ class MessageCache { $wgMsgCacheExpiry ); } + return self::$instance; } @@ -123,7 +124,7 @@ class MessageCache { } /** - * @param ObjectCache $memCached A cache instance. If none, fall back to CACHE_NONE. + * @param BagOStuff $memCached A cache instance. If none, fall back to CACHE_NONE. * @param bool $useDB * @param int $expiry Lifetime for cache. @see $mExpiry. */ @@ -147,14 +148,15 @@ class MessageCache { $this->mParserOptions = new ParserOptions; $this->mParserOptions->setEditSection( false ); } + return $this->mParserOptions; } /** * Try to load the cache from a local file. * - * @param string $hash the hash of contents, to check validity. - * @param Mixed $code Optional language code, see documenation of load(). + * @param string $hash The hash of contents, to check validity. + * @param string $code Optional language code, see documenation of load(). * @return array The cache array */ function getLocalCache( $hash, $code ) { @@ -179,15 +181,20 @@ class MessageCache { $serialized .= fread( $file, 100000 ); } fclose( $file ); + return unserialize( $serialized ); } else { fclose( $file ); + return false; // Wrong hash } } /** * Save the cache to a local file. + * @param string $serialized + * @param string $hash + * @param string $code */ function saveToLocal( $serialized, $hash, $code ) { global $wgCacheDirectory; @@ -201,6 +208,7 @@ class MessageCache { if ( !$file ) { wfDebug( "Unable to open local cache file for writing\n" ); + return; } @@ -227,7 +235,7 @@ class MessageCache { * or false if populating empty cache fails. Also returns true if MessageCache * is disabled. * - * @param bool|String $code Language to which load messages + * @param bool|string $code Language to which load messages * @throws MWException * @return bool */ @@ -253,6 +261,7 @@ class MessageCache { wfDebug( __METHOD__ . ": disabled\n" ); $shownDisabled = true; } + return true; } @@ -415,6 +424,7 @@ class MessageCache { $info = implode( ', ', $where ); wfDebug( __METHOD__ . ": Loading $code... $info\n" ); wfProfileOut( __METHOD__ ); + return $success; } @@ -502,6 +512,7 @@ class MessageCache { $cache['VERSION'] = MSG_CACHE_VERSION; $cache['EXPIRY'] = wfTimestamp( TS_MW, time() + $this->mExpiry ); wfProfileOut( __METHOD__ ); + return $cache; } @@ -517,6 +528,7 @@ class MessageCache { if ( $this->mDisable ) { wfProfileOut( __METHOD__ ); + return; } @@ -561,7 +573,7 @@ class MessageCache { // Update the message in the message blob store global $wgContLang; - MessageBlobStore::updateMessage( $wgContLang->lcfirst( $msg ) ); + MessageBlobStore::getInstance()->updateMessage( $wgContLang->lcfirst( $msg ) ); wfRunHooks( 'MessageCacheReplace', array( $title, $text ) ); @@ -571,7 +583,7 @@ class MessageCache { /** * Is the given cache array expired due to time passing or a version change? * - * @param $cache + * @param array $cache * @return bool */ protected function isCacheExpired( $cache ) { @@ -584,6 +596,7 @@ class MessageCache { if ( wfTimestampNow() >= $cache['EXPIRY'] ) { return true; } + return false; } @@ -617,6 +630,7 @@ class MessageCache { } wfProfileOut( __METHOD__ ); + return $success; } @@ -627,7 +641,7 @@ class MessageCache { * a timeout of MessageCache::MSG_LOCK_TIMEOUT. * * @param string $key - * @return Boolean: success + * @return bool Success */ function lock( $key ) { $lockKey = $key . ':lock'; @@ -676,20 +690,20 @@ class MessageCache { * * Fallbacks will be just that: fallbacks. A fallback language will never be reached if * the message is available *anywhere* in the language for which it is a fallback. * - * @param string $key the message key + * @param string $key The message key * @param bool $useDB If true, look for the message in the DB, false - * to use only the compiled l10n cache. + * to use only the compiled l10n cache. * @param bool|string|object $langcode Code of the language to get the message for. - * - If string and a valid code, will create a standard language object - * - If string but not a valid code, will create a basic language object - * - If boolean and false, create object from the current users language - * - If boolean and true, create object from the wikis content language - * - If language object, use it as given - * @param bool $isFullKey specifies whether $key is a two part key - * "msg/lang". + * - If string and a valid code, will create a standard language object + * - If string but not a valid code, will create a basic language object + * - If boolean and false, create object from the current users language + * - If boolean and true, create object from the wikis content language + * - If language object, use it as given + * @param bool $isFullKey Specifies whether $key is a two part key "msg/lang". * - * @throws MWException when given an invalid key - * @return string|bool False if the message doesn't exist, otherwise the message (which can be empty) + * @throws MWException When given an invalid key + * @return string|bool False if the message doesn't exist, otherwise the + * message (which can be empty) */ function get( $key, $useDB = true, $langcode = true, $isFullKey = false ) { global $wgContLang; @@ -716,17 +730,28 @@ class MessageCache { // Normalise title-case input (with some inlining) $lckey = strtr( $key, ' ', '_' ); - if ( ord( $key ) < 128 ) { + if ( ord( $lckey ) < 128 ) { $lckey[0] = strtolower( $lckey[0] ); - $uckey = ucfirst( $lckey ); } else { $lckey = $wgContLang->lcfirst( $lckey ); + } + + wfRunHooks( 'MessageCache::get', array( &$lckey ) ); + + if ( ord( $lckey ) < 128 ) { + $uckey = ucfirst( $lckey ); + } else { $uckey = $wgContLang->ucfirst( $lckey ); } // Loop through each language in the fallback list until we find something useful $lang = wfGetLangObj( $langcode ); - $message = $this->getMessageFromFallbackChain( $lang, $lckey, $uckey, !$this->mDisable && $useDB ); + $message = $this->getMessageFromFallbackChain( + $lang, + $lckey, + $uckey, + !$this->mDisable && $useDB + ); // If we still have no message, maybe the key was in fact a full key so try that if ( $message === false ) { @@ -804,7 +829,8 @@ class MessageCache { return $message; } - list( $fallbackChain, $siteFallbackChain ) = Language::getFallbacksIncludingSiteLanguage( $langcode ); + list( $fallbackChain, $siteFallbackChain ) = + Language::getFallbacksIncludingSiteLanguage( $langcode ); // Next try checking the database for all of the fallback languages of the requested language. if ( $useDB ) { @@ -897,11 +923,13 @@ class MessageCache { if ( $entry ) { if ( substr( $entry, 0, 1 ) === ' ' ) { $this->mCache[$code][$title] = $entry; + // The message exists, so make sure a string // is returned. return (string)substr( $entry, 1 ); } elseif ( $entry === '!NONEXISTENT' ) { $this->mCache[$code][$title] = '!NONEXISTENT'; + return false; } else { # Corrupt/obsolete entry, delete it @@ -983,6 +1011,7 @@ class MessageCache { $this->mInParser = false; $popts->setUserLang( $userlang ); } + return $message; } @@ -996,13 +1025,14 @@ class MessageCache { $wgParser->firstCallInit(); # Clone it and store it $class = $wgParserConf['class']; - if ( $class == 'Parser_DiffTest' ) { + if ( $class == 'ParserDiffTest' ) { # Uncloneable $this->mParser = new $class( $wgParserConf ); } else { $this->mParser = clone $wgParser; } } + return $this->mParser; } @@ -1043,6 +1073,7 @@ class MessageCache { $this->mInParser = false; wfProfileOut( __METHOD__ ); + return $res; } @@ -1069,7 +1100,7 @@ class MessageCache { } /** - * @param $key + * @param string $key * @return array */ public function figureMessage( $key ) { @@ -1085,6 +1116,7 @@ class MessageCache { } $message = implode( '/', $pieces ); + return array( $message, $lang ); } @@ -1094,7 +1126,7 @@ class MessageCache { * for which MediaWiki:msgkey exists. If $code is another language code, this * will ONLY return message keys for which MediaWiki:msgkey/$code exists. * @param string $code Language code - * @return array of message keys (strings) + * @return array Array of message keys (strings) */ public function getAllMessageKeys( $code ) { global $wgContLang; @@ -1109,6 +1141,7 @@ class MessageCache { unset( $cache['EXPIRY'] ); // Remove any !NONEXISTENT keys $cache = array_diff( $cache, array( '!NONEXISTENT' ) ); + // Keys may appear with a capital first letter. lcfirst them. return array_map( array( $wgContLang, 'lcfirst' ), array_keys( $cache ) ); } diff --git a/includes/cache/ObjectFileCache.php b/includes/cache/ObjectFileCache.php index ed1e49a6..c7ef0443 100644 --- a/includes/cache/ObjectFileCache.php +++ b/includes/cache/ObjectFileCache.php @@ -29,8 +29,8 @@ class ObjectFileCache extends FileCacheBase { /** * Construct an ObjectFileCache from a key and a type - * @param $key string - * @param $type string + * @param string $key + * @param string $type * @return ObjectFileCache */ public static function newFromKey( $key, $type ) { diff --git a/includes/cache/ResourceFileCache.php b/includes/cache/ResourceFileCache.php index 2ad7b853..55da52c5 100644 --- a/includes/cache/ResourceFileCache.php +++ b/includes/cache/ResourceFileCache.php @@ -34,7 +34,7 @@ class ResourceFileCache extends FileCacheBase { /** * Construct an ResourceFileCache from a context - * @param $context ResourceLoaderContext + * @param ResourceLoaderContext $context * @return ResourceFileCache */ public static function newFromContext( ResourceLoaderContext $context ) { @@ -58,7 +58,7 @@ class ResourceFileCache extends FileCacheBase { /** * Check if an RL request can be cached. * Caller is responsible for checking if any modules are private. - * @param $context ResourceLoaderContext + * @param ResourceLoaderContext $context * @return bool */ public static function useFileCache( ResourceLoaderContext $context ) { @@ -80,8 +80,10 @@ class ResourceFileCache extends FileCacheBase { } elseif ( $query === 'debug' && $val === 'false' ) { continue; } + return false; } + return true; // cacheable } @@ -104,6 +106,7 @@ class ResourceFileCache extends FileCacheBase { $this->getMissesRecent() >= self::MISS_THRESHOLD // many misses ); } + return $this->mCacheWorthy; } } diff --git a/includes/cache/SquidUpdate.php b/includes/cache/SquidUpdate.php deleted file mode 100644 index 71afeba9..00000000 --- a/includes/cache/SquidUpdate.php +++ /dev/null @@ -1,300 +0,0 @@ -<?php -/** - * Squid cache purging. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * http://www.gnu.org/copyleft/gpl.html - * - * @file - * @ingroup Cache - */ - -/** - * Handles purging appropriate Squid URLs given a title (or titles) - * @ingroup Cache - */ -class SquidUpdate { - - /** - * Collection of URLs to purge. - * @var array - */ - protected $urlArr; - - /** - * @param array $urlArr Collection of URLs to purge - * @param bool|int $maxTitles Maximum number of unique URLs to purge - */ - public function __construct( $urlArr = array(), $maxTitles = false ) { - global $wgMaxSquidPurgeTitles; - if ( $maxTitles === false ) { - $maxTitles = $wgMaxSquidPurgeTitles; - } - - // Remove duplicate URLs from list - $urlArr = array_unique( $urlArr ); - if ( count( $urlArr ) > $maxTitles ) { - // Truncate to desired maximum URL count - $urlArr = array_slice( $urlArr, 0, $maxTitles ); - } - $this->urlArr = $urlArr; - } - - /** - * Create a SquidUpdate from the given Title object. - * - * The resulting SquidUpdate will purge the given Title's URLs as well as - * the pages that link to it. Capped at $wgMaxSquidPurgeTitles total URLs. - * - * @param Title $title - * @return SquidUpdate - */ - public static function newFromLinksTo( Title $title ) { - global $wgMaxSquidPurgeTitles; - wfProfileIn( __METHOD__ ); - - # Get a list of URLs linking to this page - $dbr = wfGetDB( DB_SLAVE ); - $res = $dbr->select( array( 'links', 'page' ), - array( 'page_namespace', 'page_title' ), - array( - 'pl_namespace' => $title->getNamespace(), - 'pl_title' => $title->getDBkey(), - 'pl_from=page_id' ), - __METHOD__ ); - $blurlArr = $title->getSquidURLs(); - if ( $res->numRows() <= $wgMaxSquidPurgeTitles ) { - foreach ( $res as $BL ) { - $tobj = Title::makeTitle( $BL->page_namespace, $BL->page_title ); - $blurlArr[] = $tobj->getInternalURL(); - } - } - - wfProfileOut( __METHOD__ ); - return new SquidUpdate( $blurlArr ); - } - - /** - * Create a SquidUpdate from an array of Title objects, or a TitleArray object - * - * @param array $titles - * @param array $urlArr - * @return SquidUpdate - */ - public static function newFromTitles( $titles, $urlArr = array() ) { - global $wgMaxSquidPurgeTitles; - $i = 0; - foreach ( $titles as $title ) { - $urlArr[] = $title->getInternalURL(); - if ( $i++ > $wgMaxSquidPurgeTitles ) { - break; - } - } - return new SquidUpdate( $urlArr ); - } - - /** - * @param Title $title - * @return SquidUpdate - */ - public static function newSimplePurge( Title $title ) { - $urlArr = $title->getSquidURLs(); - return new SquidUpdate( $urlArr ); - } - - /** - * Purges the list of URLs passed to the constructor. - */ - public function doUpdate() { - self::purge( $this->urlArr ); - } - - /** - * Purges a list of Squids defined in $wgSquidServers. - * $urlArr should contain the full URLs to purge as values - * (example: $urlArr[] = 'http://my.host/something') - * XXX report broken Squids per mail or log - * - * @param array $urlArr List of full URLs to purge - */ - public static function purge( $urlArr ) { - global $wgSquidServers, $wgHTCPRouting; - - if ( !$urlArr ) { - return; - } - - wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) . "\n" ); - - if ( $wgHTCPRouting ) { - self::HTCPPurge( $urlArr ); - } - - wfProfileIn( __METHOD__ ); - - // Remove duplicate URLs - $urlArr = array_unique( $urlArr ); - // Maximum number of parallel connections per squid - $maxSocketsPerSquid = 8; - // Number of requests to send per socket - // 400 seems to be a good tradeoff, opening a socket takes a while - $urlsPerSocket = 400; - $socketsPerSquid = ceil( count( $urlArr ) / $urlsPerSocket ); - if ( $socketsPerSquid > $maxSocketsPerSquid ) { - $socketsPerSquid = $maxSocketsPerSquid; - } - - $pool = new SquidPurgeClientPool; - $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSquid ) ); - foreach ( $wgSquidServers as $server ) { - foreach ( $chunks as $chunk ) { - $client = new SquidPurgeClient( $server ); - foreach ( $chunk as $url ) { - $client->queuePurge( $url ); - } - $pool->addClient( $client ); - } - } - $pool->run(); - - wfProfileOut( __METHOD__ ); - } - - /** - * Send Hyper Text Caching Protocol (HTCP) CLR requests. - * - * @throws MWException - * @param array $urlArr Collection of URLs to purge - */ - public static function HTCPPurge( $urlArr ) { - global $wgHTCPRouting, $wgHTCPMulticastTTL; - wfProfileIn( __METHOD__ ); - - // HTCP CLR operation - $htcpOpCLR = 4; - - // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h) - if ( !defined( "IPPROTO_IP" ) ) { - define( "IPPROTO_IP", 0 ); - define( "IP_MULTICAST_LOOP", 34 ); - define( "IP_MULTICAST_TTL", 33 ); - } - - // pfsockopen doesn't work because we need set_sock_opt - $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP ); - if ( ! $conn ) { - $errstr = socket_strerror( socket_last_error() ); - wfDebugLog( 'squid', __METHOD__ . - ": Error opening UDP socket: $errstr\n" ); - wfProfileOut( __METHOD__ ); - return; - } - - // Set socket options - socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 ); - if ( $wgHTCPMulticastTTL != 1 ) { - // Set multicast time to live (hop count) option on socket - socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL, - $wgHTCPMulticastTTL ); - } - - // Remove duplicate URLs from collection - $urlArr = array_unique( $urlArr ); - foreach ( $urlArr as $url ) { - if ( !is_string( $url ) ) { - wfProfileOut( __METHOD__ ); - throw new MWException( 'Bad purge URL' ); - } - $url = self::expand( $url ); - $conf = self::getRuleForURL( $url, $wgHTCPRouting ); - if ( !$conf ) { - wfDebugLog( 'squid', __METHOD__ . - "No HTCP rule configured for URL {$url} , skipping\n" ); - continue; - } - - if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) { - // Normalize single entries - $conf = array( $conf ); - } - foreach ( $conf as $subconf ) { - if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) { - wfProfileOut( __METHOD__ ); - throw new MWException( "Invalid HTCP rule for URL $url\n" ); - } - } - - // Construct a minimal HTCP request diagram - // as per RFC 2756 - // Opcode 'CLR', no response desired, no auth - $htcpTransID = rand(); - - $htcpSpecifier = pack( 'na4na*na8n', - 4, 'HEAD', strlen( $url ), $url, - 8, 'HTTP/1.0', 0 ); - - $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier ); - $htcpLen = 4 + $htcpDataLen + 2; - - // Note! Squid gets the bit order of the first - // word wrong, wrt the RFC. Apparently no other - // implementation exists, so adapt to Squid - $htcpPacket = pack( 'nxxnCxNxxa*n', - $htcpLen, $htcpDataLen, $htcpOpCLR, - $htcpTransID, $htcpSpecifier, 2 ); - - wfDebugLog( 'squid', __METHOD__ . - "Purging URL $url via HTCP\n" ); - foreach ( $conf as $subconf ) { - socket_sendto( $conn, $htcpPacket, $htcpLen, 0, - $subconf['host'], $subconf['port'] ); - } - } - wfProfileOut( __METHOD__ ); - } - - /** - * Expand local URLs to fully-qualified URLs using the internal protocol - * and host defined in $wgInternalServer. Input that's already fully- - * qualified will be passed through unchanged. - * - * This is used to generate purge URLs that may be either local to the - * main wiki or include a non-native host, such as images hosted on a - * second internal server. - * - * Client functions should not need to call this. - * - * @param string $url - * @return string - */ - public static function expand( $url ) { - return wfExpandUrl( $url, PROTO_INTERNAL ); - } - - /** - * Find the HTCP routing rule to use for a given URL. - * @param string $url URL to match - * @param array $rules Array of rules, see $wgHTCPRouting for format and behavior - * @return mixed Element of $rules that matched, or false if nothing matched - */ - private static function getRuleForURL( $url, $rules ) { - foreach ( $rules as $regex => $routing ) { - if ( $regex === '' || preg_match( $regex, $url ) ) { - return $routing; - } - } - return false; - } -} diff --git a/includes/cache/UserCache.php b/includes/cache/UserCache.php index 6085f586..7f36f5a6 100644 --- a/includes/cache/UserCache.php +++ b/includes/cache/UserCache.php @@ -36,23 +36,26 @@ class UserCache { if ( $instance === null ) { $instance = new self(); } + return $instance; } - protected function __construct() {} + protected function __construct() { + } /** * Get a property of a user based on their user ID * - * @param $userId integer User ID + * @param int $userId User ID * @param string $prop User property - * @return mixed The property or false if the user does not exist + * @return mixed|bool The property or false if the user does not exist */ public function getProp( $userId, $prop ) { if ( !isset( $this->cache[$userId][$prop] ) ) { wfDebug( __METHOD__ . ": querying DB for prop '$prop' for user ID '$userId'.\n" ); $this->doQuery( array( $userId ) ); // cache miss } + return isset( $this->cache[$userId][$prop] ) ? $this->cache[$userId][$prop] : false; // user does not exist? @@ -61,8 +64,9 @@ class UserCache { /** * Get the name of a user or return $ip if the user ID is 0 * - * @param integer $userId + * @param int $userId * @param string $ip + * @return string * @since 1.22 */ public function getUserName( $userId, $ip ) { @@ -73,7 +77,7 @@ class UserCache { * Preloads user names for given list of users. * @param array $userIds List of user IDs * @param array $options Option flags; include 'userpage' and 'usertalk' - * @param string $caller the calling method + * @param string $caller The calling method */ public function doQuery( array $userIds, $options = array(), $caller = '' ) { wfProfileIn( __METHOD__ ); @@ -136,7 +140,7 @@ class UserCache { /** * Check if a cache type is in $options and was not loaded for this user * - * @param $uid integer user ID + * @param int $uid User ID * @param string $type Cache type * @param array $options Requested cache types * @return bool diff --git a/includes/cache/bloom/BloomCache.php b/includes/cache/bloom/BloomCache.php new file mode 100644 index 00000000..236db954 --- /dev/null +++ b/includes/cache/bloom/BloomCache.php @@ -0,0 +1,323 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * Persistent bloom filter used to avoid expensive lookups + * + * @since 1.24 + */ +abstract class BloomCache { + /** @var string Unique ID for key namespacing */ + protected $cacheID; + + /** @var array Map of (id => BloomCache) */ + protected static $instances = array(); + + /** + * @param string $id + * @return BloomCache + */ + final public static function get( $id ) { + global $wgBloomFilterStores; + + if ( !isset( self::$instances[$id] ) ) { + if ( isset( $wgBloomFilterStores[$id] ) ) { + $class = $wgBloomFilterStores[$id]['class']; + self::$instances[$id] = new $class( $wgBloomFilterStores[$id] ); + } else { + wfDebug( "No bloom filter store '$id'; using EmptyBloomCache." ); + return new EmptyBloomCache( array() ); + } + } + + return self::$instances[$id]; + } + + /** + * Create a new bloom cache instance from configuration. + * This should only be called from within BloomCache. + * + * @param array $config Parameters include: + * - cacheID : Prefix to all bloom filter names that is unique to this cache. + * It should only consist of alphanumberic, '-', and '_' characters. + * This ID is what avoids collisions if multiple logical caches + * use the same storage system, so this should be set carefully. + */ + public function __construct( array $config ) { + $this->cacheID = $config['cacheId']; + if ( !preg_match( '!^[a-zA-Z0-9-_]{1,32}$!', $this->cacheID ) ) { + throw new MWException( "Cache ID '{$this->cacheID}' is invalid." ); + } + } + + /** + * Check if a member is set in the bloom filter + * + * A member being set means that it *might* have been added. + * A member not being set means it *could not* have been added. + * + * This abstracts over isHit() to deal with filter updates and readiness. + * A class must exist with the name BloomFilter<type> and a static public + * mergeAndCheck() method. The later takes the following arguments: + * (BloomCache $bcache, $domain, $virtualKey, array $status) + * The method should return a bool indicating whether to use the filter. + * + * The 'shared' bloom key must be used for any updates and will be used + * for the membership check if the method returns true. Since the key is shared, + * the method should never use delete(). The filter cannot be used in cases where + * membership in the filter needs to be taken away. In such cases, code *cannot* + * use this method - instead, it can directly use the other BloomCache methods + * to manage custom filters with their own keys (e.g. not 'shared'). + * + * @param string $domain + * @param string $type + * @param string $member + * @return bool True if set, false if not (also returns true on error) + */ + final public function check( $domain, $type, $member ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) { + try { + $virtualKey = "$domain:$type"; + + $status = $this->getStatus( $virtualKey ); + if ( $status == false ) { + wfDebug( "Could not query virtual bloom filter '$virtualKey'." ); + return null; + } + + $useFilter = call_user_func_array( + array( "BloomFilter{$type}", 'mergeAndCheck' ), + array( $this, $domain, $virtualKey, $status ) + ); + + if ( $useFilter ) { + return ( $this->isHit( 'shared', "$virtualKey:$member" ) !== false ); + } + } catch ( MWException $e ) { + MWExceptionHandler::logException( $e ); + return true; + } + } + + return true; + } + + /** + * Inform the bloom filter of a new member in order to keep it up to date + * + * @param string $domain + * @param string $type + * @param string|array $members + * @return bool Success + */ + final public function insert( $domain, $type, $members ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) { + try { + $virtualKey = "$domain:$type"; + $prefixedMembers = array(); + foreach ( (array)$members as $member ) { + $prefixedMembers[] = "$virtualKey:$member"; + } + + return $this->add( 'shared', $prefixedMembers ); + } catch ( MWException $e ) { + MWExceptionHandler::logException( $e ); + return false; + } + } + + return true; + } + + /** + * Create a new bloom filter at $key (if one does not exist yet) + * + * @param string $key + * @param integer $size Bit length [default: 1000000] + * @param float $precision [default: .001] + * @return bool Success + */ + final public function init( $key, $size = 1000000, $precision = .001 ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doInit( "{$this->cacheID}:$key", $size, min( .1, $precision ) ); + } + + /** + * Add a member to the bloom filter at $key + * + * @param string $key + * @param string|array $members + * @return bool Success + */ + final public function add( $key, $members ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doAdd( "{$this->cacheID}:$key", (array)$members ); + } + + /** + * Check if a member is set in the bloom filter. + * + * A member being set means that it *might* have been added. + * A member not being set means it *could not* have been added. + * + * If this returns true, then the caller usually should do the + * expensive check (whatever that may be). It can be avoided otherwise. + * + * @param string $key + * @param string $member + * @return bool|null True if set, false if not, null on error + */ + final public function isHit( $key, $member ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doIsHit( "{$this->cacheID}:$key", $member ); + } + + /** + * Destroy a bloom filter at $key + * + * @param string $key + * @return bool Success + */ + final public function delete( $key ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doDelete( "{$this->cacheID}:$key" ); + } + + /** + * Set the status map of the virtual bloom filter at $key + * + * @param string $virtualKey + * @param array $values Map including some of (lastID, asOfTime, epoch) + * @return bool Success + */ + final public function setStatus( $virtualKey, array $values ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doSetStatus( "{$this->cacheID}:$virtualKey", $values ); + } + + /** + * Get the status map of the virtual bloom filter at $key + * + * The map includes: + * - lastID : the highest ID of the items merged in + * - asOfTime : UNIX timestamp that the filter is up-to-date as of + * - epoch : UNIX timestamp that filter started being populated + * Unset fields will have a null value. + * + * @param string $virtualKey + * @return array|bool False on failure + */ + final public function getStatus( $virtualKey ) { + $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ ); + + return $this->doGetStatus( "{$this->cacheID}:$virtualKey" ); + } + + /** + * Get an exclusive lock on a filter for updates + * + * @param string $virtualKey + * @return ScopedCallback|ScopedLock|null Returns null if acquisition failed + */ + public function getScopedLock( $virtualKey ) { + return null; + } + + /** + * @param string $key + * @param integer $size Bit length + * @param float $precision + * @return bool Success + */ + abstract protected function doInit( $key, $size, $precision ); + + /** + * @param string $key + * @param array $members + * @return bool Success + */ + abstract protected function doAdd( $key, array $members ); + + /** + * @param string $key + * @param string $member + * @return bool|null + */ + abstract protected function doIsHit( $key, $member ); + + /** + * @param string $key + * @return bool Success + */ + abstract protected function doDelete( $key ); + + /** + * @param string $virtualKey + * @param array $values + * @return bool Success + */ + abstract protected function doSetStatus( $virtualKey, array $values ); + + /** + * @param string $key + * @return array|bool + */ + abstract protected function doGetStatus( $key ); +} + +class EmptyBloomCache extends BloomCache { + public function __construct( array $config ) { + parent::__construct( array( 'cacheId' => 'none' ) ); + } + + protected function doInit( $key, $size, $precision ) { + return true; + } + + protected function doAdd( $key, array $members ) { + return true; + } + + protected function doIsHit( $key, $member ) { + return true; + } + + protected function doDelete( $key ) { + return true; + } + + protected function doSetStatus( $virtualKey, array $values ) { + return true; + } + + protected function doGetStatus( $virtualKey ) { + return array( 'lastID' => null, 'asOfTime' => null, 'epoch' => null ) ; + } +} diff --git a/includes/cache/bloom/BloomCacheRedis.php b/includes/cache/bloom/BloomCacheRedis.php new file mode 100644 index 00000000..212e5e8b --- /dev/null +++ b/includes/cache/bloom/BloomCacheRedis.php @@ -0,0 +1,370 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * Bloom filter implemented using Redis + * + * The Redis server must be >= 2.6 and should have volatile-lru or volatile-ttl + * if there is any eviction policy. It should not be allkeys-* in any case. Also, + * this can be used in a simple master/slave setup or with Redis Sentinel preferably. + * + * Some bits are based on https://github.com/ErikDubbelboer/redis-lua-scaling-bloom-filter + * but are simplified to use a single filter instead of up to 32 filters. + * + * @since 1.24 + */ +class BloomCacheRedis extends BloomCache { + /** @var RedisConnectionPool */ + protected $redisPool; + /** @var RedisLockManager */ + protected $lockMgr; + /** @var array */ + protected $servers; + /** @var integer Federate each filter into this many redis bitfield objects */ + protected $segments = 128; + + /** + * @params include: + * - redisServers : list of servers (address:<port>) (the first is the master) + * - redisConf : additional redis configuration + * + * @param array $config + */ + public function __construct( array $config ) { + parent::__construct( $config ); + + $redisConf = $config['redisConfig']; + $redisConf['serializer'] = 'none'; // manage that in this class + $this->redisPool = RedisConnectionPool::singleton( $redisConf ); + $this->servers = $config['redisServers']; + $this->lockMgr = new RedisLockManager( array( + 'lockServers' => array( 'srv1' => $this->servers[0] ), + 'srvsByBucket' => array( 0 => array( 'srv1' ) ), + 'redisConfig' => $config['redisConfig'] + ) ); + } + + protected function doInit( $key, $size, $precision ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return false; + } + + // 80000000 items at p = .001 take up 500MB and fit into one value. + // Do not hit the 512MB redis value limit by reducing the demands. + $size = min( $size, 80000000 * $this->segments ); + $precision = max( round( $precision, 3 ), .001 ); + $epoch = microtime( true ); + + static $script = +<<<LUA + local kMetadata, kData = unpack(KEYS) + local aEntries, aPrec, aEpoch = unpack(ARGV) + if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then + redis.call('DEL',kMetadata) + redis.call('HSET',kMetadata,'entries',aEntries) + redis.call('HSET',kMetadata,'precision',aPrec) + redis.call('HSET',kMetadata,'epoch',aEpoch) + redis.call('SET',kData,'') + return 1 + end + return 0 +LUA; + + $res = false; + try { + $conn->script( 'load', $script ); + $conn->multi( Redis::MULTI ); + for ( $i = 0; $i < $this->segments; ++$i ) { + $res = $conn->luaEval( $script, + array( + "$key:$i:bloom-metadata", # KEYS[1] + "$key:$i:bloom-data", # KEYS[2] + ceil( $size / $this->segments ), # ARGV[1] + $precision, # ARGV[2] + $epoch # ARGV[3] + ), + 2 # number of first argument(s) that are keys + ); + } + $results = $conn->exec(); + $res = $results && !in_array( false, $results, true ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return ( $res !== false ); + } + + protected function doAdd( $key, array $members ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return false; + } + + static $script = +<<<LUA + local kMetadata, kData = unpack(KEYS) + local aMember = unpack(ARGV) + + -- Check if the filter was initialized + if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then + return false + end + + -- Initial expected entries and desired precision + local entries = 1*redis.call('HGET',kMetadata,'entries') + local precision = 1*redis.call('HGET',kMetadata,'precision') + local hash = redis.sha1hex(aMember) + + -- Based on the math from: http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives + -- 0.480453013 = ln(2)^2 + local bits = math.ceil((entries * math.log(precision)) / -0.480453013) + + -- 0.693147180 = ln(2) + local k = math.floor(0.693147180 * bits / entries) + + -- This uses a variation on: + -- 'Less Hashing, Same Performance: Building a Better Bloom Filter' + -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf + local h = { } + h[0] = tonumber(string.sub(hash, 1, 8 ), 16) + h[1] = tonumber(string.sub(hash, 9, 16), 16) + h[2] = tonumber(string.sub(hash, 17, 24), 16) + h[3] = tonumber(string.sub(hash, 25, 32), 16) + + for i=1, k do + local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits + redis.call('SETBIT', kData, pos, 1) + end + + return 1 +LUA; + + $res = false; + try { + $conn->script( 'load', $script ); + $conn->multi( Redis::PIPELINE ); + foreach ( $members as $member ) { + $i = $this->getSegment( $member ); + $conn->luaEval( $script, + array( + "$key:$i:bloom-metadata", # KEYS[1], + "$key:$i:bloom-data", # KEYS[2] + $member # ARGV[1] + ), + 2 # number of first argument(s) that are keys + ); + } + $results = $conn->exec(); + $res = $results && !in_array( false, $results, true ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + if ( $res === false ) { + wfDebug( "Could not add to the '$key' bloom filter; it may be missing." ); + } + + return ( $res !== false ); + } + + protected function doSetStatus( $virtualKey, array $values ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return null; + } + + $res = false; + try { + $res = $conn->hMSet( "$virtualKey:filter-metadata", $values ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return ( $res !== false ); + } + + protected function doGetStatus( $virtualKey ) { + $conn = $this->getConnection( 'slave' ); + if ( !$conn ) { + return false; + } + + $res = false; + try { + $res = $conn->hGetAll( "$virtualKey:filter-metadata" ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + if ( is_array( $res ) ) { + $res['lastID'] = isset( $res['lastID'] ) ? $res['lastID'] : null; + $res['asOfTime'] = isset( $res['asOfTime'] ) ? $res['asOfTime'] : null; + $res['epoch'] = isset( $res['epoch'] ) ? $res['epoch'] : null; + } + + return $res; + } + + protected function doIsHit( $key, $member ) { + $conn = $this->getConnection( 'slave' ); + if ( !$conn ) { + return null; + } + + static $script = +<<<LUA + local kMetadata, kData = unpack(KEYS) + local aMember = unpack(ARGV) + + -- Check if the filter was initialized + if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then + return false + end + + -- Initial expected entries and desired precision. + -- This determines the size of the first and subsequent filters. + local entries = redis.call('HGET',kMetadata,'entries') + local precision = redis.call('HGET',kMetadata,'precision') + local hash = redis.sha1hex(aMember) + + -- This uses a variation on: + -- 'Less Hashing, Same Performance: Building a Better Bloom Filter' + -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf + local h = { } + h[0] = tonumber(string.sub(hash, 1, 8 ), 16) + h[1] = tonumber(string.sub(hash, 9, 16), 16) + h[2] = tonumber(string.sub(hash, 17, 24), 16) + h[3] = tonumber(string.sub(hash, 25, 32), 16) + + -- 0.480453013 = ln(2)^2 + local bits = math.ceil((entries * math.log(precision)) / -0.480453013) + + -- 0.693147180 = ln(2) + local k = math.floor(0.693147180 * bits / entries) + + local found = 1 + for i=1, k do + local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits + if redis.call('GETBIT', kData, pos) == 0 then + found = 0 + break + end + end + + return found +LUA; + + $res = null; + try { + $i = $this->getSegment( $member ); + $res = $conn->luaEval( $script, + array( + "$key:$i:bloom-metadata", # KEYS[1], + "$key:$i:bloom-data", # KEYS[2] + $member # ARGV[1] + ), + 2 # number of first argument(s) that are keys + ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return is_int( $res ) ? (bool)$res : null; + } + + protected function doDelete( $key ) { + $conn = $this->getConnection( 'master' ); + if ( !$conn ) { + return false; + } + + $res = false; + try { + $keys = array(); + for ( $i = 0; $i < $this->segments; ++$i ) { + $keys[] = "$key:$i:bloom-metadata"; + $keys[] = "$key:$i:bloom-data"; + } + $res = $conn->delete( $keys ); + } catch ( RedisException $e ) { + $this->handleException( $conn, $e ); + } + + return ( $res !== false ); + } + + public function getScopedLock( $virtualKey ) { + $status = Status::newGood(); + return ScopedLock::factory( $this->lockMgr, + array( $virtualKey ), LockManager::LOCK_EX, $status ); + } + + /** + * @param string $member + * @return integer + */ + protected function getSegment( $member ) { + return hexdec( substr( md5( $member ), 0, 2 ) ) % $this->segments; + } + + /** + * $param string $to (master/slave) + * @return RedisConnRef|bool Returns false on failure + */ + protected function getConnection( $to ) { + if ( $to === 'master' ) { + $conn = $this->redisPool->getConnection( $this->servers[0] ); + } else { + static $lastServer = null; + + $conn = false; + if ( $lastServer ) { + $conn = $this->redisPool->getConnection( $lastServer ); + if ( $conn ) { + return $conn; // reuse connection + } + } + $servers = $this->servers; + $attempts = min( 3, count( $servers ) ); + for ( $i = 1; $i <= $attempts; ++$i ) { + $index = mt_rand( 0, count( $servers ) - 1 ); + $conn = $this->redisPool->getConnection( $servers[$index] ); + if ( $conn ) { + $lastServer = $servers[$index]; + return $conn; + } + unset( $servers[$index] ); // skip next time + } + } + + return $conn; + } + + /** + * @param RedisConnRef $conn + * @param Exception $e + */ + protected function handleException( RedisConnRef $conn, $e ) { + $this->redisPool->handleError( $conn, $e ); + } +} diff --git a/includes/cache/bloom/BloomFilters.php b/includes/cache/bloom/BloomFilters.php new file mode 100644 index 00000000..9b710d79 --- /dev/null +++ b/includes/cache/bloom/BloomFilters.php @@ -0,0 +1,79 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * @since 1.24 + */ +class BloomFilterTitleHasLogs { + public static function mergeAndCheck( + BloomCache $bcache, $domain, $virtualKey, array $status + ) { + $age = microtime( true ) - $status['asOfTime']; // seconds + $scopedLock = ( mt_rand( 1, (int)pow( 3, max( 0, 5 - $age ) ) ) == 1 ) + ? $bcache->getScopedLock( $virtualKey ) + : false; + + if ( $scopedLock ) { + $updates = self::merge( $bcache, $domain, $virtualKey, $status ); + if ( isset( $updates['asOfTime'] ) ) { + $age = ( microtime( true ) - $updates['asOfTime'] ); + } + } + + return ( $age < 30 ); + } + + public static function merge( + BloomCache $bcache, $domain, $virtualKey, array $status + ) { + $limit = 1000; + $dbr = wfGetDB( DB_SLAVE, array(), $domain ); + $res = $dbr->select( 'logging', + array( 'log_namespace', 'log_title', 'log_id', 'log_timestamp' ), + array( 'log_id > ' . $dbr->addQuotes( (int)$status['lastID'] ) ), + __METHOD__, + array( 'ORDER BY' => 'log_id', 'LIMIT' => $limit ) + ); + + $updates = array(); + if ( $res->numRows() > 0 ) { + $members = array(); + foreach ( $res as $row ) { + $members[] = "$virtualKey:{$row->log_namespace}:{$row->log_title}"; + } + $lastID = $row->log_id; + $lastTime = $row->log_timestamp; + if ( !$bcache->add( 'shared', $members ) ) { + return false; + } + $updates['lastID'] = $lastID; + $updates['asOfTime'] = wfTimestamp( TS_UNIX, $lastTime ); + } else { + $updates['asOfTime'] = microtime( true ); + } + + $updates['epoch'] = $status['epoch'] ?: microtime( true ); + + $bcache->setStatus( $virtualKey, $updates ); + + return $updates; + } +} |