diff options
Diffstat (limited to 'includes/deferred')
| -rw-r--r-- | includes/deferred/CallableUpdate.php | 29 | ||||
| -rw-r--r-- | includes/deferred/DataUpdate.php | 125 | ||||
| -rw-r--r-- | includes/deferred/DeferredUpdates.php | 136 | ||||
| -rw-r--r-- | includes/deferred/HTMLCacheUpdate.php | 70 | ||||
| -rw-r--r-- | includes/deferred/LinksUpdate.php | 1019 | ||||
| -rw-r--r-- | includes/deferred/SearchUpdate.php | 209 | ||||
| -rw-r--r-- | includes/deferred/SiteStatsUpdate.php | 254 | ||||
| -rw-r--r-- | includes/deferred/SqlDataUpdate.php | 159 | ||||
| -rw-r--r-- | includes/deferred/SquidUpdate.php | 311 | ||||
| -rw-r--r-- | includes/deferred/ViewCountUpdate.php | 119 |
10 files changed, 2431 insertions, 0 deletions
diff --git a/includes/deferred/CallableUpdate.php b/includes/deferred/CallableUpdate.php new file mode 100644 index 00000000..808626d0 --- /dev/null +++ b/includes/deferred/CallableUpdate.php @@ -0,0 +1,29 @@ +<?php + +/** + * Deferrable Update for closure/callback + */ +class MWCallableUpdate implements DeferrableUpdate { + /** + * @var Closure|callable + */ + private $callback; + + /** + * @param callable $callback + * @throws MWException + */ + public function __construct( $callback ) { + if ( !is_callable( $callback ) ) { + throw new MWException( 'Not a valid callback/closure!' ); + } + $this->callback = $callback; + } + + /** + * Run the update + */ + public function doUpdate() { + call_user_func( $this->callback ); + } +} diff --git a/includes/deferred/DataUpdate.php b/includes/deferred/DataUpdate.php new file mode 100644 index 00000000..ed12c601 --- /dev/null +++ b/includes/deferred/DataUpdate.php @@ -0,0 +1,125 @@ +<?php +/** + * Base code for update jobs that do something with some secondary + * data extracted from article. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Abstract base class for update jobs that do something with some secondary + * data extracted from article. + * + * @note subclasses should NOT start or commit transactions in their doUpdate() method, + * a transaction will automatically be wrapped around the update. If need be, + * subclasses can override the beginTransaction() and commitTransaction() methods. + */ +abstract class DataUpdate implements DeferrableUpdate { + /** + * Constructor + */ + public function __construct() { + # noop + } + + /** + * Begin an appropriate transaction, if any. + * This default implementation does nothing. + */ + public function beginTransaction() { + //noop + } + + /** + * Commit the transaction started via beginTransaction, if any. + * This default implementation does nothing. + */ + public function commitTransaction() { + //noop + } + + /** + * Abort / roll back the transaction started via beginTransaction, if any. + * This default implementation does nothing. + */ + public function rollbackTransaction() { + //noop + } + + /** + * Convenience method, calls doUpdate() on every DataUpdate in the array. + * + * This methods supports transactions logic by first calling beginTransaction() + * on all updates in the array, then calling doUpdate() on each, and, if all goes well, + * then calling commitTransaction() on each update. If an error occurs, + * rollbackTransaction() will be called on any update object that had beginTransaction() + * called but not yet commitTransaction(). + * + * This allows for limited transactional logic across multiple backends for storing + * secondary data. + * + * @param array $updates A list of DataUpdate instances + * @throws Exception|null + */ + public static function runUpdates( $updates ) { + if ( empty( $updates ) ) { + return; # nothing to do + } + + $open_transactions = array(); + $exception = null; + + /** + * @var $update DataUpdate + * @var $trans DataUpdate + */ + + try { + // begin transactions + foreach ( $updates as $update ) { + $update->beginTransaction(); + $open_transactions[] = $update; + } + + // do work + foreach ( $updates as $update ) { + $update->doUpdate(); + } + + // commit transactions + while ( count( $open_transactions ) > 0 ) { + $trans = array_pop( $open_transactions ); + $trans->commitTransaction(); + } + } catch ( Exception $ex ) { + $exception = $ex; + wfDebug( "Caught exception, will rethrow after rollback: " . + $ex->getMessage() . "\n" ); + } + + // rollback remaining transactions + while ( count( $open_transactions ) > 0 ) { + $trans = array_pop( $open_transactions ); + $trans->rollbackTransaction(); + } + + if ( $exception ) { + throw $exception; // rethrow after cleanup + } + } +} diff --git a/includes/deferred/DeferredUpdates.php b/includes/deferred/DeferredUpdates.php new file mode 100644 index 00000000..b0c1899f --- /dev/null +++ b/includes/deferred/DeferredUpdates.php @@ -0,0 +1,136 @@ +<?php +/** + * Interface and manager for deferred updates. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Interface that deferrable updates should implement. Basically required so we + * can validate input on DeferredUpdates::addUpdate() + * + * @since 1.19 + */ +interface DeferrableUpdate { + /** + * Perform the actual work + */ + function doUpdate(); +} + +/** + * Class for managing the deferred updates. + * + * @since 1.19 + */ +class DeferredUpdates { + /** + * Store of updates to be deferred until the end of the request. + */ + private static $updates = array(); + + /** + * Add an update to the deferred list + * @param DeferrableUpdate $update Some object that implements doUpdate() + */ + public static function addUpdate( DeferrableUpdate $update ) { + array_push( self::$updates, $update ); + } + + /** + * HTMLCacheUpdates are the most common deferred update people use. This + * is a shortcut method for that. + * @see HTMLCacheUpdate::__construct() + * @param Title $title + * @param string $table + */ + public static function addHTMLCacheUpdate( $title, $table ) { + self::addUpdate( new HTMLCacheUpdate( $title, $table ) ); + } + + /** + * Add a callable update. In a lot of cases, we just need a callback/closure, + * defining a new DeferrableUpdate object is not necessary + * @see MWCallableUpdate::__construct() + * @param callable $callable + */ + public static function addCallableUpdate( $callable ) { + self::addUpdate( new MWCallableUpdate( $callable ) ); + } + + /** + * Do any deferred updates and clear the list + * + * @param string $commit Set to 'commit' to commit after every update to + * prevent lock contention + */ + public static function doUpdates( $commit = '' ) { + global $wgDeferredUpdateList; + + wfProfileIn( __METHOD__ ); + + $updates = array_merge( $wgDeferredUpdateList, self::$updates ); + + // No need to get master connections in case of empty updates array + if ( !count( $updates ) ) { + wfProfileOut( __METHOD__ ); + + return; + } + + $dbw = false; + $doCommit = $commit == 'commit'; + if ( $doCommit ) { + $dbw = wfGetDB( DB_MASTER ); + } + + while ( $updates ) { + self::clearPendingUpdates(); + + /** @var DeferrableUpdate $update */ + foreach ( $updates as $update ) { + try { + $update->doUpdate(); + + if ( $doCommit && $dbw->trxLevel() ) { + $dbw->commit( __METHOD__, 'flush' ); + } + } catch ( MWException $e ) { + // We don't want exceptions thrown during deferred updates to + // be reported to the user since the output is already sent. + // Instead we just log them. + if ( !$e instanceof ErrorPageError ) { + MWExceptionHandler::logException( $e ); + } + } + } + $updates = array_merge( $wgDeferredUpdateList, self::$updates ); + } + + wfProfileOut( __METHOD__ ); + } + + /** + * Clear all pending updates without performing them. Generally, you don't + * want or need to call this. Unit tests need it though. + */ + public static function clearPendingUpdates() { + global $wgDeferredUpdateList; + $wgDeferredUpdateList = self::$updates = array(); + } +} diff --git a/includes/deferred/HTMLCacheUpdate.php b/includes/deferred/HTMLCacheUpdate.php new file mode 100644 index 00000000..54fa5943 --- /dev/null +++ b/includes/deferred/HTMLCacheUpdate.php @@ -0,0 +1,70 @@ +<?php +/** + * HTML cache invalidation of all pages linking to a given title. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * Class to invalidate the HTML cache of all the pages linking to a given title. + * + * @ingroup Cache + */ +class HTMLCacheUpdate implements DeferrableUpdate { + /** @var Title */ + public $mTitle; + + /** @var string */ + public $mTable; + + /** + * @param Title $titleTo + * @param string $table + */ + function __construct( Title $titleTo, $table ) { + $this->mTitle = $titleTo; + $this->mTable = $table; + } + + public function doUpdate() { + wfProfileIn( __METHOD__ ); + + $job = new HTMLCacheUpdateJob( + $this->mTitle, + array( + 'table' => $this->mTable, + ) + Job::newRootJobParams( // "overall" refresh links job info + "htmlCacheUpdate:{$this->mTable}:{$this->mTitle->getPrefixedText()}" + ) + ); + + $count = $this->mTitle->getBacklinkCache()->getNumLinks( $this->mTable, 100 ); + if ( $count >= 100 ) { // many backlinks + JobQueueGroup::singleton()->push( $job ); + JobQueueGroup::singleton()->deduplicateRootJob( $job ); + } else { // few backlinks ($count might be off even if 0) + $dbw = wfGetDB( DB_MASTER ); + $dbw->onTransactionIdle( function () use ( $job ) { + $job->run(); // just do the purge query now + } ); + } + + wfProfileOut( __METHOD__ ); + } +} diff --git a/includes/deferred/LinksUpdate.php b/includes/deferred/LinksUpdate.php new file mode 100644 index 00000000..45d26648 --- /dev/null +++ b/includes/deferred/LinksUpdate.php @@ -0,0 +1,1019 @@ +<?php +/** + * Updater for link tracking tables after a page edit. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * See docs/deferred.txt + * + * @todo document (e.g. one-sentence top-level class description). + */ +class LinksUpdate extends SqlDataUpdate { + // @todo make members protected, but make sure extensions don't break + + /** @var int Page ID of the article linked from */ + public $mId; + + /** @var Title Title object of the article linked from */ + public $mTitle; + + /** @var ParserOutput */ + public $mParserOutput; + + /** @var array Map of title strings to IDs for the links in the document */ + public $mLinks; + + /** @var array DB keys of the images used, in the array key only */ + public $mImages; + + /** @var array Map of title strings to IDs for the template references, including broken ones */ + public $mTemplates; + + /** @var array URLs of external links, array key only */ + public $mExternals; + + /** @var array Map of category names to sort keys */ + public $mCategories; + + /** @var array Map of language codes to titles */ + public $mInterlangs; + + /** @var array Map of arbitrary name to value */ + public $mProperties; + + /** @var DatabaseBase Database connection reference */ + public $mDb; + + /** @var array SELECT options to be used */ + public $mOptions; + + /** @var bool Whether to queue jobs for recursive updates */ + public $mRecursive; + + /** + * @var null|array Added links if calculated. + */ + private $linkInsertions = null; + + /** + * @var null|array Deleted links if calculated. + */ + private $linkDeletions = null; + + /** + * Constructor + * + * @param Title $title Title of the page we're updating + * @param ParserOutput $parserOutput Output from a full parse of this page + * @param bool $recursive Queue jobs for recursive updates? + * @throws MWException + */ + function __construct( $title, $parserOutput, $recursive = true ) { + parent::__construct( false ); // no implicit transaction + + if ( !( $title instanceof Title ) ) { + throw new MWException( "The calling convention to LinksUpdate::LinksUpdate() has changed. " . + "Please see Article::editUpdates() for an invocation example.\n" ); + } + + if ( !( $parserOutput instanceof ParserOutput ) ) { + throw new MWException( "The calling convention to LinksUpdate::__construct() has changed. " . + "Please see WikiPage::doEditUpdates() for an invocation example.\n" ); + } + + $this->mTitle = $title; + $this->mId = $title->getArticleID(); + + if ( !$this->mId ) { + throw new MWException( "The Title object did not provide an article " . + "ID. Perhaps the page doesn't exist?" ); + } + + $this->mParserOutput = $parserOutput; + + $this->mLinks = $parserOutput->getLinks(); + $this->mImages = $parserOutput->getImages(); + $this->mTemplates = $parserOutput->getTemplates(); + $this->mExternals = $parserOutput->getExternalLinks(); + $this->mCategories = $parserOutput->getCategories(); + $this->mProperties = $parserOutput->getProperties(); + $this->mInterwikis = $parserOutput->getInterwikiLinks(); + + # Convert the format of the interlanguage links + # I didn't want to change it in the ParserOutput, because that array is passed all + # the way back to the skin, so either a skin API break would be required, or an + # inefficient back-conversion. + $ill = $parserOutput->getLanguageLinks(); + $this->mInterlangs = array(); + foreach ( $ill as $link ) { + list( $key, $title ) = explode( ':', $link, 2 ); + $this->mInterlangs[$key] = $title; + } + + foreach ( $this->mCategories as &$sortkey ) { + # If the sortkey is longer then 255 bytes, + # it truncated by DB, and then doesn't get + # matched when comparing existing vs current + # categories, causing bug 25254. + # Also. substr behaves weird when given "". + if ( $sortkey !== '' ) { + $sortkey = substr( $sortkey, 0, 255 ); + } + } + + $this->mRecursive = $recursive; + + wfRunHooks( 'LinksUpdateConstructed', array( &$this ) ); + } + + /** + * Update link tables with outgoing links from an updated article + */ + public function doUpdate() { + wfRunHooks( 'LinksUpdate', array( &$this ) ); + $this->doIncrementalUpdate(); + wfRunHooks( 'LinksUpdateComplete', array( &$this ) ); + } + + protected function doIncrementalUpdate() { + wfProfileIn( __METHOD__ ); + + # Page links + $existing = $this->getExistingLinks(); + $this->linkDeletions = $this->getLinkDeletions( $existing ); + $this->linkInsertions = $this->getLinkInsertions( $existing ); + $this->incrTableUpdate( 'pagelinks', 'pl', $this->linkDeletions, $this->linkInsertions ); + + # Image links + $existing = $this->getExistingImages(); + + $imageDeletes = $this->getImageDeletions( $existing ); + $this->incrTableUpdate( 'imagelinks', 'il', $imageDeletes, + $this->getImageInsertions( $existing ) ); + + # Invalidate all image description pages which had links added or removed + $imageUpdates = $imageDeletes + array_diff_key( $this->mImages, $existing ); + $this->invalidateImageDescriptions( $imageUpdates ); + + # External links + $existing = $this->getExistingExternals(); + $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ), + $this->getExternalInsertions( $existing ) ); + + # Language links + $existing = $this->getExistingInterlangs(); + $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ), + $this->getInterlangInsertions( $existing ) ); + + # Inline interwiki links + $existing = $this->getExistingInterwikis(); + $this->incrTableUpdate( 'iwlinks', 'iwl', $this->getInterwikiDeletions( $existing ), + $this->getInterwikiInsertions( $existing ) ); + + # Template links + $existing = $this->getExistingTemplates(); + $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ), + $this->getTemplateInsertions( $existing ) ); + + # Category links + $existing = $this->getExistingCategories(); + + $categoryDeletes = $this->getCategoryDeletions( $existing ); + + $this->incrTableUpdate( 'categorylinks', 'cl', $categoryDeletes, + $this->getCategoryInsertions( $existing ) ); + + # Invalidate all categories which were added, deleted or changed (set symmetric difference) + $categoryInserts = array_diff_assoc( $this->mCategories, $existing ); + $categoryUpdates = $categoryInserts + $categoryDeletes; + $this->invalidateCategories( $categoryUpdates ); + $this->updateCategoryCounts( $categoryInserts, $categoryDeletes ); + + # Page properties + $existing = $this->getExistingProperties(); + + $propertiesDeletes = $this->getPropertyDeletions( $existing ); + + $this->incrTableUpdate( 'page_props', 'pp', $propertiesDeletes, + $this->getPropertyInsertions( $existing ) ); + + # Invalidate the necessary pages + $changed = $propertiesDeletes + array_diff_assoc( $this->mProperties, $existing ); + $this->invalidateProperties( $changed ); + + # Update the links table freshness for this title + $this->updateLinksTimestamp(); + + # Refresh links of all pages including this page + # This will be in a separate transaction + if ( $this->mRecursive ) { + $this->queueRecursiveJobs(); + } + + wfProfileOut( __METHOD__ ); + } + + /** + * Queue recursive jobs for this page + * + * Which means do LinksUpdate on all pages that include the current page, + * using the job queue. + */ + function queueRecursiveJobs() { + self::queueRecursiveJobsForTable( $this->mTitle, 'templatelinks' ); + if ( $this->mTitle->getNamespace() == NS_FILE ) { + // Process imagelinks in case the title is or was a redirect + self::queueRecursiveJobsForTable( $this->mTitle, 'imagelinks' ); + } + } + + /** + * Queue a RefreshLinks job for any table. + * + * @param Title $title Title to do job for + * @param string $table Table to use (e.g. 'templatelinks') + */ + public static function queueRecursiveJobsForTable( Title $title, $table ) { + wfProfileIn( __METHOD__ ); + if ( $title->getBacklinkCache()->hasLinks( $table ) ) { + $job = new RefreshLinksJob( + $title, + array( + 'table' => $table, + 'recursive' => true, + ) + Job::newRootJobParams( // "overall" refresh links job info + "refreshlinks:{$table}:{$title->getPrefixedText()}" + ) + ); + JobQueueGroup::singleton()->push( $job ); + JobQueueGroup::singleton()->deduplicateRootJob( $job ); + } + wfProfileOut( __METHOD__ ); + } + + /** + * @param array $cats + */ + function invalidateCategories( $cats ) { + $this->invalidatePages( NS_CATEGORY, array_keys( $cats ) ); + } + + /** + * Update all the appropriate counts in the category table. + * @param array $added Associative array of category name => sort key + * @param array $deleted Associative array of category name => sort key + */ + function updateCategoryCounts( $added, $deleted ) { + $a = WikiPage::factory( $this->mTitle ); + $a->updateCategoryCounts( + array_keys( $added ), array_keys( $deleted ) + ); + } + + /** + * @param array $images + */ + function invalidateImageDescriptions( $images ) { + $this->invalidatePages( NS_FILE, array_keys( $images ) ); + } + + /** + * Update a table by doing a delete query then an insert query + * @param string $table Table name + * @param string $prefix Field name prefix + * @param array $deletions + * @param array $insertions Rows to insert + */ + function incrTableUpdate( $table, $prefix, $deletions, $insertions ) { + if ( $table == 'page_props' ) { + $fromField = 'pp_page'; + } else { + $fromField = "{$prefix}_from"; + } + $where = array( $fromField => $this->mId ); + if ( $table == 'pagelinks' || $table == 'templatelinks' || $table == 'iwlinks' ) { + if ( $table == 'iwlinks' ) { + $baseKey = 'iwl_prefix'; + } else { + $baseKey = "{$prefix}_namespace"; + } + $clause = $this->mDb->makeWhereFrom2d( $deletions, $baseKey, "{$prefix}_title" ); + if ( $clause ) { + $where[] = $clause; + } else { + $where = false; + } + } else { + if ( $table == 'langlinks' ) { + $toField = 'll_lang'; + } elseif ( $table == 'page_props' ) { + $toField = 'pp_propname'; + } else { + $toField = $prefix . '_to'; + } + if ( count( $deletions ) ) { + $where[$toField] = array_keys( $deletions ); + } else { + $where = false; + } + } + if ( $where ) { + $this->mDb->delete( $table, $where, __METHOD__ ); + } + if ( count( $insertions ) ) { + $this->mDb->insert( $table, $insertions, __METHOD__, 'IGNORE' ); + wfRunHooks( 'LinksUpdateAfterInsert', array( $this, $table, $insertions ) ); + } + } + + /** + * Get an array of pagelinks insertions for passing to the DB + * Skips the titles specified by the 2-D array $existing + * @param array $existing + * @return array + */ + private function getLinkInsertions( $existing = array() ) { + $arr = array(); + foreach ( $this->mLinks as $ns => $dbkeys ) { + $diffs = isset( $existing[$ns] ) + ? array_diff_key( $dbkeys, $existing[$ns] ) + : $dbkeys; + foreach ( $diffs as $dbk => $id ) { + $arr[] = array( + 'pl_from' => $this->mId, + 'pl_from_namespace' => $this->mTitle->getNamespace(), + 'pl_namespace' => $ns, + 'pl_title' => $dbk + ); + } + } + + return $arr; + } + + /** + * Get an array of template insertions. Like getLinkInsertions() + * @param array $existing + * @return array + */ + private function getTemplateInsertions( $existing = array() ) { + $arr = array(); + foreach ( $this->mTemplates as $ns => $dbkeys ) { + $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys; + foreach ( $diffs as $dbk => $id ) { + $arr[] = array( + 'tl_from' => $this->mId, + 'tl_from_namespace' => $this->mTitle->getNamespace(), + 'tl_namespace' => $ns, + 'tl_title' => $dbk + ); + } + } + + return $arr; + } + + /** + * Get an array of image insertions + * Skips the names specified in $existing + * @param array $existing + * @return array + */ + private function getImageInsertions( $existing = array() ) { + $arr = array(); + $diffs = array_diff_key( $this->mImages, $existing ); + foreach ( $diffs as $iname => $dummy ) { + $arr[] = array( + 'il_from' => $this->mId, + 'il_from_namespace' => $this->mTitle->getNamespace(), + 'il_to' => $iname + ); + } + + return $arr; + } + + /** + * Get an array of externallinks insertions. Skips the names specified in $existing + * @param array $existing + * @return array + */ + private function getExternalInsertions( $existing = array() ) { + $arr = array(); + $diffs = array_diff_key( $this->mExternals, $existing ); + foreach ( $diffs as $url => $dummy ) { + foreach ( wfMakeUrlIndexes( $url ) as $index ) { + $arr[] = array( + 'el_id' => $this->mDb->nextSequenceValue( 'externallinks_el_id_seq' ), + 'el_from' => $this->mId, + 'el_to' => $url, + 'el_index' => $index, + ); + } + } + + return $arr; + } + + /** + * Get an array of category insertions + * + * @param array $existing Mapping existing category names to sort keys. If both + * match a link in $this, the link will be omitted from the output + * + * @return array + */ + private function getCategoryInsertions( $existing = array() ) { + global $wgContLang, $wgCategoryCollation; + $diffs = array_diff_assoc( $this->mCategories, $existing ); + $arr = array(); + foreach ( $diffs as $name => $prefix ) { + $nt = Title::makeTitleSafe( NS_CATEGORY, $name ); + $wgContLang->findVariantLink( $name, $nt, true ); + + if ( $this->mTitle->getNamespace() == NS_CATEGORY ) { + $type = 'subcat'; + } elseif ( $this->mTitle->getNamespace() == NS_FILE ) { + $type = 'file'; + } else { + $type = 'page'; + } + + # Treat custom sortkeys as a prefix, so that if multiple + # things are forced to sort as '*' or something, they'll + # sort properly in the category rather than in page_id + # order or such. + $sortkey = Collation::singleton()->getSortKey( + $this->mTitle->getCategorySortkey( $prefix ) ); + + $arr[] = array( + 'cl_from' => $this->mId, + 'cl_to' => $name, + 'cl_sortkey' => $sortkey, + 'cl_timestamp' => $this->mDb->timestamp(), + 'cl_sortkey_prefix' => $prefix, + 'cl_collation' => $wgCategoryCollation, + 'cl_type' => $type, + ); + } + + return $arr; + } + + /** + * Get an array of interlanguage link insertions + * + * @param array $existing Mapping existing language codes to titles + * + * @return array + */ + private function getInterlangInsertions( $existing = array() ) { + $diffs = array_diff_assoc( $this->mInterlangs, $existing ); + $arr = array(); + foreach ( $diffs as $lang => $title ) { + $arr[] = array( + 'll_from' => $this->mId, + 'll_lang' => $lang, + 'll_title' => $title + ); + } + + return $arr; + } + + /** + * Get an array of page property insertions + * @param array $existing + * @return array + */ + function getPropertyInsertions( $existing = array() ) { + $diffs = array_diff_assoc( $this->mProperties, $existing ); + + $arr = array(); + foreach ( array_keys( $diffs ) as $name ) { + $arr[] = $this->getPagePropRowData( $name ); + } + + return $arr; + } + + /** + * Returns an associative array to be used for inserting a row into + * the page_props table. Besides the given property name, this will + * include the page id from $this->mId and any property value from + * $this->mProperties. + * + * The array returned will include the pp_sortkey field if this + * is present in the database (as indicated by $wgPagePropsHaveSortkey). + * The sortkey value is currently determined by getPropertySortKeyValue(). + * + * @note this assumes that $this->mProperties[$prop] is defined. + * + * @param string $prop The name of the property. + * + * @return array + */ + private function getPagePropRowData( $prop ) { + global $wgPagePropsHaveSortkey; + + $value = $this->mProperties[$prop]; + + $row = array( + 'pp_page' => $this->mId, + 'pp_propname' => $prop, + 'pp_value' => $value, + ); + + if ( $wgPagePropsHaveSortkey ) { + $row['pp_sortkey'] = $this->getPropertySortKeyValue( $value ); + } + + return $row; + } + + /** + * Determines the sort key for the given property value. + * This will return $value if it is a float or int, + * 1 or resp. 0 if it is a bool, and null otherwise. + * + * @note In the future, we may allow the sortkey to be specified explicitly + * in ParserOutput::setProperty. + * + * @param mixed $value + * + * @return float|null + */ + private function getPropertySortKeyValue( $value ) { + if ( is_int( $value ) || is_float( $value ) || is_bool( $value ) ) { + return floatval( $value ); + } + + return null; + } + + /** + * Get an array of interwiki insertions for passing to the DB + * Skips the titles specified by the 2-D array $existing + * @param array $existing + * @return array + */ + private function getInterwikiInsertions( $existing = array() ) { + $arr = array(); + foreach ( $this->mInterwikis as $prefix => $dbkeys ) { + $diffs = isset( $existing[$prefix] ) + ? array_diff_key( $dbkeys, $existing[$prefix] ) + : $dbkeys; + + foreach ( $diffs as $dbk => $id ) { + $arr[] = array( + 'iwl_from' => $this->mId, + 'iwl_prefix' => $prefix, + 'iwl_title' => $dbk + ); + } + } + + return $arr; + } + + /** + * Given an array of existing links, returns those links which are not in $this + * and thus should be deleted. + * @param array $existing + * @return array + */ + private function getLinkDeletions( $existing ) { + $del = array(); + foreach ( $existing as $ns => $dbkeys ) { + if ( isset( $this->mLinks[$ns] ) ) { + $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks[$ns] ); + } else { + $del[$ns] = $existing[$ns]; + } + } + + return $del; + } + + /** + * Given an array of existing templates, returns those templates which are not in $this + * and thus should be deleted. + * @param array $existing + * @return array + */ + private function getTemplateDeletions( $existing ) { + $del = array(); + foreach ( $existing as $ns => $dbkeys ) { + if ( isset( $this->mTemplates[$ns] ) ) { + $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates[$ns] ); + } else { + $del[$ns] = $existing[$ns]; + } + } + + return $del; + } + + /** + * Given an array of existing images, returns those images which are not in $this + * and thus should be deleted. + * @param array $existing + * @return array + */ + private function getImageDeletions( $existing ) { + return array_diff_key( $existing, $this->mImages ); + } + + /** + * Given an array of existing external links, returns those links which are not + * in $this and thus should be deleted. + * @param array $existing + * @return array + */ + private function getExternalDeletions( $existing ) { + return array_diff_key( $existing, $this->mExternals ); + } + + /** + * Given an array of existing categories, returns those categories which are not in $this + * and thus should be deleted. + * @param array $existing + * @return array + */ + private function getCategoryDeletions( $existing ) { + return array_diff_assoc( $existing, $this->mCategories ); + } + + /** + * Given an array of existing interlanguage links, returns those links which are not + * in $this and thus should be deleted. + * @param array $existing + * @return array + */ + private function getInterlangDeletions( $existing ) { + return array_diff_assoc( $existing, $this->mInterlangs ); + } + + /** + * Get array of properties which should be deleted. + * @param array $existing + * @return array + */ + function getPropertyDeletions( $existing ) { + return array_diff_assoc( $existing, $this->mProperties ); + } + + /** + * Given an array of existing interwiki links, returns those links which are not in $this + * and thus should be deleted. + * @param array $existing + * @return array + */ + private function getInterwikiDeletions( $existing ) { + $del = array(); + foreach ( $existing as $prefix => $dbkeys ) { + if ( isset( $this->mInterwikis[$prefix] ) ) { + $del[$prefix] = array_diff_key( $existing[$prefix], $this->mInterwikis[$prefix] ); + } else { + $del[$prefix] = $existing[$prefix]; + } + } + + return $del; + } + + /** + * Get an array of existing links, as a 2-D array + * + * @return array + */ + private function getExistingLinks() { + $res = $this->mDb->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ), + array( 'pl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + if ( !isset( $arr[$row->pl_namespace] ) ) { + $arr[$row->pl_namespace] = array(); + } + $arr[$row->pl_namespace][$row->pl_title] = 1; + } + + return $arr; + } + + /** + * Get an array of existing templates, as a 2-D array + * + * @return array + */ + private function getExistingTemplates() { + $res = $this->mDb->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ), + array( 'tl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + if ( !isset( $arr[$row->tl_namespace] ) ) { + $arr[$row->tl_namespace] = array(); + } + $arr[$row->tl_namespace][$row->tl_title] = 1; + } + + return $arr; + } + + /** + * Get an array of existing images, image names in the keys + * + * @return array + */ + private function getExistingImages() { + $res = $this->mDb->select( 'imagelinks', array( 'il_to' ), + array( 'il_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->il_to] = 1; + } + + return $arr; + } + + /** + * Get an array of existing external links, URLs in the keys + * + * @return array + */ + private function getExistingExternals() { + $res = $this->mDb->select( 'externallinks', array( 'el_to' ), + array( 'el_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->el_to] = 1; + } + + return $arr; + } + + /** + * Get an array of existing categories, with the name in the key and sort key in the value. + * + * @return array + */ + private function getExistingCategories() { + $res = $this->mDb->select( 'categorylinks', array( 'cl_to', 'cl_sortkey_prefix' ), + array( 'cl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->cl_to] = $row->cl_sortkey_prefix; + } + + return $arr; + } + + /** + * Get an array of existing interlanguage links, with the language code in the key and the + * title in the value. + * + * @return array + */ + private function getExistingInterlangs() { + $res = $this->mDb->select( 'langlinks', array( 'll_lang', 'll_title' ), + array( 'll_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->ll_lang] = $row->ll_title; + } + + return $arr; + } + + /** + * Get an array of existing inline interwiki links, as a 2-D array + * @return array (prefix => array(dbkey => 1)) + */ + protected function getExistingInterwikis() { + $res = $this->mDb->select( 'iwlinks', array( 'iwl_prefix', 'iwl_title' ), + array( 'iwl_from' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + if ( !isset( $arr[$row->iwl_prefix] ) ) { + $arr[$row->iwl_prefix] = array(); + } + $arr[$row->iwl_prefix][$row->iwl_title] = 1; + } + + return $arr; + } + + /** + * Get an array of existing categories, with the name in the key and sort key in the value. + * + * @return array Array of property names and values + */ + private function getExistingProperties() { + $res = $this->mDb->select( 'page_props', array( 'pp_propname', 'pp_value' ), + array( 'pp_page' => $this->mId ), __METHOD__, $this->mOptions ); + $arr = array(); + foreach ( $res as $row ) { + $arr[$row->pp_propname] = $row->pp_value; + } + + return $arr; + } + + /** + * Return the title object of the page being updated + * @return Title + */ + public function getTitle() { + return $this->mTitle; + } + + /** + * Returns parser output + * @since 1.19 + * @return ParserOutput + */ + public function getParserOutput() { + return $this->mParserOutput; + } + + /** + * Return the list of images used as generated by the parser + * @return array + */ + public function getImages() { + return $this->mImages; + } + + /** + * Invalidate any necessary link lists related to page property changes + * @param array $changed + */ + private function invalidateProperties( $changed ) { + global $wgPagePropLinkInvalidations; + + foreach ( $changed as $name => $value ) { + if ( isset( $wgPagePropLinkInvalidations[$name] ) ) { + $inv = $wgPagePropLinkInvalidations[$name]; + if ( !is_array( $inv ) ) { + $inv = array( $inv ); + } + foreach ( $inv as $table ) { + $update = new HTMLCacheUpdate( $this->mTitle, $table ); + $update->doUpdate(); + } + } + } + } + + /** + * Fetch page links added by this LinksUpdate. Only available after the update is complete. + * @since 1.22 + * @return null|array Array of Titles + */ + public function getAddedLinks() { + if ( $this->linkInsertions === null ) { + return null; + } + $result = array(); + foreach ( $this->linkInsertions as $insertion ) { + $result[] = Title::makeTitle( $insertion['pl_namespace'], $insertion['pl_title'] ); + } + + return $result; + } + + /** + * Fetch page links removed by this LinksUpdate. Only available after the update is complete. + * @since 1.22 + * @return null|array Array of Titles + */ + public function getRemovedLinks() { + if ( $this->linkDeletions === null ) { + return null; + } + $result = array(); + foreach ( $this->linkDeletions as $ns => $titles ) { + foreach ( $titles as $title => $unused ) { + $result[] = Title::makeTitle( $ns, $title ); + } + } + + return $result; + } + + /** + * Update links table freshness + */ + protected function updateLinksTimestamp() { + if ( $this->mId ) { + // The link updates made here only reflect the freshness of the parser output + $timestamp = $this->mParserOutput->getCacheTime(); + $this->mDb->update( 'page', + array( 'page_links_updated' => $this->mDb->timestamp( $timestamp ) ), + array( 'page_id' => $this->mId ), + __METHOD__ + ); + } + } +} + +/** + * Update object handling the cleanup of links tables after a page was deleted. + **/ +class LinksDeletionUpdate extends SqlDataUpdate { + /** @var WikiPage The WikiPage that was deleted */ + protected $mPage; + + /** + * Constructor + * + * @param WikiPage $page Page we are updating + * @throws MWException + */ + function __construct( WikiPage $page ) { + parent::__construct( false ); // no implicit transaction + + $this->mPage = $page; + + if ( !$page->exists() ) { + throw new MWException( "Page ID not known, perhaps the page doesn't exist?" ); + } + } + + /** + * Do some database updates after deletion + */ + public function doUpdate() { + $title = $this->mPage->getTitle(); + $id = $this->mPage->getId(); + + # Delete restrictions for it + $this->mDb->delete( 'page_restrictions', array( 'pr_page' => $id ), __METHOD__ ); + + # Fix category table counts + $cats = array(); + $res = $this->mDb->select( 'categorylinks', 'cl_to', array( 'cl_from' => $id ), __METHOD__ ); + + foreach ( $res as $row ) { + $cats[] = $row->cl_to; + } + + $this->mPage->updateCategoryCounts( array(), $cats ); + + # If using cascading deletes, we can skip some explicit deletes + if ( !$this->mDb->cascadingDeletes() ) { + # Delete outgoing links + $this->mDb->delete( 'pagelinks', array( 'pl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'imagelinks', array( 'il_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'categorylinks', array( 'cl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'templatelinks', array( 'tl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'externallinks', array( 'el_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'langlinks', array( 'll_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'iwlinks', array( 'iwl_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'redirect', array( 'rd_from' => $id ), __METHOD__ ); + $this->mDb->delete( 'page_props', array( 'pp_page' => $id ), __METHOD__ ); + } + + # If using cleanup triggers, we can skip some manual deletes + if ( !$this->mDb->cleanupTriggers() ) { + # Clean up recentchanges entries... + $this->mDb->delete( 'recentchanges', + array( 'rc_type != ' . RC_LOG, + 'rc_namespace' => $title->getNamespace(), + 'rc_title' => $title->getDBkey() ), + __METHOD__ ); + $this->mDb->delete( 'recentchanges', + array( 'rc_type != ' . RC_LOG, 'rc_cur_id' => $id ), + __METHOD__ ); + } + } + + /** + * Update all the appropriate counts in the category table. + * @param array $added Associative array of category name => sort key + * @param array $deleted Associative array of category name => sort key + */ + function updateCategoryCounts( $added, $deleted ) { + $a = WikiPage::factory( $this->mTitle ); + $a->updateCategoryCounts( + array_keys( $added ), array_keys( $deleted ) + ); + } +} diff --git a/includes/deferred/SearchUpdate.php b/includes/deferred/SearchUpdate.php new file mode 100644 index 00000000..5d084afd --- /dev/null +++ b/includes/deferred/SearchUpdate.php @@ -0,0 +1,209 @@ +<?php +/** + * Search index updater + * + * See deferred.txt + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Search + */ + +/** + * Database independant search index updater + * + * @ingroup Search + */ +class SearchUpdate implements DeferrableUpdate { + /** @var int Page id being updated */ + private $id = 0; + + /** @var Title Title we're updating */ + private $title; + + /** @var Content|bool Content of the page (not text) */ + private $content; + + /** + * Constructor + * + * @param int $id Page id to update + * @param Title|string $title Title of page to update + * @param Content|string|bool $c Content of the page to update. Default: false. + * If a Content object, text will be gotten from it. String is for back-compat. + * Passing false tells the backend to just update the title, not the content + */ + public function __construct( $id, $title, $c = false ) { + if ( is_string( $title ) ) { + $nt = Title::newFromText( $title ); + } else { + $nt = $title; + } + + if ( $nt ) { + $this->id = $id; + // is_string() check is back-compat for ApprovedRevs + if ( is_string( $c ) ) { + $this->content = new TextContent( $c ); + } else { + $this->content = $c ?: false; + } + $this->title = $nt; + } else { + wfDebug( "SearchUpdate object created with invalid title '$title'\n" ); + } + } + + /** + * Perform actual update for the entry + */ + public function doUpdate() { + global $wgDisableSearchUpdate; + + if ( $wgDisableSearchUpdate || !$this->id ) { + return; + } + + wfProfileIn( __METHOD__ ); + + $page = WikiPage::newFromId( $this->id, WikiPage::READ_LATEST ); + + foreach ( SearchEngine::getSearchTypes() as $type ) { + $search = SearchEngine::create( $type ); + $indexTitle = $this->indexTitle( $search ); + if ( !$search->supports( 'search-update' ) ) { + continue; + } + + $normalTitle = $search->normalizeText( $indexTitle ); + + if ( $page === null ) { + $search->delete( $this->id, $normalTitle ); + continue; + } elseif ( $this->content === false ) { + $search->updateTitle( $this->id, $normalTitle ); + continue; + } + + $text = $search->getTextFromContent( $this->title, $this->content ); + if ( !$search->textAlreadyUpdatedForIndex() ) { + $text = self::updateText( $text ); + } + + # Perform the actual update + $search->update( $this->id, $normalTitle, $search->normalizeText( $text ) ); + } + + wfProfileOut( __METHOD__ ); + } + + /** + * Clean text for indexing. Only really suitable for indexing in databases. + * If you're using a real search engine, you'll probably want to override + * this behavior and do something nicer with the original wikitext. + * @param string $text + * @return string + */ + public static function updateText( $text ) { + global $wgContLang; + + # Language-specific strip/conversion + $text = $wgContLang->normalizeForSearch( $text ); + $lc = SearchEngine::legalSearchChars() . '&#;'; + + wfProfileIn( __METHOD__ . '-regexps' ); + $text = preg_replace( "/<\\/?\\s*[A-Za-z][^>]*?>/", + ' ', $wgContLang->lc( " " . $text . " " ) ); # Strip HTML markup + $text = preg_replace( "/(^|\\n)==\\s*([^\\n]+)\\s*==(\\s)/sD", + "\\1\\2 \\2 \\2\\3", $text ); # Emphasize headings + + # Strip external URLs + $uc = "A-Za-z0-9_\\/:.,~%\\-+&;#?!=()@\\x80-\\xFF"; + $protos = "http|https|ftp|mailto|news|gopher"; + $pat = "/(^|[^\\[])({$protos}):[{$uc}]+([^{$uc}]|$)/"; + $text = preg_replace( $pat, "\\1 \\3", $text ); + + $p1 = "/([^\\[])\\[({$protos}):[{$uc}]+]/"; + $p2 = "/([^\\[])\\[({$protos}):[{$uc}]+\\s+([^\\]]+)]/"; + $text = preg_replace( $p1, "\\1 ", $text ); + $text = preg_replace( $p2, "\\1 \\3 ", $text ); + + # Internal image links + $pat2 = "/\\[\\[image:([{$uc}]+)\\.(gif|png|jpg|jpeg)([^{$uc}])/i"; + $text = preg_replace( $pat2, " \\1 \\3", $text ); + + $text = preg_replace( "/([^{$lc}])([{$lc}]+)]]([a-z]+)/", + "\\1\\2 \\2\\3", $text ); # Handle [[game]]s + + # Strip all remaining non-search characters + $text = preg_replace( "/[^{$lc}]+/", " ", $text ); + + # Handle 's, s' + # + # $text = preg_replace( "/([{$lc}]+)'s /", "\\1 \\1's ", $text ); + # $text = preg_replace( "/([{$lc}]+)s' /", "\\1s ", $text ); + # + # These tail-anchored regexps are insanely slow. The worst case comes + # when Japanese or Chinese text (ie, no word spacing) is written on + # a wiki configured for Western UTF-8 mode. The Unicode characters are + # expanded to hex codes and the "words" are very long paragraph-length + # monstrosities. On a large page the above regexps may take over 20 + # seconds *each* on a 1GHz-level processor. + # + # Following are reversed versions which are consistently fast + # (about 3 milliseconds on 1GHz-level processor). + # + $text = strrev( preg_replace( "/ s'([{$lc}]+)/", " s'\\1 \\1", strrev( $text ) ) ); + $text = strrev( preg_replace( "/ 's([{$lc}]+)/", " s\\1", strrev( $text ) ) ); + + # Strip wiki '' and ''' + $text = preg_replace( "/''[']*/", " ", $text ); + wfProfileOut( __METHOD__ . '-regexps' ); + + return $text; + } + + /** + * Get a string representation of a title suitable for + * including in a search index + * + * @param SearchEngine $search + * @return string A stripped-down title string ready for the search index + */ + private function indexTitle( SearchEngine $search ) { + global $wgContLang; + + $ns = $this->title->getNamespace(); + $title = $this->title->getText(); + + $lc = $search->legalSearchChars() . '&#;'; + $t = $wgContLang->normalizeForSearch( $title ); + $t = preg_replace( "/[^{$lc}]+/", ' ', $t ); + $t = $wgContLang->lc( $t ); + + # Handle 's, s' + $t = preg_replace( "/([{$lc}]+)'s( |$)/", "\\1 \\1's ", $t ); + $t = preg_replace( "/([{$lc}]+)s'( |$)/", "\\1s ", $t ); + + $t = preg_replace( "/\\s+/", ' ', $t ); + + if ( $ns == NS_FILE ) { + $t = preg_replace( "/ (png|gif|jpg|jpeg|ogg)$/", "", $t ); + } + return trim( $t ); + } +} diff --git a/includes/deferred/SiteStatsUpdate.php b/includes/deferred/SiteStatsUpdate.php new file mode 100644 index 00000000..7bfafee8 --- /dev/null +++ b/includes/deferred/SiteStatsUpdate.php @@ -0,0 +1,254 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Class for handling updates to the site_stats table + */ +class SiteStatsUpdate implements DeferrableUpdate { + /** @var int */ + protected $views = 0; + + /** @var int */ + protected $edits = 0; + + /** @var int */ + protected $pages = 0; + + /** @var int */ + protected $articles = 0; + + /** @var int */ + protected $users = 0; + + /** @var int */ + protected $images = 0; + + // @todo deprecate this constructor + function __construct( $views, $edits, $good, $pages = 0, $users = 0 ) { + $this->views = $views; + $this->edits = $edits; + $this->articles = $good; + $this->pages = $pages; + $this->users = $users; + } + + /** + * @param array $deltas + * @return SiteStatsUpdate + */ + public static function factory( array $deltas ) { + $update = new self( 0, 0, 0 ); + + $fields = array( 'views', 'edits', 'pages', 'articles', 'users', 'images' ); + foreach ( $fields as $field ) { + if ( isset( $deltas[$field] ) && $deltas[$field] ) { + $update->$field = $deltas[$field]; + } + } + + return $update; + } + + public function doUpdate() { + global $wgSiteStatsAsyncFactor; + + $rate = $wgSiteStatsAsyncFactor; // convenience + // If set to do so, only do actual DB updates 1 every $rate times. + // The other times, just update "pending delta" values in memcached. + if ( $rate && ( $rate < 0 || mt_rand( 0, $rate - 1 ) != 0 ) ) { + $this->doUpdatePendingDeltas(); + } else { + // Need a separate transaction because this a global lock + wfGetDB( DB_MASTER )->onTransactionIdle( array( $this, 'tryDBUpdateInternal' ) ); + } + } + + /** + * Do not call this outside of SiteStatsUpdate + */ + public function tryDBUpdateInternal() { + global $wgSiteStatsAsyncFactor; + + $dbw = wfGetDB( DB_MASTER ); + $lockKey = wfMemcKey( 'site_stats' ); // prepend wiki ID + $pd = array(); + if ( $wgSiteStatsAsyncFactor ) { + // Lock the table so we don't have double DB/memcached updates + if ( !$dbw->lockIsFree( $lockKey, __METHOD__ ) + || !$dbw->lock( $lockKey, __METHOD__, 1 ) // 1 sec timeout + ) { + $this->doUpdatePendingDeltas(); + + return; + } + $pd = $this->getPendingDeltas(); + // Piggy-back the async deltas onto those of this stats update.... + $this->views += ( $pd['ss_total_views']['+'] - $pd['ss_total_views']['-'] ); + $this->edits += ( $pd['ss_total_edits']['+'] - $pd['ss_total_edits']['-'] ); + $this->articles += ( $pd['ss_good_articles']['+'] - $pd['ss_good_articles']['-'] ); + $this->pages += ( $pd['ss_total_pages']['+'] - $pd['ss_total_pages']['-'] ); + $this->users += ( $pd['ss_users']['+'] - $pd['ss_users']['-'] ); + $this->images += ( $pd['ss_images']['+'] - $pd['ss_images']['-'] ); + } + + // Build up an SQL query of deltas and apply them... + $updates = ''; + $this->appendUpdate( $updates, 'ss_total_views', $this->views ); + $this->appendUpdate( $updates, 'ss_total_edits', $this->edits ); + $this->appendUpdate( $updates, 'ss_good_articles', $this->articles ); + $this->appendUpdate( $updates, 'ss_total_pages', $this->pages ); + $this->appendUpdate( $updates, 'ss_users', $this->users ); + $this->appendUpdate( $updates, 'ss_images', $this->images ); + if ( $updates != '' ) { + $dbw->update( 'site_stats', array( $updates ), array(), __METHOD__ ); + } + + if ( $wgSiteStatsAsyncFactor ) { + // Decrement the async deltas now that we applied them + $this->removePendingDeltas( $pd ); + // Commit the updates and unlock the table + $dbw->unlock( $lockKey, __METHOD__ ); + } + } + + /** + * @param DatabaseBase $dbw + * @return bool|mixed + */ + public static function cacheUpdate( $dbw ) { + global $wgActiveUserDays; + $dbr = wfGetDB( DB_SLAVE, array( 'SpecialStatistics', 'vslow' ) ); + # Get non-bot users than did some recent action other than making accounts. + # If account creation is included, the number gets inflated ~20+ fold on enwiki. + $activeUsers = $dbr->selectField( + 'recentchanges', + 'COUNT( DISTINCT rc_user_text )', + array( + 'rc_user != 0', + 'rc_bot' => 0, + 'rc_log_type != ' . $dbr->addQuotes( 'newusers' ) . ' OR rc_log_type IS NULL', + 'rc_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( wfTimestamp( TS_UNIX ) + - $wgActiveUserDays * 24 * 3600 ) ), + ), + __METHOD__ + ); + $dbw->update( + 'site_stats', + array( 'ss_active_users' => intval( $activeUsers ) ), + array( 'ss_row_id' => 1 ), + __METHOD__ + ); + + return $activeUsers; + } + + protected function doUpdatePendingDeltas() { + $this->adjustPending( 'ss_total_views', $this->views ); + $this->adjustPending( 'ss_total_edits', $this->edits ); + $this->adjustPending( 'ss_good_articles', $this->articles ); + $this->adjustPending( 'ss_total_pages', $this->pages ); + $this->adjustPending( 'ss_users', $this->users ); + $this->adjustPending( 'ss_images', $this->images ); + } + + /** + * @param string $sql + * @param string $field + * @param int $delta + */ + protected function appendUpdate( &$sql, $field, $delta ) { + if ( $delta ) { + if ( $sql ) { + $sql .= ','; + } + if ( $delta < 0 ) { + $sql .= "$field=$field-" . abs( $delta ); + } else { + $sql .= "$field=$field+" . abs( $delta ); + } + } + } + + /** + * @param string $type + * @param string $sign ('+' or '-') + * @return string + */ + private function getTypeCacheKey( $type, $sign ) { + return wfMemcKey( 'sitestatsupdate', 'pendingdelta', $type, $sign ); + } + + /** + * Adjust the pending deltas for a stat type. + * Each stat type has two pending counters, one for increments and decrements + * @param string $type + * @param int $delta Delta (positive or negative) + */ + protected function adjustPending( $type, $delta ) { + global $wgMemc; + + if ( $delta < 0 ) { // decrement + $key = $this->getTypeCacheKey( $type, '-' ); + } else { // increment + $key = $this->getTypeCacheKey( $type, '+' ); + } + + $magnitude = abs( $delta ); + if ( !$wgMemc->incr( $key, $magnitude ) ) { // not there? + if ( !$wgMemc->add( $key, $magnitude ) ) { // race? + $wgMemc->incr( $key, $magnitude ); + } + } + } + + /** + * Get pending delta counters for each stat type + * @return array Positive and negative deltas for each type + */ + protected function getPendingDeltas() { + global $wgMemc; + + $pending = array(); + foreach ( array( 'ss_total_views', 'ss_total_edits', + 'ss_good_articles', 'ss_total_pages', 'ss_users', 'ss_images' ) as $type + ) { + // Get pending increments and pending decrements + $pending[$type]['+'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '+' ) ); + $pending[$type]['-'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '-' ) ); + } + + return $pending; + } + + /** + * Reduce pending delta counters after updates have been applied + * @param array $pd Result of getPendingDeltas(), used for DB update + */ + protected function removePendingDeltas( array $pd ) { + global $wgMemc; + + foreach ( $pd as $type => $deltas ) { + foreach ( $deltas as $sign => $magnitude ) { + // Lower the pending counter now that we applied these changes + $wgMemc->decr( $this->getTypeCacheKey( $type, $sign ), $magnitude ); + } + } + } +} diff --git a/includes/deferred/SqlDataUpdate.php b/includes/deferred/SqlDataUpdate.php new file mode 100644 index 00000000..9c58503f --- /dev/null +++ b/includes/deferred/SqlDataUpdate.php @@ -0,0 +1,159 @@ +<?php +/** + * Base code for update jobs that put some secondary data extracted + * from article content into the database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Abstract base class for update jobs that put some secondary data extracted + * from article content into the database. + * + * @note subclasses should NOT start or commit transactions in their doUpdate() method, + * a transaction will automatically be wrapped around the update. Starting another + * one would break the outer transaction bracket. If need be, subclasses can override + * the beginTransaction() and commitTransaction() methods. + */ +abstract class SqlDataUpdate extends DataUpdate { + /** @var DatabaseBase Database connection reference */ + protected $mDb; + + /** @var array SELECT options to be used (array) */ + protected $mOptions; + + /** @var bool Whether a transaction is open on this object (internal use only!) */ + private $mHasTransaction; + + /** @var bool Whether this update should be wrapped in a transaction */ + protected $mUseTransaction; + + /** + * Constructor + * + * @param bool $withTransaction Whether this update should be wrapped in a + * transaction (default: true). A transaction is only started if no + * transaction is already in progress, see beginTransaction() for details. + */ + public function __construct( $withTransaction = true ) { + global $wgAntiLockFlags; + + parent::__construct(); + + if ( $wgAntiLockFlags & ALF_NO_LINK_LOCK ) { + $this->mOptions = array(); + } else { + $this->mOptions = array( 'FOR UPDATE' ); + } + + // @todo Get connection only when it's needed? Make sure that doesn't + // break anything, especially transactions! + $this->mDb = wfGetDB( DB_MASTER ); + + $this->mWithTransaction = $withTransaction; + $this->mHasTransaction = false; + } + + /** + * Begin a database transaction, if $withTransaction was given as true in + * the constructor for this SqlDataUpdate. + * + * Because nested transactions are not supported by the Database class, + * this implementation checks Database::trxLevel() and only opens a + * transaction if none is already active. + */ + public function beginTransaction() { + if ( !$this->mWithTransaction ) { + return; + } + + // NOTE: nested transactions are not supported, only start a transaction if none is open + if ( $this->mDb->trxLevel() === 0 ) { + $this->mDb->begin( get_class( $this ) . '::beginTransaction' ); + $this->mHasTransaction = true; + } + } + + /** + * Commit the database transaction started via beginTransaction (if any). + */ + public function commitTransaction() { + if ( $this->mHasTransaction ) { + $this->mDb->commit( get_class( $this ) . '::commitTransaction' ); + $this->mHasTransaction = false; + } + } + + /** + * Abort the database transaction started via beginTransaction (if any). + */ + public function abortTransaction() { + if ( $this->mHasTransaction ) { //XXX: actually... maybe always? + $this->mDb->rollback( get_class( $this ) . '::abortTransaction' ); + $this->mHasTransaction = false; + } + } + + /** + * Invalidate the cache of a list of pages from a single namespace. + * This is intended for use by subclasses. + * + * @param int $namespace Namespace number + * @param array $dbkeys + */ + protected function invalidatePages( $namespace, array $dbkeys ) { + if ( $dbkeys === array() ) { + return; + } + + /** + * Determine which pages need to be updated + * This is necessary to prevent the job queue from smashing the DB with + * large numbers of concurrent invalidations of the same page + */ + $now = $this->mDb->timestamp(); + $ids = array(); + $res = $this->mDb->select( 'page', array( 'page_id' ), + array( + 'page_namespace' => $namespace, + 'page_title' => $dbkeys, + 'page_touched < ' . $this->mDb->addQuotes( $now ) + ), __METHOD__ + ); + + foreach ( $res as $row ) { + $ids[] = $row->page_id; + } + + if ( $ids === array() ) { + return; + } + + /** + * Do the update + * We still need the page_touched condition, in case the row has changed since + * the non-locking select above. + */ + $this->mDb->update( 'page', array( 'page_touched' => $now ), + array( + 'page_id' => $ids, + 'page_touched < ' . $this->mDb->addQuotes( $now ) + ), __METHOD__ + ); + } +} diff --git a/includes/deferred/SquidUpdate.php b/includes/deferred/SquidUpdate.php new file mode 100644 index 00000000..0dcff44a --- /dev/null +++ b/includes/deferred/SquidUpdate.php @@ -0,0 +1,311 @@ +<?php +/** + * Squid cache purging. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * Handles purging appropriate Squid URLs given a title (or titles) + * @ingroup Cache + */ +class SquidUpdate { + /** + * Collection of URLs to purge. + * @var array + */ + protected $urlArr; + + /** + * @param array $urlArr Collection of URLs to purge + * @param bool|int $maxTitles Maximum number of unique URLs to purge + */ + public function __construct( $urlArr = array(), $maxTitles = false ) { + global $wgMaxSquidPurgeTitles; + if ( $maxTitles === false ) { + $maxTitles = $wgMaxSquidPurgeTitles; + } + + // Remove duplicate URLs from list + $urlArr = array_unique( $urlArr ); + if ( count( $urlArr ) > $maxTitles ) { + // Truncate to desired maximum URL count + $urlArr = array_slice( $urlArr, 0, $maxTitles ); + } + $this->urlArr = $urlArr; + } + + /** + * Create a SquidUpdate from the given Title object. + * + * The resulting SquidUpdate will purge the given Title's URLs as well as + * the pages that link to it. Capped at $wgMaxSquidPurgeTitles total URLs. + * + * @param Title $title + * @return SquidUpdate + */ + public static function newFromLinksTo( Title $title ) { + global $wgMaxSquidPurgeTitles; + wfProfileIn( __METHOD__ ); + + # Get a list of URLs linking to this page + $dbr = wfGetDB( DB_SLAVE ); + $res = $dbr->select( array( 'links', 'page' ), + array( 'page_namespace', 'page_title' ), + array( + 'pl_namespace' => $title->getNamespace(), + 'pl_title' => $title->getDBkey(), + 'pl_from=page_id' ), + __METHOD__ ); + $blurlArr = $title->getSquidURLs(); + if ( $res->numRows() <= $wgMaxSquidPurgeTitles ) { + foreach ( $res as $BL ) { + $tobj = Title::makeTitle( $BL->page_namespace, $BL->page_title ); + $blurlArr[] = $tobj->getInternalURL(); + } + } + + wfProfileOut( __METHOD__ ); + + return new SquidUpdate( $blurlArr ); + } + + /** + * Create a SquidUpdate from an array of Title objects, or a TitleArray object + * + * @param array $titles + * @param array $urlArr + * @return SquidUpdate + */ + public static function newFromTitles( $titles, $urlArr = array() ) { + global $wgMaxSquidPurgeTitles; + $i = 0; + /** @var Title $title */ + foreach ( $titles as $title ) { + $urlArr[] = $title->getInternalURL(); + if ( $i++ > $wgMaxSquidPurgeTitles ) { + break; + } + } + + return new SquidUpdate( $urlArr ); + } + + /** + * @param Title $title + * @return SquidUpdate + */ + public static function newSimplePurge( Title $title ) { + $urlArr = $title->getSquidURLs(); + + return new SquidUpdate( $urlArr ); + } + + /** + * Purges the list of URLs passed to the constructor. + */ + public function doUpdate() { + self::purge( $this->urlArr ); + } + + /** + * Purges a list of Squids defined in $wgSquidServers. + * $urlArr should contain the full URLs to purge as values + * (example: $urlArr[] = 'http://my.host/something') + * XXX report broken Squids per mail or log + * + * @param array $urlArr List of full URLs to purge + */ + public static function purge( $urlArr ) { + global $wgSquidServers, $wgHTCPRouting; + + if ( !$urlArr ) { + return; + } + + wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) ); + + if ( $wgHTCPRouting ) { + self::HTCPPurge( $urlArr ); + } + + wfProfileIn( __METHOD__ ); + + // Remove duplicate URLs + $urlArr = array_unique( $urlArr ); + // Maximum number of parallel connections per squid + $maxSocketsPerSquid = 8; + // Number of requests to send per socket + // 400 seems to be a good tradeoff, opening a socket takes a while + $urlsPerSocket = 400; + $socketsPerSquid = ceil( count( $urlArr ) / $urlsPerSocket ); + if ( $socketsPerSquid > $maxSocketsPerSquid ) { + $socketsPerSquid = $maxSocketsPerSquid; + } + + $pool = new SquidPurgeClientPool; + $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSquid ) ); + foreach ( $wgSquidServers as $server ) { + foreach ( $chunks as $chunk ) { + $client = new SquidPurgeClient( $server ); + foreach ( $chunk as $url ) { + $client->queuePurge( $url ); + } + $pool->addClient( $client ); + } + } + $pool->run(); + + wfProfileOut( __METHOD__ ); + } + + /** + * Send Hyper Text Caching Protocol (HTCP) CLR requests. + * + * @throws MWException + * @param array $urlArr Collection of URLs to purge + */ + public static function HTCPPurge( $urlArr ) { + global $wgHTCPRouting, $wgHTCPMulticastTTL; + wfProfileIn( __METHOD__ ); + + // HTCP CLR operation + $htcpOpCLR = 4; + + // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h) + if ( !defined( "IPPROTO_IP" ) ) { + define( "IPPROTO_IP", 0 ); + define( "IP_MULTICAST_LOOP", 34 ); + define( "IP_MULTICAST_TTL", 33 ); + } + + // pfsockopen doesn't work because we need set_sock_opt + $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP ); + if ( !$conn ) { + $errstr = socket_strerror( socket_last_error() ); + wfDebugLog( 'squid', __METHOD__ . + ": Error opening UDP socket: $errstr" ); + wfProfileOut( __METHOD__ ); + + return; + } + + // Set socket options + socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 ); + if ( $wgHTCPMulticastTTL != 1 ) { + // Set multicast time to live (hop count) option on socket + socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL, + $wgHTCPMulticastTTL ); + } + + // Remove duplicate URLs from collection + $urlArr = array_unique( $urlArr ); + // Get sequential trx IDs for packet loss counting + $ids = UIDGenerator::newSequentialPerNodeIDs( + 'squidhtcppurge', 32, count( $urlArr ), UIDGenerator::QUICK_VOLATILE + ); + + foreach ( $urlArr as $url ) { + if ( !is_string( $url ) ) { + wfProfileOut( __METHOD__ ); + throw new MWException( 'Bad purge URL' ); + } + $url = self::expand( $url ); + $conf = self::getRuleForURL( $url, $wgHTCPRouting ); + if ( !$conf ) { + wfDebugLog( 'squid', __METHOD__ . + "No HTCP rule configured for URL {$url} , skipping" ); + continue; + } + + if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) { + // Normalize single entries + $conf = array( $conf ); + } + foreach ( $conf as $subconf ) { + if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) { + wfProfileOut( __METHOD__ ); + throw new MWException( "Invalid HTCP rule for URL $url\n" ); + } + } + + // Construct a minimal HTCP request diagram + // as per RFC 2756 + // Opcode 'CLR', no response desired, no auth + $htcpTransID = current( $ids ); + next( $ids ); + + $htcpSpecifier = pack( 'na4na*na8n', + 4, 'HEAD', strlen( $url ), $url, + 8, 'HTTP/1.0', 0 ); + + $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier ); + $htcpLen = 4 + $htcpDataLen + 2; + + // Note! Squid gets the bit order of the first + // word wrong, wrt the RFC. Apparently no other + // implementation exists, so adapt to Squid + $htcpPacket = pack( 'nxxnCxNxxa*n', + $htcpLen, $htcpDataLen, $htcpOpCLR, + $htcpTransID, $htcpSpecifier, 2 ); + + wfDebugLog( 'squid', __METHOD__ . + "Purging URL $url via HTCP" ); + foreach ( $conf as $subconf ) { + socket_sendto( $conn, $htcpPacket, $htcpLen, 0, + $subconf['host'], $subconf['port'] ); + } + } + wfProfileOut( __METHOD__ ); + } + + /** + * Expand local URLs to fully-qualified URLs using the internal protocol + * and host defined in $wgInternalServer. Input that's already fully- + * qualified will be passed through unchanged. + * + * This is used to generate purge URLs that may be either local to the + * main wiki or include a non-native host, such as images hosted on a + * second internal server. + * + * Client functions should not need to call this. + * + * @param string $url + * @return string + */ + public static function expand( $url ) { + return wfExpandUrl( $url, PROTO_INTERNAL ); + } + + /** + * Find the HTCP routing rule to use for a given URL. + * @param string $url URL to match + * @param array $rules Array of rules, see $wgHTCPRouting for format and behavior + * @return mixed Element of $rules that matched, or false if nothing matched + */ + private static function getRuleForURL( $url, $rules ) { + foreach ( $rules as $regex => $routing ) { + if ( $regex === '' || preg_match( $regex, $url ) ) { + return $routing; + } + } + + return false; + } +} diff --git a/includes/deferred/ViewCountUpdate.php b/includes/deferred/ViewCountUpdate.php new file mode 100644 index 00000000..8282295b --- /dev/null +++ b/includes/deferred/ViewCountUpdate.php @@ -0,0 +1,119 @@ +<?php +/** + * Update for the 'page_counter' field + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Update for the 'page_counter' field, when $wgDisableCounters is false. + * + * Depending on $wgHitcounterUpdateFreq, this will directly increment the + * 'page_counter' field or use the 'hitcounter' table and then collect the data + * from that table to update the 'page_counter' field in a batch operation. + */ +class ViewCountUpdate implements DeferrableUpdate { + /** @var int Page ID to increment the view count */ + protected $id; + + /** + * Constructor + * + * @param int $id Page ID to increment the view count + */ + public function __construct( $id ) { + $this->id = intval( $id ); + } + + /** + * Run the update + */ + public function doUpdate() { + global $wgHitcounterUpdateFreq; + + $dbw = wfGetDB( DB_MASTER ); + + if ( $wgHitcounterUpdateFreq <= 1 || $dbw->getType() == 'sqlite' ) { + $id = $this->id; + $method = __METHOD__; + $dbw->onTransactionIdle( function () use ( $dbw, $id, $method ) { + try { + $dbw->update( 'page', + array( 'page_counter = page_counter + 1' ), + array( 'page_id' => $id ), + $method + ); + } catch ( DBError $e ) { + MWExceptionHandler::logException( $e ); + } + } ); + return; + } + + # Not important enough to warrant an error page in case of failure + try { + // Since `hitcounter` is non-transactional, the contention is minimal + $dbw->insert( 'hitcounter', array( 'hc_id' => $this->id ), __METHOD__ ); + $checkfreq = intval( $wgHitcounterUpdateFreq / 25 + 1 ); + if ( rand() % $checkfreq == 0 && $dbw->lastErrno() == 0 ) { + $this->collect(); + } + } catch ( DBError $e ) { + MWExceptionHandler::logException( $e ); + } + } + + protected function collect() { + global $wgHitcounterUpdateFreq; + + $dbw = wfGetDB( DB_MASTER ); + + $rown = $dbw->selectField( 'hitcounter', 'COUNT(*)', array(), __METHOD__ ); + if ( $rown < $wgHitcounterUpdateFreq ) { + return; + } + + wfProfileIn( __METHOD__ . '-collect' ); + $old_user_abort = ignore_user_abort( true ); + + $dbType = $dbw->getType(); + $tabletype = $dbType == 'mysql' ? "ENGINE=HEAP " : ''; + $hitcounterTable = $dbw->tableName( 'hitcounter' ); + $acchitsTable = $dbw->tableName( 'acchits' ); + $pageTable = $dbw->tableName( 'page' ); + + $dbw->lockTables( array(), array( 'hitcounter' ), __METHOD__, false ); + $dbw->query( "CREATE TEMPORARY TABLE $acchitsTable $tabletype AS " . + "SELECT hc_id,COUNT(*) AS hc_n FROM $hitcounterTable " . + 'GROUP BY hc_id', __METHOD__ ); + $dbw->delete( 'hitcounter', '*', __METHOD__ ); + $dbw->unlockTables( __METHOD__ ); + + if ( $dbType == 'mysql' ) { + $dbw->query( "UPDATE $pageTable,$acchitsTable SET page_counter=page_counter + hc_n " . + 'WHERE page_id = hc_id', __METHOD__ ); + } else { + $dbw->query( "UPDATE $pageTable SET page_counter=page_counter + hc_n " . + "FROM $acchitsTable WHERE page_id = hc_id", __METHOD__ ); + } + $dbw->query( "DROP TABLE $acchitsTable", __METHOD__ ); + + ignore_user_abort( $old_user_abort ); + wfProfileOut( __METHOD__ . '-collect' ); + } +} |