diff options
Diffstat (limited to 'includes/job')
-rw-r--r-- | includes/job/DoubleRedirectJob.php | 30 | ||||
-rw-r--r-- | includes/job/EmaillingJob.php | 15 | ||||
-rw-r--r-- | includes/job/EnotifNotifyJob.php | 15 | ||||
-rw-r--r-- | includes/job/Job.php (renamed from includes/job/JobQueue.php) | 92 | ||||
-rw-r--r-- | includes/job/RefreshLinksJob.php | 162 | ||||
-rw-r--r-- | includes/job/UploadFromUrlJob.php | 33 |
6 files changed, 257 insertions, 90 deletions
diff --git a/includes/job/DoubleRedirectJob.php b/includes/job/DoubleRedirectJob.php index 2b7cd7c8..08af9975 100644 --- a/includes/job/DoubleRedirectJob.php +++ b/includes/job/DoubleRedirectJob.php @@ -1,6 +1,21 @@ <?php /** - * Job to fix double redirects after moving a page + * Job to fix double redirects after moving a page. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html * * @file * @ingroup JobQueue @@ -21,7 +36,7 @@ class DoubleRedirectJob extends Job { /** * Insert jobs into the job queue to fix redirects to the given title - * @param $reason String: the reason for the fix, see message double-redirect-fixed-<reason> + * @param $reason String: the reason for the fix, see message "double-redirect-fixed-<reason>" * @param $redirTitle Title: the title which has changed, redirects pointing to this title are fixed * @param $destTitle bool Not used */ @@ -74,7 +89,7 @@ class DoubleRedirectJob extends Job { return false; } - $targetRev = Revision::newFromTitle( $this->title ); + $targetRev = Revision::newFromTitle( $this->title, false, Revision::READ_LATEST ); if ( !$targetRev ) { wfDebug( __METHOD__.": target redirect already deleted, ignoring\n" ); return true; @@ -126,8 +141,9 @@ class DoubleRedirectJob extends Job { $oldUser = $wgUser; $wgUser = $this->getUser(); $article = WikiPage::factory( $this->title ); - $reason = wfMsgForContent( 'double-redirect-fixed-' . $this->reason, - $this->redirTitle->getPrefixedText(), $newTitle->getPrefixedText() ); + $reason = wfMessage( 'double-redirect-fixed-' . $this->reason, + $this->redirTitle->getPrefixedText(), $newTitle->getPrefixedText() + )->inContentLanguage()->text(); $article->doEdit( $newText, $reason, EDIT_UPDATE | EDIT_SUPPRESS_RC, false, $this->getUser() ); $wgUser = $oldUser; @@ -139,7 +155,7 @@ class DoubleRedirectJob extends Job { * * @param $title Title * - * @return false if the specified title is not a redirect, or if it is a circular redirect + * @return bool if the specified title is not a redirect, or if it is a circular redirect */ public static function getFinalDestination( $title ) { $dbw = wfGetDB( DB_MASTER ); @@ -179,7 +195,7 @@ class DoubleRedirectJob extends Job { */ function getUser() { if ( !self::$user ) { - self::$user = User::newFromName( wfMsgForContent( 'double-redirect-fixer' ), false ); + self::$user = User::newFromName( wfMessage( 'double-redirect-fixer' )->inContentLanguage()->text(), false ); # FIXME: newFromName could return false on a badly configured wiki. if ( !self::$user->isLoggedIn() ) { self::$user->addToDatabase(); diff --git a/includes/job/EmaillingJob.php b/includes/job/EmaillingJob.php index 89b74a41..d3599882 100644 --- a/includes/job/EmaillingJob.php +++ b/includes/job/EmaillingJob.php @@ -2,6 +2,21 @@ /** * Old job for notification emails. * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * * @file * @ingroup JobQueue */ diff --git a/includes/job/EnotifNotifyJob.php b/includes/job/EnotifNotifyJob.php index eb154ece..b4c925e9 100644 --- a/includes/job/EnotifNotifyJob.php +++ b/includes/job/EnotifNotifyJob.php @@ -2,6 +2,21 @@ /** * Job for notification emails. * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * * @file * @ingroup JobQueue */ diff --git a/includes/job/JobQueue.php b/includes/job/Job.php index e7c66719..d777a5d4 100644 --- a/includes/job/JobQueue.php +++ b/includes/job/Job.php @@ -1,15 +1,26 @@ <?php /** - * Job queue base code + * Job queue base code. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html * * @file * @defgroup JobQueue JobQueue */ -if ( !defined( 'MEDIAWIKI' ) ) { - die( "This file is part of MediaWiki, it is not a valid entry point\n" ); -} - /** * Class to both describe a background job and handle jobs. * @@ -56,7 +67,7 @@ abstract class Job { $dbw = wfGetDB( DB_MASTER ); - $dbw->begin(); + $dbw->begin( __METHOD__ ); $row = $dbw->selectRow( 'job', @@ -67,7 +78,7 @@ abstract class Job { ); if ( $row === false ) { - $dbw->commit(); + $dbw->commit( __METHOD__ ); wfProfileOut( __METHOD__ ); return false; } @@ -75,7 +86,7 @@ abstract class Job { /* Ensure we "own" this row */ $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); $affected = $dbw->affectedRows(); - $dbw->commit(); + $dbw->commit( __METHOD__ ); if ( $affected == 0 ) { wfProfileOut( __METHOD__ ); @@ -102,7 +113,6 @@ abstract class Job { * @return Job or false if there's no jobs */ static function pop( $offset = 0 ) { - global $wgJobTypesExcludedFromDefaultQueue; wfProfileIn( __METHOD__ ); $dbr = wfGetDB( DB_SLAVE ); @@ -113,12 +123,9 @@ abstract class Job { NB: If random fetch previously was used, offset will always be ahead of few entries */ - $conditions = array(); - if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) { - foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) { - $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType ); - } - } + + $conditions = self::defaultQueueConditions(); + $offset = intval( $offset ); $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' ); @@ -146,13 +153,12 @@ abstract class Job { $dbw = wfGetDB( DB_MASTER ); $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); $affected = $dbw->affectedRows(); - $dbw->commit(); if ( !$affected ) { // Failed, someone else beat us to it // Try getting a random row - $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob', - 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ ); + $row = $dbw->selectRow( 'job', array( 'minjob' => 'MIN(job_id)', + 'maxjob' => 'MAX(job_id)' ), '1=1', __METHOD__ ); if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) { // No jobs to get wfProfileOut( __METHOD__ ); @@ -170,7 +176,6 @@ abstract class Job { // Delete the random row $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); $affected = $dbw->affectedRows(); - $dbw->commit(); if ( !$affected ) { // Random job gone before we exclusively deleted it @@ -186,6 +191,12 @@ abstract class Job { $namespace = $row->job_namespace; $dbkey = $row->job_title; $title = Title::makeTitleSafe( $namespace, $dbkey ); + + if ( is_null( $title ) ) { + wfProfileOut( __METHOD__ ); + return false; + } + $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id ); // Remove any duplicates it may have later in the queue @@ -200,11 +211,12 @@ abstract class Job { * * @param $command String: Job command * @param $title Title: Associated title - * @param $params Array: Job parameters + * @param $params Array|bool: Job parameters * @param $id Int: Job identifier + * @throws MWException * @return Job */ - static function factory( $command, $title, $params = false, $id = 0 ) { + static function factory( $command, Title $title, $params = false, $id = 0 ) { global $wgJobClasses; if( isset( $wgJobClasses[$command] ) ) { $class = $wgJobClasses[$command]; @@ -260,16 +272,16 @@ abstract class Job { $rows[] = $job->insertFields(); if ( count( $rows ) >= 50 ) { # Do a small transaction to avoid slave lag - $dbw->begin(); + $dbw->begin( __METHOD__ ); $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); - $dbw->commit(); + $dbw->commit( __METHOD__ ); $rows = array(); } } if ( $rows ) { // last chunk - $dbw->begin(); + $dbw->begin( __METHOD__ ); $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); - $dbw->commit(); + $dbw->commit( __METHOD__ ); } wfIncrStats( 'job-insert', count( $jobs ) ); } @@ -302,6 +314,27 @@ abstract class Job { wfIncrStats( 'job-insert', count( $jobs ) ); } + + /** + * SQL conditions to apply on most JobQueue queries + * + * Whenever we exclude jobs types from the default queue, we want to make + * sure that queries to the job queue actually ignore them. + * + * @return array SQL conditions suitable for Database:: methods + */ + static function defaultQueueConditions( ) { + global $wgJobTypesExcludedFromDefaultQueue; + $conditions = array(); + if ( count( $wgJobTypesExcludedFromDefaultQueue ) > 0 ) { + $dbr = wfGetDB( DB_SLAVE ); + foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) { + $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType ); + } + } + return $conditions; + } + /*------------------------------------------------------------------------- * Non-static functions *------------------------------------------------------------------------*/ @@ -309,8 +342,8 @@ abstract class Job { /** * @param $command * @param $title - * @param $params array - * @param int $id + * @param $params array|bool + * @param $id int */ function __construct( $command, $title, $params = false, $id = 0 ) { $this->command = $command; @@ -368,11 +401,12 @@ abstract class Job { $fields = $this->insertFields(); unset( $fields['job_id'] ); + unset( $fields['job_timestamp'] ); $dbw = wfGetDB( DB_MASTER ); - $dbw->begin(); + $dbw->begin( __METHOD__ ); $dbw->delete( 'job', $fields, __METHOD__ ); $affected = $dbw->affectedRows(); - $dbw->commit(); + $dbw->commit( __METHOD__ ); if ( $affected ) { wfIncrStats( 'job-dup-delete', $affected ); } diff --git a/includes/job/RefreshLinksJob.php b/includes/job/RefreshLinksJob.php index 1aa206f0..b23951c6 100644 --- a/includes/job/RefreshLinksJob.php +++ b/includes/job/RefreshLinksJob.php @@ -2,6 +2,21 @@ /** * Job to update links for a given title. * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * * @file * @ingroup JobQueue */ @@ -22,7 +37,6 @@ class RefreshLinksJob extends Job { * @return boolean success */ function run() { - global $wgParser, $wgContLang; wfProfileIn( __METHOD__ ); $linkCache = LinkCache::singleton(); @@ -34,24 +48,41 @@ class RefreshLinksJob extends Job { return false; } - $revision = Revision::newFromTitle( $this->title ); + # Wait for the DB of the current/next slave DB handle to catch up to the master. + # This way, we get the correct page_latest for templates or files that just changed + # milliseconds ago, having triggered this job to begin with. + if ( isset( $this->params['masterPos'] ) ) { + wfGetLB()->waitFor( $this->params['masterPos'] ); + } + + $revision = Revision::newFromTitle( $this->title, false, Revision::READ_NORMAL ); if ( !$revision ) { - $this->error = 'refreshLinks: Article not found "' . $this->title->getPrefixedDBkey() . '"'; + $this->error = 'refreshLinks: Article not found "' . + $this->title->getPrefixedDBkey() . '"'; wfProfileOut( __METHOD__ ); - return false; + return false; // XXX: what if it was just deleted? } - wfProfileIn( __METHOD__.'-parse' ); - $options = ParserOptions::newFromUserAndLang( new User, $wgContLang ); - $parserOutput = $wgParser->parse( $revision->getText(), $this->title, $options, true, true, $revision->getId() ); - wfProfileOut( __METHOD__.'-parse' ); - wfProfileIn( __METHOD__.'-update' ); - $update = new LinksUpdate( $this->title, $parserOutput, false ); - $update->doUpdate(); - wfProfileOut( __METHOD__.'-update' ); + self::runForTitleInternal( $this->title, $revision, __METHOD__ ); + wfProfileOut( __METHOD__ ); return true; } + + public static function runForTitleInternal( Title $title, Revision $revision, $fname ) { + global $wgParser, $wgContLang; + + wfProfileIn( $fname . '-parse' ); + $options = ParserOptions::newFromUserAndLang( new User, $wgContLang ); + $parserOutput = $wgParser->parse( + $revision->getText(), $title, $options, true, true, $revision->getId() ); + wfProfileOut( $fname . '-parse' ); + + wfProfileIn( $fname . '-update' ); + $updates = $parserOutput->getSecondaryDataUpdates( $title, false ); + DataUpdate::runUpdates( $updates ); + wfProfileOut( $fname . '-update' ); + } } /** @@ -61,6 +92,7 @@ class RefreshLinksJob extends Job { * @ingroup JobQueue */ class RefreshLinksJob2 extends Job { + const MAX_TITLES_RUN = 10; function __construct( $title, $params, $id = 0 ) { parent::__construct( 'refreshLinks2', $title, $params, $id ); @@ -71,60 +103,100 @@ class RefreshLinksJob2 extends Job { * @return boolean success */ function run() { - global $wgParser, $wgContLang; - wfProfileIn( __METHOD__ ); $linkCache = LinkCache::singleton(); $linkCache->clear(); - if( is_null( $this->title ) ) { + if ( is_null( $this->title ) ) { $this->error = "refreshLinks2: Invalid title"; wfProfileOut( __METHOD__ ); return false; - } - if( !isset($this->params['start']) || !isset($this->params['end']) ) { + } elseif ( !isset( $this->params['start'] ) || !isset( $this->params['end'] ) ) { $this->error = "refreshLinks2: Invalid params"; wfProfileOut( __METHOD__ ); return false; } + // Back compat for pre-r94435 jobs $table = isset( $this->params['table'] ) ? $this->params['table'] : 'templatelinks'; - $titles = $this->title->getBacklinkCache()->getLinks( - $table, $this->params['start'], $this->params['end']); - - # Not suitable for page load triggered job running! - # Gracefully switch to refreshLinks jobs if this happens. - if( php_sapi_name() != 'cli' ) { + + // Avoid slave lag when fetching templates + if ( isset( $this->params['masterPos'] ) ) { + $masterPos = $this->params['masterPos']; + } elseif ( wfGetLB()->getServerCount() > 1 ) { + $masterPos = wfGetLB()->getMasterPos(); + } else { + $masterPos = false; + } + + $titles = $this->title->getBacklinkCache()->getLinks( + $table, $this->params['start'], $this->params['end'] ); + + if ( $titles->count() > self::MAX_TITLES_RUN ) { + # We don't want to parse too many pages per job as it can starve other jobs. + # If there are too many pages to parse, break this up into smaller jobs. By passing + # in the master position here we can cut down on the time spent waiting for slaves to + # catch up by the runners handling these jobs since time will have passed between now + # and when they pop these jobs off the queue. + $start = 0; // batch start + $end = 0; // batch end + $bsize = 0; // batch size + $first = true; // first of batch + $jobs = array(); + foreach ( $titles as $title ) { + $start = $first ? $title->getArticleId() : $start; + $end = $title->getArticleId(); + $first = false; + if ( ++$bsize >= self::MAX_TITLES_RUN ) { + $jobs[] = new RefreshLinksJob2( $this->title, array( + 'table' => $table, + 'start' => $start, + 'end' => $end, + 'masterPos' => $masterPos + ) ); + $first = true; + $start = $end = $bsize = 0; + } + } + if ( $bsize > 0 ) { // group remaining pages into a job + $jobs[] = new RefreshLinksJob2( $this->title, array( + 'table' => $table, + 'start' => $start, + 'end' => $end, + 'masterPos' => $masterPos + ) ); + } + Job::batchInsert( $jobs ); + } elseif ( php_sapi_name() != 'cli' ) { + # Not suitable for page load triggered job running! + # Gracefully switch to refreshLinks jobs if this happens. $jobs = array(); foreach ( $titles as $title ) { - $jobs[] = new RefreshLinksJob( $title, '' ); + $jobs[] = new RefreshLinksJob( $title, array( 'masterPos' => $masterPos ) ); } Job::batchInsert( $jobs ); - - wfProfileOut( __METHOD__ ); - return true; - } - $options = ParserOptions::newFromUserAndLang( new User, $wgContLang ); - # Re-parse each page that transcludes this page and update their tracking links... - foreach ( $titles as $title ) { - $revision = Revision::newFromTitle( $title ); - if ( !$revision ) { - $this->error = 'refreshLinks: Article not found "' . $title->getPrefixedDBkey() . '"'; - wfProfileOut( __METHOD__ ); - return false; + } else { + # Wait for the DB of the current/next slave DB handle to catch up to the master. + # This way, we get the correct page_latest for templates or files that just changed + # milliseconds ago, having triggered this job to begin with. + if ( $masterPos ) { + wfGetLB()->waitFor( $masterPos ); + } + # Re-parse each page that transcludes this page and update their tracking links... + foreach ( $titles as $title ) { + $revision = Revision::newFromTitle( $title, false, Revision::READ_NORMAL ); + if ( !$revision ) { + $this->error = 'refreshLinks: Article not found "' . + $title->getPrefixedDBkey() . '"'; + continue; // skip this page + } + RefreshLinksJob::runForTitleInternal( $title, $revision, __METHOD__ ); + wfWaitForSlaves(); } - wfProfileIn( __METHOD__.'-parse' ); - $parserOutput = $wgParser->parse( $revision->getText(), $title, $options, true, true, $revision->getId() ); - wfProfileOut( __METHOD__.'-parse' ); - wfProfileIn( __METHOD__.'-update' ); - $update = new LinksUpdate( $title, $parserOutput, false ); - $update->doUpdate(); - wfProfileOut( __METHOD__.'-update' ); - wfWaitForSlaves(); } - wfProfileOut( __METHOD__ ); + wfProfileOut( __METHOD__ ); return true; } } diff --git a/includes/job/UploadFromUrlJob.php b/includes/job/UploadFromUrlJob.php index 26f6e4ba..e06f68e4 100644 --- a/includes/job/UploadFromUrlJob.php +++ b/includes/job/UploadFromUrlJob.php @@ -2,6 +2,21 @@ /** * Job for asynchronous upload-by-url. * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * * @file * @ingroup JobQueue */ @@ -67,10 +82,10 @@ class UploadFromUrlJob extends Job { if ( $this->params['leaveMessage'] ) { $this->user->leaveUserMessage( - wfMsg( 'upload-warning-subj' ), - wfMsg( 'upload-warning-msg', + wfMessage( 'upload-warning-subj' )->text(), + wfMessage( 'upload-warning-msg', $key, - $this->params['url'] ) + $this->params['url'] )->text() ); } else { wfSetupSession( $this->params['sessionId'] ); @@ -104,17 +119,17 @@ class UploadFromUrlJob extends Job { protected function leaveMessage( $status ) { if ( $this->params['leaveMessage'] ) { if ( $status->isGood() ) { - $this->user->leaveUserMessage( wfMsg( 'upload-success-subj' ), - wfMsg( 'upload-success-msg', + $this->user->leaveUserMessage( wfMessage( 'upload-success-subj' )->text(), + wfMessage( 'upload-success-msg', $this->upload->getTitle()->getText(), $this->params['url'] - ) ); + )->text() ); } else { - $this->user->leaveUserMessage( wfMsg( 'upload-failure-subj' ), - wfMsg( 'upload-failure-msg', + $this->user->leaveUserMessage( wfMessage( 'upload-failure-subj' )->text(), + wfMessage( 'upload-failure-msg', $status->getWikiText(), $this->params['url'] - ) ); + )->text() ); } } else { wfSetupSession( $this->params['sessionId'] ); |