diff options
Diffstat (limited to 'includes/jobqueue/jobs')
| -rw-r--r-- | includes/jobqueue/jobs/AssembleUploadChunksJob.php | 19 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/DuplicateJob.php | 2 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/EnqueueJob.php | 88 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/HTMLCacheUpdateJob.php | 46 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/NullJob.php | 2 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/PublishStashedFileJob.php | 22 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/RecentChangesUpdateJob.php | 223 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/RefreshLinksJob.php | 16 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/RefreshLinksJob2.php | 141 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/ThumbnailRenderJob.php | 109 | ||||
| -rw-r--r-- | includes/jobqueue/jobs/UploadFromUrlJob.php | 2 |
11 files changed, 462 insertions, 208 deletions
diff --git a/includes/jobqueue/jobs/AssembleUploadChunksJob.php b/includes/jobqueue/jobs/AssembleUploadChunksJob.php index 9e9bda6f..b7f09e77 100644 --- a/includes/jobqueue/jobs/AssembleUploadChunksJob.php +++ b/includes/jobqueue/jobs/AssembleUploadChunksJob.php @@ -35,26 +35,16 @@ class AssembleUploadChunksJob extends Job { public function run() { $scope = RequestContext::importScopedSession( $this->params['session'] ); $context = RequestContext::getMain(); + $user = $context->getUser(); try { - $user = $context->getUser(); if ( !$user->isLoggedIn() ) { $this->setLastError( "Could not load the author user from session." ); return false; } - if ( count( $_SESSION ) === 0 ) { - // Empty session probably indicates that we didn't associate - // with the session correctly. Note that being able to load - // the user does not necessarily mean the session was loaded. - // Most likely cause by suhosin.session.encrypt = On. - $this->setLastError( "Error associating with user session. " . - "Try setting suhosin.session.encrypt = Off" ); - - return false; - } - UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Poll', 'stage' => 'assembling', 'status' => Status::newGood() ) ); @@ -70,6 +60,7 @@ class AssembleUploadChunksJob extends Job { $status = $upload->concatenateChunks(); if ( !$status->isGood() ) { UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Failure', 'stage' => 'assembling', 'status' => $status ) ); @@ -93,6 +84,7 @@ class AssembleUploadChunksJob extends Job { // Cache the info so the user doesn't have to wait forever to get the final info UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Success', @@ -102,8 +94,9 @@ class AssembleUploadChunksJob extends Job { 'status' => Status::newGood() ) ); - } catch ( MWException $e ) { + } catch ( Exception $e ) { UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Failure', diff --git a/includes/jobqueue/jobs/DuplicateJob.php b/includes/jobqueue/jobs/DuplicateJob.php index 1fa6cefe..c5e3a234 100644 --- a/includes/jobqueue/jobs/DuplicateJob.php +++ b/includes/jobqueue/jobs/DuplicateJob.php @@ -18,7 +18,7 @@ * http://www.gnu.org/copyleft/gpl.html * * @file - * @ingroup Cache + * @ingroup JobQueue */ /** diff --git a/includes/jobqueue/jobs/EnqueueJob.php b/includes/jobqueue/jobs/EnqueueJob.php new file mode 100644 index 00000000..46fb2aa7 --- /dev/null +++ b/includes/jobqueue/jobs/EnqueueJob.php @@ -0,0 +1,88 @@ +<?php +/** + * Router job that takes jobs and enqueues them. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup JobQueue + */ + +/** + * Router job that takes jobs and enqueues them to their proper queues + * + * This can be used for several things: + * - a) Making multi-job enqueues more robust by atomically enqueueing + * a single job that pushes the actual jobs (with retry logic) + * - b) Masking the latency of pushing jobs to different queues/wikis + * - c) Low-latency enqueues to push jobs from warm to hot datacenters + * + * @ingroup JobQueue + * @since 1.25 + */ +final class EnqueueJob extends Job { + /** + * Callers should use the factory methods instead + * + * @param Title $title + * @param array $params Job parameters + */ + function __construct( $title, $params ) { + parent::__construct( 'enqueue', $title, $params ); + } + + /** + * @param Job|JobSpecification|array $jobs + * @return JobRouteJob + */ + public static function newFromLocalJobs( $jobs ) { + $jobs = is_array( $jobs ) ? $jobs : array( $jobs ); + + return self::newFromJobsByWiki( array( wfWikiID() => $jobs ) ); + } + + /** + * @param array $jobsByWiki Map of (wiki => JobSpecification list) + * @return JobRouteJob + */ + public static function newFromJobsByWiki( array $jobsByWiki ) { + $jobMapsByWiki = array(); + foreach ( $jobsByWiki as $wiki => $jobs ) { + $jobMapsByWiki[$wiki] = array(); + foreach ( $jobs as $job ) { + if ( $job instanceof JobSpecification ) { + $jobMapsByWiki[$wiki][] = $job->toSerializableArray(); + } else { + throw new InvalidArgumentException( "Jobs must be of type JobSpecification." ); + } + } + } + + return new self( Title::newMainPage(), array( 'jobsByWiki' => $jobMapsByWiki ) ); + } + + public function run() { + foreach ( $this->params['jobsByWiki'] as $wiki => $jobMaps ) { + $jobSpecs = array(); + foreach ( $jobMaps as $jobMap ) { + $jobSpecs[] = JobSpecification::newFromArray( $jobMap ); + } + JobQueueGroup::singleton( $wiki )->push( $jobSpecs ); + } + + return true; + } +} diff --git a/includes/jobqueue/jobs/HTMLCacheUpdateJob.php b/includes/jobqueue/jobs/HTMLCacheUpdateJob.php index 4d1e72c9..e5e521c3 100644 --- a/includes/jobqueue/jobs/HTMLCacheUpdateJob.php +++ b/includes/jobqueue/jobs/HTMLCacheUpdateJob.php @@ -18,6 +18,7 @@ * http://www.gnu.org/copyleft/gpl.html * * @file + * @ingroup JobQueue * @ingroup Cache */ @@ -26,9 +27,9 @@ * * This job comes in a few variants: * - a) Recursive jobs to purge caches for backlink pages for a given title. - * These jobs have have (recursive:true,table:<table>) set. + * These jobs have (recursive:true,table:<table>) set. * - b) Jobs to purge caches for a set of titles (the job title is ignored). - * These jobs have have (pages:(<page ID>:(<namespace>,<title>),...) set. + * These jobs have (pages:(<page ID>:(<namespace>,<title>),...) set. * * @ingroup JobQueue */ @@ -42,17 +43,8 @@ class HTMLCacheUpdateJob extends Job { function run() { global $wgUpdateRowsPerJob, $wgUpdateRowsPerQuery; - static $expected = array( 'recursive', 'pages' ); // new jobs have one of these - - $oldRangeJob = false; - if ( !array_intersect( array_keys( $this->params ), $expected ) ) { - // B/C for older job params formats that lack these fields: - // a) base jobs with just ("table") and b) range jobs with ("table","start","end") - if ( isset( $this->params['start'] ) && isset( $this->params['end'] ) ) { - $oldRangeJob = true; - } else { - $this->params['recursive'] = true; // base job - } + if ( isset( $this->params['table'] ) && !isset( $this->params['pages'] ) ) { + $this->params['recursive'] = true; // b/c; base job } // Job to purge all (or a range of) backlink pages for a page @@ -67,29 +59,15 @@ class HTMLCacheUpdateJob extends Job { array( 'params' => $this->getRootJobParams() ) ); JobQueueGroup::singleton()->push( $jobs ); - // Job to purge pages for for a set of titles + // Job to purge pages for a set of titles } elseif ( isset( $this->params['pages'] ) ) { $this->invalidateTitles( $this->params['pages'] ); - // B/C for job to purge a range of backlink pages for a given page - } elseif ( $oldRangeJob ) { - $titleArray = $this->title->getBacklinkCache()->getLinks( - $this->params['table'], $this->params['start'], $this->params['end'] ); - - $pages = array(); // same format BacklinkJobUtils uses - foreach ( $titleArray as $tl ) { - $pages[$tl->getArticleId()] = array( $tl->getNamespace(), $tl->getDbKey() ); - } - - $jobs = array(); - foreach ( array_chunk( $pages, $wgUpdateRowsPerJob ) as $pageChunk ) { - $jobs[] = new HTMLCacheUpdateJob( $this->title, - array( - 'table' => $this->params['table'], - 'pages' => $pageChunk - ) + $this->getRootJobParams() // carry over information for de-duplication - ); - } - JobQueueGroup::singleton()->push( $jobs ); + // Job to update a single title + } else { + $t = $this->title; + $this->invalidateTitles( array( + $t->getArticleID() => array( $t->getNamespace(), $t->getDBkey() ) + ) ); } return true; diff --git a/includes/jobqueue/jobs/NullJob.php b/includes/jobqueue/jobs/NullJob.php index 66291e9d..f94d6ebc 100644 --- a/includes/jobqueue/jobs/NullJob.php +++ b/includes/jobqueue/jobs/NullJob.php @@ -18,7 +18,7 @@ * http://www.gnu.org/copyleft/gpl.html * * @file - * @ingroup Cache + * @ingroup JobQueue */ /** diff --git a/includes/jobqueue/jobs/PublishStashedFileJob.php b/includes/jobqueue/jobs/PublishStashedFileJob.php index 918a392d..a922dd3d 100644 --- a/includes/jobqueue/jobs/PublishStashedFileJob.php +++ b/includes/jobqueue/jobs/PublishStashedFileJob.php @@ -19,12 +19,14 @@ * * @file * @ingroup Upload + * @ingroup JobQueue */ /** * Upload a file from the upload stash into the local file repo. * * @ingroup Upload + * @ingroup JobQueue */ class PublishStashedFileJob extends Job { public function __construct( $title, $params ) { @@ -35,26 +37,16 @@ class PublishStashedFileJob extends Job { public function run() { $scope = RequestContext::importScopedSession( $this->params['session'] ); $context = RequestContext::getMain(); + $user = $context->getUser(); try { - $user = $context->getUser(); if ( !$user->isLoggedIn() ) { $this->setLastError( "Could not load the author user from session." ); return false; } - if ( count( $_SESSION ) === 0 ) { - // Empty session probably indicates that we didn't associate - // with the session correctly. Note that being able to load - // the user does not necessarily mean the session was loaded. - // Most likely cause by suhosin.session.encrypt = On. - $this->setLastError( "Error associating with user session. " . - "Try setting suhosin.session.encrypt = Off" ); - - return false; - } - UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Poll', 'stage' => 'publish', 'status' => Status::newGood() ) ); @@ -72,6 +64,7 @@ class PublishStashedFileJob extends Job { $status = Status::newFatal( 'verification-error' ); $status->value = array( 'verification' => $verification ); UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status ) ); @@ -89,6 +82,7 @@ class PublishStashedFileJob extends Job { ); if ( !$status->isGood() ) { UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status ) ); @@ -106,6 +100,7 @@ class PublishStashedFileJob extends Job { // Cache the info so the user doesn't have to wait forever to get the final info UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Success', @@ -115,8 +110,9 @@ class PublishStashedFileJob extends Job { 'status' => Status::newGood() ) ); - } catch ( MWException $e ) { + } catch ( Exception $e ) { UploadBase::setSessionStatus( + $user, $this->params['filekey'], array( 'result' => 'Failure', diff --git a/includes/jobqueue/jobs/RecentChangesUpdateJob.php b/includes/jobqueue/jobs/RecentChangesUpdateJob.php new file mode 100644 index 00000000..cc04595d --- /dev/null +++ b/includes/jobqueue/jobs/RecentChangesUpdateJob.php @@ -0,0 +1,223 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + * @ingroup JobQueue + */ + +/** + * Job for pruning recent changes + * + * @ingroup JobQueue + * @since 1.25 + */ +class RecentChangesUpdateJob extends Job { + function __construct( $title, $params ) { + parent::__construct( 'recentChangesUpdate', $title, $params ); + + if ( !isset( $params['type'] ) ) { + throw new Exception( "Missing 'type' parameter." ); + } + + $this->removeDuplicates = true; + } + + /** + * @return RecentChangesUpdateJob + */ + final public static function newPurgeJob() { + return new self( + SpecialPage::getTitleFor( 'Recentchanges' ), array( 'type' => 'purge' ) + ); + } + + /** + * @return RecentChangesUpdateJob + * @since 1.26 + */ + final public static function newCacheUpdateJob() { + return new self( + SpecialPage::getTitleFor( 'Recentchanges' ), array( 'type' => 'cacheUpdate' ) + ); + } + + public function run() { + if ( $this->params['type'] === 'purge' ) { + $this->purgeExpiredRows(); + } elseif ( $this->params['type'] === 'cacheUpdate' ) { + $this->updateActiveUsers(); + } else { + throw new InvalidArgumentException( + "Invalid 'type' parameter '{$this->params['type']}'." ); + } + + return true; + } + + protected function purgeExpiredRows() { + global $wgRCMaxAge; + + $lockKey = wfWikiID() . ':recentchanges-prune'; + + $dbw = wfGetDB( DB_MASTER ); + if ( !$dbw->lock( $lockKey, __METHOD__, 1 ) ) { + return; // already in progress + } + $batchSize = 100; // Avoid slave lag + + $cutoff = $dbw->timestamp( time() - $wgRCMaxAge ); + do { + $rcIds = $dbw->selectFieldValues( 'recentchanges', + 'rc_id', + array( 'rc_timestamp < ' . $dbw->addQuotes( $cutoff ) ), + __METHOD__, + array( 'LIMIT' => $batchSize ) + ); + if ( $rcIds ) { + $dbw->delete( 'recentchanges', array( 'rc_id' => $rcIds ), __METHOD__ ); + } + // Commit in chunks to avoid slave lag + $dbw->commit( __METHOD__, 'flush' ); + + if ( count( $rcIds ) === $batchSize ) { + // There might be more, so try waiting for slaves + if ( !wfWaitForSlaves( null, false, false, /* $timeout = */ 3 ) ) { + // Another job will continue anyway + break; + } + } + } while ( $rcIds ); + + $dbw->unlock( $lockKey, __METHOD__ ); + } + + protected function updateActiveUsers() { + global $wgActiveUserDays; + + // Users that made edits at least this many days ago are "active" + $days = $wgActiveUserDays; + // Pull in the full window of active users in this update + $window = $wgActiveUserDays * 86400; + + $dbw = wfGetDB( DB_MASTER ); + // JobRunner uses DBO_TRX, but doesn't call begin/commit itself; + // onTransactionIdle() will run immediately since there is no trx. + $dbw->onTransactionIdle( function() use ( $dbw, $days, $window ) { + // Avoid disconnect/ping() cycle that makes locks fall off + $dbw->setSessionOptions( array( 'connTimeout' => 900 ) ); + + $lockKey = wfWikiID() . '-activeusers'; + if ( !$dbw->lock( $lockKey, __METHOD__, 1 ) ) { + return false; // exclusive update (avoids duplicate entries) + } + + $nowUnix = time(); + // Get the last-updated timestamp for the cache + $cTime = $dbw->selectField( 'querycache_info', + 'qci_timestamp', + array( 'qci_type' => 'activeusers' ) + ); + $cTimeUnix = $cTime ? wfTimestamp( TS_UNIX, $cTime ) : 1; + + // Pick the date range to fetch from. This is normally from the last + // update to till the present time, but has a limited window for sanity. + // If the window is limited, multiple runs are need to fully populate it. + $sTimestamp = max( $cTimeUnix, $nowUnix - $days * 86400 ); + $eTimestamp = min( $sTimestamp + $window, $nowUnix ); + + // Get all the users active since the last update + $res = $dbw->select( + array( 'recentchanges' ), + array( 'rc_user_text', 'lastedittime' => 'MAX(rc_timestamp)' ), + array( + 'rc_user > 0', // actual accounts + 'rc_type != ' . $dbw->addQuotes( RC_EXTERNAL ), // no wikidata + 'rc_log_type IS NULL OR rc_log_type != ' . $dbw->addQuotes( 'newusers' ), + 'rc_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $sTimestamp ) ), + 'rc_timestamp <= ' . $dbw->addQuotes( $dbw->timestamp( $eTimestamp ) ) + ), + __METHOD__, + array( + 'GROUP BY' => array( 'rc_user_text' ), + 'ORDER BY' => 'NULL' // avoid filesort + ) + ); + $names = array(); + foreach ( $res as $row ) { + $names[$row->rc_user_text] = $row->lastedittime; + } + + // Rotate out users that have not edited in too long (according to old data set) + $dbw->delete( 'querycachetwo', + array( + 'qcc_type' => 'activeusers', + 'qcc_value < ' . $dbw->addQuotes( $nowUnix - $days * 86400 ) // TS_UNIX + ), + __METHOD__ + ); + + // Find which of the recently active users are already accounted for + if ( count( $names ) ) { + $res = $dbw->select( 'querycachetwo', + array( 'user_name' => 'qcc_title' ), + array( + 'qcc_type' => 'activeusers', + 'qcc_namespace' => NS_USER, + 'qcc_title' => array_keys( $names ) ), + __METHOD__ + ); + foreach ( $res as $row ) { + unset( $names[$row->user_name] ); + } + } + + // Insert the users that need to be added to the list + if ( count( $names ) ) { + $newRows = array(); + foreach ( $names as $name => $lastEditTime ) { + $newRows[] = array( + 'qcc_type' => 'activeusers', + 'qcc_namespace' => NS_USER, + 'qcc_title' => $name, + 'qcc_value' => wfTimestamp( TS_UNIX, $lastEditTime ), + 'qcc_namespacetwo' => 0, // unused + 'qcc_titletwo' => '' // unused + ); + } + foreach ( array_chunk( $newRows, 500 ) as $rowBatch ) { + $dbw->insert( 'querycachetwo', $rowBatch, __METHOD__ ); + wfWaitForSlaves(); + } + } + + // If a transaction was already started, it might have an old + // snapshot, so kludge the timestamp range back as needed. + $asOfTimestamp = min( $eTimestamp, (int)$dbw->trxTimestamp() ); + + // Touch the data freshness timestamp + $dbw->replace( 'querycache_info', + array( 'qci_type' ), + array( 'qci_type' => 'activeusers', + 'qci_timestamp' => $dbw->timestamp( $asOfTimestamp ) ), // not always $now + __METHOD__ + ); + + $dbw->unlock( $lockKey, __METHOD__ ); + } ); + } +} diff --git a/includes/jobqueue/jobs/RefreshLinksJob.php b/includes/jobqueue/jobs/RefreshLinksJob.php index f82af273..1252b0b5 100644 --- a/includes/jobqueue/jobs/RefreshLinksJob.php +++ b/includes/jobqueue/jobs/RefreshLinksJob.php @@ -26,9 +26,9 @@ * * This job comes in a few variants: * - a) Recursive jobs to update links for backlink pages for a given title. - * These jobs have have (recursive:true,table:<table>) set. + * These jobs have (recursive:true,table:<table>) set. * - b) Jobs to update links for a set of pages (the job title is ignored). - * These jobs have have (pages:(<page ID>:(<namespace>,<title>),...) set. + * These jobs have (pages:(<page ID>:(<namespace>,<title>),...) set. * - c) Jobs to update links for a single page (the job title) * These jobs need no extra fields set. * @@ -39,6 +39,10 @@ class RefreshLinksJob extends Job { function __construct( $title, $params = '' ) { parent::__construct( 'refreshLinks', $title, $params ); + // A separate type is used just for cascade-protected backlinks + if ( !empty( $this->params['prioritize'] ) ) { + $this->command .= 'Prioritized'; + } // Base backlink update jobs and per-title update jobs can be de-duplicated. // If template A changes twice before any jobs run, a clean queue will have: // (A base, A base) @@ -86,7 +90,7 @@ class RefreshLinksJob extends Job { array( 'params' => $extraParams ) ); JobQueueGroup::singleton()->push( $jobs ); - // Job to update link tables for for a set of titles + // Job to update link tables for a set of titles } elseif ( isset( $this->params['pages'] ) ) { foreach ( $this->params['pages'] as $pageId => $nsAndKey ) { list( $ns, $dbKey ) = $nsAndKey; @@ -100,6 +104,10 @@ class RefreshLinksJob extends Job { return true; } + /** + * @param Title $title + * @return bool + */ protected function runForTitle( Title $title = null ) { $linkCache = LinkCache::singleton(); $linkCache->clear(); @@ -157,7 +165,7 @@ class RefreshLinksJob extends Job { $ellapsed = microtime( true ) - $start; // If it took a long time to render, then save this back to the cache to avoid // wasted CPU by other apaches or job runners. We don't want to always save to - // cache as this cause cause high cache I/O and LRU churn when a template changes. + // cache as this can cause high cache I/O and LRU churn when a template changes. if ( $ellapsed >= self::PARSE_THRESHOLD_SEC && $page->isParserCacheUsed( $parserOptions, $revision->getId() ) && $parserOutput->isCacheable() diff --git a/includes/jobqueue/jobs/RefreshLinksJob2.php b/includes/jobqueue/jobs/RefreshLinksJob2.php deleted file mode 100644 index 97405aeb..00000000 --- a/includes/jobqueue/jobs/RefreshLinksJob2.php +++ /dev/null @@ -1,141 +0,0 @@ -<?php -/** - * Job to update links for a given title. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * http://www.gnu.org/copyleft/gpl.html - * - * @file - * @ingroup JobQueue - */ - -/** - * Background job to update links for titles in certain backlink range by page ID. - * Newer version for high use templates. This is deprecated by RefreshLinksPartitionJob. - * - * @ingroup JobQueue - * @deprecated since 1.23 - */ -class RefreshLinksJob2 extends Job { - function __construct( $title, $params ) { - parent::__construct( 'refreshLinks2', $title, $params ); - // Base jobs for large templates can easily be de-duplicated - $this->removeDuplicates = !isset( $params['start'] ) && !isset( $params['end'] ); - } - - /** - * Run a refreshLinks2 job - * @return bool Success - */ - function run() { - global $wgUpdateRowsPerJob; - - $linkCache = LinkCache::singleton(); - $linkCache->clear(); - - if ( is_null( $this->title ) ) { - $this->error = "refreshLinks2: Invalid title"; - return false; - } - - // Back compat for pre-r94435 jobs - $table = isset( $this->params['table'] ) ? $this->params['table'] : 'templatelinks'; - - // Avoid slave lag when fetching templates. - // When the outermost job is run, we know that the caller that enqueued it must have - // committed the relevant changes to the DB by now. At that point, record the master - // position and pass it along as the job recursively breaks into smaller range jobs. - // Hopefully, when leaf jobs are popped, the slaves will have reached that position. - if ( isset( $this->params['masterPos'] ) ) { - $masterPos = $this->params['masterPos']; - } elseif ( wfGetLB()->getServerCount() > 1 ) { - $masterPos = wfGetLB()->getMasterPos(); - } else { - $masterPos = false; - } - - $tbc = $this->title->getBacklinkCache(); - - $jobs = array(); // jobs to insert - if ( isset( $this->params['start'] ) && isset( $this->params['end'] ) ) { - # This is a partition job to trigger the insertion of leaf jobs... - $jobs = array_merge( $jobs, $this->getSingleTitleJobs( $table, $masterPos ) ); - } else { - # This is a base job to trigger the insertion of partitioned jobs... - if ( $tbc->getNumLinks( $table, $wgUpdateRowsPerJob + 1 ) <= $wgUpdateRowsPerJob ) { - # Just directly insert the single per-title jobs - $jobs = array_merge( $jobs, $this->getSingleTitleJobs( $table, $masterPos ) ); - } else { - # Insert the partition jobs to make per-title jobs - foreach ( $tbc->partition( $table, $wgUpdateRowsPerJob ) as $batch ) { - list( $start, $end ) = $batch; - $jobs[] = new RefreshLinksJob2( $this->title, - array( - 'table' => $table, - 'start' => $start, - 'end' => $end, - 'masterPos' => $masterPos, - ) + $this->getRootJobParams() // carry over information for de-duplication - ); - } - } - } - - if ( count( $jobs ) ) { - JobQueueGroup::singleton()->push( $jobs ); - } - - return true; - } - - /** - * @param string $table - * @param mixed $masterPos - * @return array - */ - protected function getSingleTitleJobs( $table, $masterPos ) { - # The "start"/"end" fields are not set for the base jobs - $start = isset( $this->params['start'] ) ? $this->params['start'] : false; - $end = isset( $this->params['end'] ) ? $this->params['end'] : false; - $titles = $this->title->getBacklinkCache()->getLinks( $table, $start, $end ); - # Convert into single page refresh links jobs. - # This handles well when in sapi mode and is useful in any case for job - # de-duplication. If many pages use template A, and that template itself - # uses template B, then an edit to both will create many duplicate jobs. - # Roughly speaking, for each page, one of the "RefreshLinksJob" jobs will - # get run first, and when it does, it will remove the duplicates. Of course, - # one page could have its job popped when the other page's job is still - # buried within the logic of a refreshLinks2 job. - $jobs = array(); - foreach ( $titles as $title ) { - $jobs[] = new RefreshLinksJob( $title, - array( 'masterPos' => $masterPos ) + $this->getRootJobParams() - ); // carry over information for de-duplication - } - return $jobs; - } - - /** - * @return array - */ - public function getDeduplicationInfo() { - $info = parent::getDeduplicationInfo(); - // Don't let highly unique "masterPos" values ruin duplicate detection - if ( is_array( $info['params'] ) ) { - unset( $info['params']['masterPos'] ); - } - return $info; - } -} diff --git a/includes/jobqueue/jobs/ThumbnailRenderJob.php b/includes/jobqueue/jobs/ThumbnailRenderJob.php new file mode 100644 index 00000000..ab381388 --- /dev/null +++ b/includes/jobqueue/jobs/ThumbnailRenderJob.php @@ -0,0 +1,109 @@ +<?php +/** + * Job for asynchronous rendering of thumbnails. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup JobQueue + */ + +/** + * Job for asynchronous rendering of thumbnails. + * + * @ingroup JobQueue + */ +class ThumbnailRenderJob extends Job { + public function __construct( $title, $params ) { + parent::__construct( 'ThumbnailRender', $title, $params ); + } + + public function run() { + global $wgUploadThumbnailRenderMethod; + + $transformParams = $this->params['transformParams']; + + $file = wfLocalFile( $this->title ); + $file->load( File::READ_LATEST ); + + if ( $file && $file->exists() ) { + if ( $wgUploadThumbnailRenderMethod === 'jobqueue' ) { + $thumb = $file->transform( $transformParams, File::RENDER_NOW ); + + if ( $thumb && !$thumb->isError() ) { + return true; + } else { + $this->setLastError( __METHOD__ . ': thumbnail couln\'t be generated' ); + return false; + } + } elseif ( $wgUploadThumbnailRenderMethod === 'http' ) { + $status = $this->hitThumbUrl( $file, $transformParams ); + + wfDebug( __METHOD__ . ": received status {$status}\n" ); + + if ( $status === 200 || $status === 301 || $status === 302 ) { + return true; + } elseif ( $status ) { + // Note that this currently happens (500) when requesting sizes larger then or + // equal to the original, which is harmless. + $this->setLastError( __METHOD__ . ': incorrect HTTP status ' . $status ); + return false; + } else { + $this->setLastError( __METHOD__ . ': HTTP request failure' ); + return false; + } + } else { + $this->setLastError( __METHOD__ . ': unknown thumbnail render method ' . $wgUploadThumbnailRenderMethod ); + return false; + } + } else { + $this->setLastError( __METHOD__ . ': file doesn\'t exist' ); + return false; + } + } + + protected function hitThumbUrl( $file, $transformParams ) { + global $wgUploadThumbnailRenderHttpCustomHost, $wgUploadThumbnailRenderHttpCustomDomain; + + $thumbName = $file->thumbName( $transformParams ); + $thumbUrl = $file->getThumbUrl( $thumbName ); + + if ( $wgUploadThumbnailRenderHttpCustomDomain ) { + $parsedUrl = wfParseUrl( $thumbUrl ); + + if ( !$parsedUrl || !isset( $parsedUrl['path'] ) || !strlen( $parsedUrl['path'] ) ) { + return false; + } + + $thumbUrl = '//' . $wgUploadThumbnailRenderHttpCustomDomain . $parsedUrl['path']; + } + + wfDebug( __METHOD__ . ": hitting url {$thumbUrl}\n" ); + + $request = MWHttpRequest::factory( $thumbUrl, + array( 'method' => 'HEAD', 'followRedirects' => true ), + __METHOD__ + ); + + if ( $wgUploadThumbnailRenderHttpCustomHost ) { + $request->setHeader( 'Host', $wgUploadThumbnailRenderHttpCustomHost ); + } + + $status = $request->execute(); + + return $request->getStatus(); + } +} diff --git a/includes/jobqueue/jobs/UploadFromUrlJob.php b/includes/jobqueue/jobs/UploadFromUrlJob.php index a09db15a..d15fd025 100644 --- a/includes/jobqueue/jobs/UploadFromUrlJob.php +++ b/includes/jobqueue/jobs/UploadFromUrlJob.php @@ -81,7 +81,7 @@ class UploadFromUrlJob extends Job { if ( $warnings ) { # Stash the upload - $key = $this->upload->stashFile(); + $key = $this->upload->stashFile( $this->user ); // @todo FIXME: This has been broken for a while. // User::leaveUserMessage() does not exist. |