diff options
Diffstat (limited to 'includes/job/jobs')
-rw-r--r-- | includes/job/jobs/AssembleUploadChunksJob.php | 118 | ||||
-rw-r--r-- | includes/job/jobs/DoubleRedirectJob.php | 218 | ||||
-rw-r--r-- | includes/job/jobs/DuplicateJob.php | 59 | ||||
-rw-r--r-- | includes/job/jobs/EmaillingJob.php | 47 | ||||
-rw-r--r-- | includes/job/jobs/EnotifNotifyJob.php | 58 | ||||
-rw-r--r-- | includes/job/jobs/HTMLCacheUpdateJob.php | 254 | ||||
-rw-r--r-- | includes/job/jobs/NullJob.php | 60 | ||||
-rw-r--r-- | includes/job/jobs/PublishStashedFileJob.php | 130 | ||||
-rw-r--r-- | includes/job/jobs/RefreshLinksJob.php | 226 | ||||
-rw-r--r-- | includes/job/jobs/UploadFromUrlJob.php | 179 |
10 files changed, 1349 insertions, 0 deletions
diff --git a/includes/job/jobs/AssembleUploadChunksJob.php b/includes/job/jobs/AssembleUploadChunksJob.php new file mode 100644 index 00000000..c5dd9eaa --- /dev/null +++ b/includes/job/jobs/AssembleUploadChunksJob.php @@ -0,0 +1,118 @@ +<?php +/** + * Assemble the segments of a chunked upload. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Upload + */ + +/** + * Assemble the segments of a chunked upload. + * + * @ingroup Upload + */ +class AssembleUploadChunksJob extends Job { + public function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'AssembleUploadChunks', $title, $params, $id ); + $this->removeDuplicates = true; + } + + public function run() { + $scope = RequestContext::importScopedSession( $this->params['session'] ); + $context = RequestContext::getMain(); + try { + $user = $context->getUser(); + if ( !$user->isLoggedIn() ) { + $this->setLastError( "Could not load the author user from session." ); + return false; + } + + UploadBase::setSessionStatus( + $this->params['filekey'], + array( 'result' => 'Poll', 'stage' => 'assembling', 'status' => Status::newGood() ) + ); + + $upload = new UploadFromChunks( $user ); + $upload->continueChunks( + $this->params['filename'], + $this->params['filekey'], + $context->getRequest() + ); + + // Combine all of the chunks into a local file and upload that to a new stash file + $status = $upload->concatenateChunks(); + if ( !$status->isGood() ) { + UploadBase::setSessionStatus( + $this->params['filekey'], + array( 'result' => 'Failure', 'stage' => 'assembling', 'status' => $status ) + ); + $this->setLastError( $status->getWikiText() ); + return false; + } + + // We have a new filekey for the fully concatenated file + $newFileKey = $upload->getLocalFile()->getFileKey(); + + // Remove the old stash file row and first chunk file + $upload->stash->removeFileNoAuth( $this->params['filekey'] ); + + // Build the image info array while we have the local reference handy + $apiMain = new ApiMain(); // dummy object (XXX) + $imageInfo = $upload->getImageInfo( $apiMain->getResult() ); + + // Cleanup any temporary local file + $upload->cleanupTempFile(); + + // Cache the info so the user doesn't have to wait forever to get the final info + UploadBase::setSessionStatus( + $this->params['filekey'], + array( + 'result' => 'Success', + 'stage' => 'assembling', + 'filekey' => $newFileKey, + 'imageinfo' => $imageInfo, + 'status' => Status::newGood() + ) + ); + } catch ( MWException $e ) { + UploadBase::setSessionStatus( + $this->params['filekey'], + array( + 'result' => 'Failure', + 'stage' => 'assembling', + 'status' => Status::newFatal( 'api-error-stashfailed' ) + ) + ); + $this->setLastError( get_class( $e ) . ": " . $e->getText() ); + return false; + } + return true; + } + + public function getDeduplicationInfo() { + $info = parent::getDeduplicationInfo(); + if ( is_array( $info['params'] ) ) { + $info['params'] = array( 'filekey' => $info['params']['filekey'] ); + } + return $info; + } + + public function allowRetries() { + return false; + } +} diff --git a/includes/job/jobs/DoubleRedirectJob.php b/includes/job/jobs/DoubleRedirectJob.php new file mode 100644 index 00000000..05abeeef --- /dev/null +++ b/includes/job/jobs/DoubleRedirectJob.php @@ -0,0 +1,218 @@ +<?php +/** + * Job to fix double redirects after moving a page. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup JobQueue + */ + +/** + * Job to fix double redirects after moving a page + * + * @ingroup JobQueue + */ +class DoubleRedirectJob extends Job { + var $reason, $redirTitle; + + /** + * @var User + */ + static $user; + + /** + * Insert jobs into the job queue to fix redirects to the given title + * @param string $reason the reason for the fix, see message "double-redirect-fixed-<reason>" + * @param $redirTitle Title: the title which has changed, redirects pointing to this title are fixed + * @param bool $destTitle Not used + */ + public static function fixRedirects( $reason, $redirTitle, $destTitle = false ) { + # Need to use the master to get the redirect table updated in the same transaction + $dbw = wfGetDB( DB_MASTER ); + $res = $dbw->select( + array( 'redirect', 'page' ), + array( 'page_namespace', 'page_title' ), + array( + 'page_id = rd_from', + 'rd_namespace' => $redirTitle->getNamespace(), + 'rd_title' => $redirTitle->getDBkey() + ), __METHOD__ ); + if ( !$res->numRows() ) { + return; + } + $jobs = array(); + foreach ( $res as $row ) { + $title = Title::makeTitle( $row->page_namespace, $row->page_title ); + if ( !$title ) { + continue; + } + + $jobs[] = new self( $title, array( + 'reason' => $reason, + 'redirTitle' => $redirTitle->getPrefixedDBkey() ) ); + # Avoid excessive memory usage + if ( count( $jobs ) > 10000 ) { + JobQueueGroup::singleton()->push( $jobs ); + $jobs = array(); + } + } + JobQueueGroup::singleton()->push( $jobs ); + } + + function __construct( $title, $params = false, $id = 0 ) { + parent::__construct( 'fixDoubleRedirect', $title, $params, $id ); + $this->reason = $params['reason']; + $this->redirTitle = Title::newFromText( $params['redirTitle'] ); + } + + /** + * @return bool + */ + function run() { + if ( !$this->redirTitle ) { + $this->setLastError( 'Invalid title' ); + return false; + } + + $targetRev = Revision::newFromTitle( $this->title, false, Revision::READ_LATEST ); + if ( !$targetRev ) { + wfDebug( __METHOD__.": target redirect already deleted, ignoring\n" ); + return true; + } + $content = $targetRev->getContent(); + $currentDest = $content ? $content->getRedirectTarget() : null; + if ( !$currentDest || !$currentDest->equals( $this->redirTitle ) ) { + wfDebug( __METHOD__.": Redirect has changed since the job was queued\n" ); + return true; + } + + # Check for a suppression tag (used e.g. in periodically archived discussions) + $mw = MagicWord::get( 'staticredirect' ); + if ( $content->matchMagicWord( $mw ) ) { + wfDebug( __METHOD__.": skipping: suppressed with __STATICREDIRECT__\n" ); + return true; + } + + # Find the current final destination + $newTitle = self::getFinalDestination( $this->redirTitle ); + if ( !$newTitle ) { + wfDebug( __METHOD__.": skipping: single redirect, circular redirect or invalid redirect destination\n" ); + return true; + } + if ( $newTitle->equals( $this->redirTitle ) ) { + # The redirect is already right, no need to change it + # This can happen if the page was moved back (say after vandalism) + wfDebug( __METHOD__.": skipping, already good\n" ); + } + + # Preserve fragment (bug 14904) + $newTitle = Title::makeTitle( $newTitle->getNamespace(), $newTitle->getDBkey(), + $currentDest->getFragment(), $newTitle->getInterwiki() ); + + # Fix the text + $newContent = $content->updateRedirect( $newTitle ); + + if ( $newContent->equals( $content ) ) { + $this->setLastError( 'Content unchanged???' ); + return false; + } + + $user = $this->getUser(); + if ( !$user ) { + $this->setLastError( 'Invalid user' ); + return false; + } + + # Save it + global $wgUser; + $oldUser = $wgUser; + $wgUser = $user; + $article = WikiPage::factory( $this->title ); + $reason = wfMessage( 'double-redirect-fixed-' . $this->reason, + $this->redirTitle->getPrefixedText(), $newTitle->getPrefixedText() + )->inContentLanguage()->text(); + $article->doEditContent( $newContent, $reason, EDIT_UPDATE | EDIT_SUPPRESS_RC, false, $user ); + $wgUser = $oldUser; + + return true; + } + + /** + * Get the final destination of a redirect + * + * @param $title Title + * + * @return bool if the specified title is not a redirect, or if it is a circular redirect + */ + public static function getFinalDestination( $title ) { + $dbw = wfGetDB( DB_MASTER ); + + $seenTitles = array(); # Circular redirect check + $dest = false; + + while ( true ) { + $titleText = $title->getPrefixedDBkey(); + if ( isset( $seenTitles[$titleText] ) ) { + wfDebug( __METHOD__, "Circular redirect detected, aborting\n" ); + return false; + } + $seenTitles[$titleText] = true; + + if ( $title->getInterwiki() ) { + // If the target is interwiki, we have to break early (bug 40352). + // Otherwise it will look up a row in the local page table + // with the namespace/page of the interwiki target which can cause + // unexpected results (e.g. X -> foo:Bar -> Bar -> .. ) + break; + } + + $row = $dbw->selectRow( + array( 'redirect', 'page' ), + array( 'rd_namespace', 'rd_title', 'rd_interwiki' ), + array( + 'rd_from=page_id', + 'page_namespace' => $title->getNamespace(), + 'page_title' => $title->getDBkey() + ), __METHOD__ ); + if ( !$row ) { + # No redirect from here, chain terminates + break; + } else { + $dest = $title = Title::makeTitle( $row->rd_namespace, $row->rd_title, '', $row->rd_interwiki ); + } + } + return $dest; + } + + /** + * Get a user object for doing edits, from a request-lifetime cache + * False will be returned if the user name specified in the + * 'double-redirect-fixer' message is invalid. + * + * @return User|bool + */ + function getUser() { + if ( !self::$user ) { + self::$user = User::newFromName( wfMessage( 'double-redirect-fixer' )->inContentLanguage()->text() ); + # User::newFromName() can return false on a badly configured wiki. + if ( self::$user && !self::$user->isLoggedIn() ) { + self::$user->addToDatabase(); + } + } + return self::$user; + } +} diff --git a/includes/job/jobs/DuplicateJob.php b/includes/job/jobs/DuplicateJob.php new file mode 100644 index 00000000..524983b8 --- /dev/null +++ b/includes/job/jobs/DuplicateJob.php @@ -0,0 +1,59 @@ +<?php +/** + * No-op job that does nothing. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * No-op job that does nothing. Used to represent duplicates. + * + * @ingroup JobQueue + */ +final class DuplicateJob extends Job { + /** + * Callers should use DuplicateJob::newFromJob() instead + * + * @param $title Title + * @param array $params job parameters + * @param $id Integer: job id + */ + function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'duplicate', $title, $params, $id ); + } + + /** + * Get a duplicate no-op version of a job + * + * @param Job $job + * @return Job + */ + public static function newFromJob( Job $job ) { + $djob = new self( $job->getTitle(), $job->getParams(), $job->getId() ); + $djob->command = $job->getType(); + $djob->params = is_array( $djob->params ) ? $djob->params : array(); + $djob->params = array( 'isDuplicate' => true ) + $djob->params; + $djob->metadata = $job->metadata; + return $djob; + } + + public function run() { + return true; + } +} diff --git a/includes/job/jobs/EmaillingJob.php b/includes/job/jobs/EmaillingJob.php new file mode 100644 index 00000000..9fbf3124 --- /dev/null +++ b/includes/job/jobs/EmaillingJob.php @@ -0,0 +1,47 @@ +<?php +/** + * Old job for notification emails. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup JobQueue + */ + +/** + * Old job used for sending single notification emails; + * kept for backwards-compatibility + * + * @ingroup JobQueue + */ +class EmaillingJob extends Job { + function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'sendMail', Title::newMainPage(), $params, $id ); + } + + function run() { + $status = UserMailer::send( + $this->params['to'], + $this->params['from'], + $this->params['subj'], + $this->params['body'], + $this->params['replyto'] + ); + + return $status->isOK(); + } + +} diff --git a/includes/job/jobs/EnotifNotifyJob.php b/includes/job/jobs/EnotifNotifyJob.php new file mode 100644 index 00000000..2be05b63 --- /dev/null +++ b/includes/job/jobs/EnotifNotifyJob.php @@ -0,0 +1,58 @@ +<?php +/** + * Job for notification emails. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup JobQueue + */ + +/** + * Job for email notification mails + * + * @ingroup JobQueue + */ +class EnotifNotifyJob extends Job { + + function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'enotifNotify', $title, $params, $id ); + } + + function run() { + $enotif = new EmailNotification(); + // Get the user from ID (rename safe). Anons are 0, so defer to name. + if( isset( $this->params['editorID'] ) && $this->params['editorID'] ) { + $editor = User::newFromId( $this->params['editorID'] ); + // B/C, only the name might be given. + } else { + # FIXME: newFromName could return false on a badly configured wiki. + $editor = User::newFromName( $this->params['editor'], false ); + } + $enotif->actuallyNotifyOnPageChange( + $editor, + $this->title, + $this->params['timestamp'], + $this->params['summary'], + $this->params['minorEdit'], + $this->params['oldid'], + $this->params['watchers'], + $this->params['pageStatus'] + ); + return true; + } + +} diff --git a/includes/job/jobs/HTMLCacheUpdateJob.php b/includes/job/jobs/HTMLCacheUpdateJob.php new file mode 100644 index 00000000..818c6abf --- /dev/null +++ b/includes/job/jobs/HTMLCacheUpdateJob.php @@ -0,0 +1,254 @@ +<?php +/** + * HTML cache invalidation of all pages linking to a given title. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * Job wrapper for HTMLCacheUpdate. Gets run whenever a related + * job gets called from the queue. + * + * This class is designed to work efficiently with small numbers of links, and + * to work reasonably well with up to ~10^5 links. Above ~10^6 links, the memory + * and time requirements of loading all backlinked IDs in doUpdate() might become + * prohibitive. The requirements measured at Wikimedia are approximately: + * + * memory: 48 bytes per row + * time: 16us per row for the query plus processing + * + * The reason this query is done is to support partitioning of the job + * by backlinked ID. The memory issue could be allieviated by doing this query in + * batches, but of course LIMIT with an offset is inefficient on the DB side. + * + * The class is nevertheless a vast improvement on the previous method of using + * File::getLinksTo() and Title::touchArray(), which uses about 2KB of memory per + * link. + * + * @ingroup JobQueue + */ +class HTMLCacheUpdateJob extends Job { + /** @var BacklinkCache */ + protected $blCache; + + protected $rowsPerJob, $rowsPerQuery; + + /** + * Construct a job + * @param $title Title: the title linked to + * @param array $params job parameters (table, start and end page_ids) + * @param $id Integer: job id + */ + function __construct( $title, $params, $id = 0 ) { + global $wgUpdateRowsPerJob, $wgUpdateRowsPerQuery; + + parent::__construct( 'htmlCacheUpdate', $title, $params, $id ); + + $this->rowsPerJob = $wgUpdateRowsPerJob; + $this->rowsPerQuery = $wgUpdateRowsPerQuery; + $this->blCache = $title->getBacklinkCache(); + } + + public function run() { + if ( isset( $this->params['start'] ) && isset( $this->params['end'] ) ) { + # This is hit when a job is actually performed + return $this->doPartialUpdate(); + } else { + # This is hit when the jobs have to be inserted + return $this->doFullUpdate(); + } + } + + /** + * Update all of the backlinks + */ + protected function doFullUpdate() { + # Get an estimate of the number of rows from the BacklinkCache + $numRows = $this->blCache->getNumLinks( $this->params['table'] ); + if ( $numRows > $this->rowsPerJob * 2 ) { + # Do fast cached partition + $this->insertPartitionJobs(); + } else { + # Get the links from the DB + $titleArray = $this->blCache->getLinks( $this->params['table'] ); + # Check if the row count estimate was correct + if ( $titleArray->count() > $this->rowsPerJob * 2 ) { + # Not correct, do accurate partition + wfDebug( __METHOD__.": row count estimate was incorrect, repartitioning\n" ); + $this->insertJobsFromTitles( $titleArray ); + } else { + $this->invalidateTitles( $titleArray ); // just do the query + } + } + return true; + } + + /** + * Update some of the backlinks, defined by a page ID range + */ + protected function doPartialUpdate() { + $titleArray = $this->blCache->getLinks( + $this->params['table'], $this->params['start'], $this->params['end'] ); + if ( $titleArray->count() <= $this->rowsPerJob * 2 ) { + # This partition is small enough, do the update + $this->invalidateTitles( $titleArray ); + } else { + # Partitioning was excessively inaccurate. Divide the job further. + # This can occur when a large number of links are added in a short + # period of time, say by updating a heavily-used template. + $this->insertJobsFromTitles( $titleArray ); + } + return true; + } + + /** + * Partition the current range given by $this->params['start'] and $this->params['end'], + * using a pre-calculated title array which gives the links in that range. + * Queue the resulting jobs. + * + * @param $titleArray array + * @param $rootJobParams array + * @return void + */ + protected function insertJobsFromTitles( $titleArray, $rootJobParams = array() ) { + // Carry over any "root job" information + $rootJobParams = $this->getRootJobParams(); + # We make subpartitions in the sense that the start of the first job + # will be the start of the parent partition, and the end of the last + # job will be the end of the parent partition. + $jobs = array(); + $start = $this->params['start']; # start of the current job + $numTitles = 0; + foreach ( $titleArray as $title ) { + $id = $title->getArticleID(); + # $numTitles is now the number of titles in the current job not + # including the current ID + if ( $numTitles >= $this->rowsPerJob ) { + # Add a job up to but not including the current ID + $jobs[] = new HTMLCacheUpdateJob( $this->title, + array( + 'table' => $this->params['table'], + 'start' => $start, + 'end' => $id - 1 + ) + $rootJobParams // carry over information for de-duplication + ); + $start = $id; + $numTitles = 0; + } + $numTitles++; + } + # Last job + $jobs[] = new HTMLCacheUpdateJob( $this->title, + array( + 'table' => $this->params['table'], + 'start' => $start, + 'end' => $this->params['end'] + ) + $rootJobParams // carry over information for de-duplication + ); + wfDebug( __METHOD__.": repartitioning into " . count( $jobs ) . " jobs\n" ); + + if ( count( $jobs ) < 2 ) { + # I don't think this is possible at present, but handling this case + # makes the code a bit more robust against future code updates and + # avoids a potential infinite loop of repartitioning + wfDebug( __METHOD__.": repartitioning failed!\n" ); + $this->invalidateTitles( $titleArray ); + } else { + JobQueueGroup::singleton()->push( $jobs ); + } + } + + /** + * @param $rootJobParams array + * @return void + */ + protected function insertPartitionJobs( $rootJobParams = array() ) { + // Carry over any "root job" information + $rootJobParams = $this->getRootJobParams(); + + $batches = $this->blCache->partition( $this->params['table'], $this->rowsPerJob ); + if ( !count( $batches ) ) { + return; // no jobs to insert + } + + $jobs = array(); + foreach ( $batches as $batch ) { + list( $start, $end ) = $batch; + $jobs[] = new HTMLCacheUpdateJob( $this->title, + array( + 'table' => $this->params['table'], + 'start' => $start, + 'end' => $end, + ) + $rootJobParams // carry over information for de-duplication + ); + } + + JobQueueGroup::singleton()->push( $jobs ); + } + + /** + * Invalidate an array (or iterator) of Title objects, right now + * @param $titleArray array + */ + protected function invalidateTitles( $titleArray ) { + global $wgUseFileCache, $wgUseSquid; + + $dbw = wfGetDB( DB_MASTER ); + $timestamp = $dbw->timestamp(); + + # Get all IDs in this query into an array + $ids = array(); + foreach ( $titleArray as $title ) { + $ids[] = $title->getArticleID(); + } + + if ( !$ids ) { + return; + } + + # Don't invalidated pages that were already invalidated + $touchedCond = isset( $this->params['rootJobTimestamp'] ) + ? array( "page_touched < " . + $dbw->addQuotes( $dbw->timestamp( $this->params['rootJobTimestamp'] ) ) ) + : array(); + + # Update page_touched + $batches = array_chunk( $ids, $this->rowsPerQuery ); + foreach ( $batches as $batch ) { + $dbw->update( 'page', + array( 'page_touched' => $timestamp ), + array( 'page_id' => $batch ) + $touchedCond, + __METHOD__ + ); + } + + # Update squid + if ( $wgUseSquid ) { + $u = SquidUpdate::newFromTitles( $titleArray ); + $u->doUpdate(); + } + + # Update file cache + if ( $wgUseFileCache ) { + foreach ( $titleArray as $title ) { + HTMLFileCache::clearFileCache( $title ); + } + } + } +} diff --git a/includes/job/jobs/NullJob.php b/includes/job/jobs/NullJob.php new file mode 100644 index 00000000..d282a8e6 --- /dev/null +++ b/includes/job/jobs/NullJob.php @@ -0,0 +1,60 @@ +<?php +/** + * Degenerate job that does nothing. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * Degenerate job that does nothing, but can optionally replace itself + * in the queue and/or sleep for a brief time period. These can be used + * to represent "no-op" jobs or test lock contention and performance. + * + * @ingroup JobQueue + */ +class NullJob extends Job { + /** + * @param $title Title (can be anything) + * @param array $params job parameters (lives, usleep) + * @param $id Integer: job id + */ + function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'null', $title, $params, $id ); + if ( !isset( $this->params['lives'] ) ) { + $this->params['lives'] = 1; + } + if ( !isset( $this->params['usleep'] ) ) { + $this->params['usleep'] = 0; + } + $this->removeDuplicates = !empty( $this->params['removeDuplicates'] ); + } + + public function run() { + if ( $this->params['usleep'] > 0 ) { + usleep( $this->params['usleep'] ); + } + if ( $this->params['lives'] > 1 ) { + $params = $this->params; + $params['lives']--; + $job = new self( $this->title, $params ); + JobQueueGroup::singleton()->push( $job ); + } + return true; + } +} diff --git a/includes/job/jobs/PublishStashedFileJob.php b/includes/job/jobs/PublishStashedFileJob.php new file mode 100644 index 00000000..d3feda28 --- /dev/null +++ b/includes/job/jobs/PublishStashedFileJob.php @@ -0,0 +1,130 @@ +<?php +/** + * Upload a file from the upload stash into the local file repo. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Upload + */ + +/** + * Upload a file from the upload stash into the local file repo. + * + * @ingroup Upload + */ +class PublishStashedFileJob extends Job { + public function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'PublishStashedFile', $title, $params, $id ); + $this->removeDuplicates = true; + } + + public function run() { + $scope = RequestContext::importScopedSession( $this->params['session'] ); + $context = RequestContext::getMain(); + try { + $user = $context->getUser(); + if ( !$user->isLoggedIn() ) { + $this->setLastError( "Could not load the author user from session." ); + return false; + } + + UploadBase::setSessionStatus( + $this->params['filekey'], + array( 'result' => 'Poll', 'stage' => 'publish', 'status' => Status::newGood() ) + ); + + $upload = new UploadFromStash( $user ); + // @TODO: initialize() causes a GET, ideally we could frontload the antivirus + // checks and anything else to the stash stage (which includes concatenation and + // the local file is thus already there). That way, instead of GET+PUT, there could + // just be a COPY operation from the stash to the public zone. + $upload->initialize( $this->params['filekey'], $this->params['filename'] ); + + // Check if the local file checks out (this is generally a no-op) + $verification = $upload->verifyUpload(); + if ( $verification['status'] !== UploadBase::OK ) { + $status = Status::newFatal( 'verification-error' ); + $status->value = array( 'verification' => $verification ); + UploadBase::setSessionStatus( + $this->params['filekey'], + array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status ) + ); + $this->setLastError( "Could not verify upload." ); + return false; + } + + // Upload the stashed file to a permanent location + $status = $upload->performUpload( + $this->params['comment'], + $this->params['text'], + $this->params['watch'], + $user + ); + if ( !$status->isGood() ) { + UploadBase::setSessionStatus( + $this->params['filekey'], + array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status ) + ); + $this->setLastError( $status->getWikiText() ); + return false; + } + + // Build the image info array while we have the local reference handy + $apiMain = new ApiMain(); // dummy object (XXX) + $imageInfo = $upload->getImageInfo( $apiMain->getResult() ); + + // Cleanup any temporary local file + $upload->cleanupTempFile(); + + // Cache the info so the user doesn't have to wait forever to get the final info + UploadBase::setSessionStatus( + $this->params['filekey'], + array( + 'result' => 'Success', + 'stage' => 'publish', + 'filename' => $upload->getLocalFile()->getName(), + 'imageinfo' => $imageInfo, + 'status' => Status::newGood() + ) + ); + } catch ( MWException $e ) { + UploadBase::setSessionStatus( + $this->params['filekey'], + array( + 'result' => 'Failure', + 'stage' => 'publish', + 'status' => Status::newFatal( 'api-error-publishfailed' ) + ) + ); + $this->setLastError( get_class( $e ) . ": " . $e->getText() ); + return false; + } + return true; + } + + public function getDeduplicationInfo() { + $info = parent::getDeduplicationInfo(); + if ( is_array( $info['params'] ) ) { + $info['params'] = array( 'filekey' => $info['params']['filekey'] ); + } + return $info; + } + + public function allowRetries() { + return false; + } +} diff --git a/includes/job/jobs/RefreshLinksJob.php b/includes/job/jobs/RefreshLinksJob.php new file mode 100644 index 00000000..9dbe8278 --- /dev/null +++ b/includes/job/jobs/RefreshLinksJob.php @@ -0,0 +1,226 @@ +<?php +/** + * Job to update links for a given title. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup JobQueue + */ + +/** + * Background job to update links for a given title. + * + * @ingroup JobQueue + */ +class RefreshLinksJob extends Job { + function __construct( $title, $params = '', $id = 0 ) { + parent::__construct( 'refreshLinks', $title, $params, $id ); + $this->removeDuplicates = true; // job is expensive + } + + /** + * Run a refreshLinks job + * @return boolean success + */ + function run() { + wfProfileIn( __METHOD__ ); + + $linkCache = LinkCache::singleton(); + $linkCache->clear(); + + if ( is_null( $this->title ) ) { + $this->error = "refreshLinks: Invalid title"; + wfProfileOut( __METHOD__ ); + return false; + } + + # Wait for the DB of the current/next slave DB handle to catch up to the master. + # This way, we get the correct page_latest for templates or files that just changed + # milliseconds ago, having triggered this job to begin with. + if ( isset( $this->params['masterPos'] ) ) { + wfGetLB()->waitFor( $this->params['masterPos'] ); + } + + $revision = Revision::newFromTitle( $this->title, false, Revision::READ_NORMAL ); + if ( !$revision ) { + $this->error = 'refreshLinks: Article not found "' . + $this->title->getPrefixedDBkey() . '"'; + wfProfileOut( __METHOD__ ); + return false; // XXX: what if it was just deleted? + } + + self::runForTitleInternal( $this->title, $revision, __METHOD__ ); + + wfProfileOut( __METHOD__ ); + return true; + } + + /** + * @return Array + */ + public function getDeduplicationInfo() { + $info = parent::getDeduplicationInfo(); + // Don't let highly unique "masterPos" values ruin duplicate detection + if ( is_array( $info['params'] ) ) { + unset( $info['params']['masterPos'] ); + } + return $info; + } + + /** + * @param $title Title + * @param $revision Revision + * @param $fname string + * @return void + */ + public static function runForTitleInternal( Title $title, Revision $revision, $fname ) { + wfProfileIn( $fname ); + $content = $revision->getContent( Revision::RAW ); + + if ( !$content ) { + // if there is no content, pretend the content is empty + $content = $revision->getContentHandler()->makeEmptyContent(); + } + + // Revision ID must be passed to the parser output to get revision variables correct + $parserOutput = $content->getParserOutput( $title, $revision->getId(), null, false ); + + $updates = $content->getSecondaryDataUpdates( $title, null, false, $parserOutput ); + DataUpdate::runUpdates( $updates ); + wfProfileOut( $fname ); + } +} + +/** + * Background job to update links for a given title. + * Newer version for high use templates. + * + * @ingroup JobQueue + */ +class RefreshLinksJob2 extends Job { + function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'refreshLinks2', $title, $params, $id ); + } + + /** + * Run a refreshLinks2 job + * @return boolean success + */ + function run() { + global $wgUpdateRowsPerJob; + + wfProfileIn( __METHOD__ ); + + $linkCache = LinkCache::singleton(); + $linkCache->clear(); + + if ( is_null( $this->title ) ) { + $this->error = "refreshLinks2: Invalid title"; + wfProfileOut( __METHOD__ ); + return false; + } + + // Back compat for pre-r94435 jobs + $table = isset( $this->params['table'] ) ? $this->params['table'] : 'templatelinks'; + + // Avoid slave lag when fetching templates. + // When the outermost job is run, we know that the caller that enqueued it must have + // committed the relevant changes to the DB by now. At that point, record the master + // position and pass it along as the job recursively breaks into smaller range jobs. + // Hopefully, when leaf jobs are popped, the slaves will have reached that position. + if ( isset( $this->params['masterPos'] ) ) { + $masterPos = $this->params['masterPos']; + } elseif ( wfGetLB()->getServerCount() > 1 ) { + $masterPos = wfGetLB()->getMasterPos(); + } else { + $masterPos = false; + } + + $tbc = $this->title->getBacklinkCache(); + + $jobs = array(); // jobs to insert + if ( isset( $this->params['start'] ) && isset( $this->params['end'] ) ) { + # This is a partition job to trigger the insertion of leaf jobs... + $jobs = array_merge( $jobs, $this->getSingleTitleJobs( $table, $masterPos ) ); + } else { + # This is a base job to trigger the insertion of partitioned jobs... + if ( $tbc->getNumLinks( $table ) <= $wgUpdateRowsPerJob ) { + # Just directly insert the single per-title jobs + $jobs = array_merge( $jobs, $this->getSingleTitleJobs( $table, $masterPos ) ); + } else { + # Insert the partition jobs to make per-title jobs + foreach ( $tbc->partition( $table, $wgUpdateRowsPerJob ) as $batch ) { + list( $start, $end ) = $batch; + $jobs[] = new RefreshLinksJob2( $this->title, + array( + 'table' => $table, + 'start' => $start, + 'end' => $end, + 'masterPos' => $masterPos, + ) + $this->getRootJobParams() // carry over information for de-duplication + ); + } + } + } + + if ( count( $jobs ) ) { + JobQueueGroup::singleton()->push( $jobs ); + } + + wfProfileOut( __METHOD__ ); + return true; + } + + /** + * @param $table string + * @param $masterPos mixed + * @return Array + */ + protected function getSingleTitleJobs( $table, $masterPos ) { + # The "start"/"end" fields are not set for the base jobs + $start = isset( $this->params['start'] ) ? $this->params['start'] : false; + $end = isset( $this->params['end'] ) ? $this->params['end'] : false; + $titles = $this->title->getBacklinkCache()->getLinks( $table, $start, $end ); + # Convert into single page refresh links jobs. + # This handles well when in sapi mode and is useful in any case for job + # de-duplication. If many pages use template A, and that template itself + # uses template B, then an edit to both will create many duplicate jobs. + # Roughly speaking, for each page, one of the "RefreshLinksJob" jobs will + # get run first, and when it does, it will remove the duplicates. Of course, + # one page could have its job popped when the other page's job is still + # buried within the logic of a refreshLinks2 job. + $jobs = array(); + foreach ( $titles as $title ) { + $jobs[] = new RefreshLinksJob( $title, + array( 'masterPos' => $masterPos ) + $this->getRootJobParams() + ); // carry over information for de-duplication + } + return $jobs; + } + + /** + * @return Array + */ + public function getDeduplicationInfo() { + $info = parent::getDeduplicationInfo(); + // Don't let highly unique "masterPos" values ruin duplicate detection + if ( is_array( $info['params'] ) ) { + unset( $info['params']['masterPos'] ); + } + return $info; + } +} diff --git a/includes/job/jobs/UploadFromUrlJob.php b/includes/job/jobs/UploadFromUrlJob.php new file mode 100644 index 00000000..87549140 --- /dev/null +++ b/includes/job/jobs/UploadFromUrlJob.php @@ -0,0 +1,179 @@ +<?php +/** + * Job for asynchronous upload-by-url. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup JobQueue + */ + +/** + * Job for asynchronous upload-by-url. + * + * This job is in fact an interface to UploadFromUrl, which is designed such + * that it does not require any globals. If it does, fix it elsewhere, do not + * add globals in here. + * + * @ingroup JobQueue + */ +class UploadFromUrlJob extends Job { + const SESSION_KEYNAME = 'wsUploadFromUrlJobData'; + + /** + * @var UploadFromUrl + */ + public $upload; + + /** + * @var User + */ + protected $user; + + public function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'uploadFromUrl', $title, $params, $id ); + } + + public function run() { + # Initialize this object and the upload object + $this->upload = new UploadFromUrl(); + $this->upload->initialize( + $this->title->getText(), + $this->params['url'], + false + ); + $this->user = User::newFromName( $this->params['userName'] ); + + # Fetch the file + $status = $this->upload->fetchFile(); + if ( !$status->isOk() ) { + $this->leaveMessage( $status ); + return true; + } + + # Verify upload + $result = $this->upload->verifyUpload(); + if ( $result['status'] != UploadBase::OK ) { + $status = $this->upload->convertVerifyErrorToStatus( $result ); + $this->leaveMessage( $status ); + return true; + } + + # Check warnings + if ( !$this->params['ignoreWarnings'] ) { + $warnings = $this->upload->checkWarnings(); + if ( $warnings ) { + + # Stash the upload + $key = $this->upload->stashFile(); + + if ( $this->params['leaveMessage'] ) { + $this->user->leaveUserMessage( + wfMessage( 'upload-warning-subj' )->text(), + wfMessage( 'upload-warning-msg', + $key, + $this->params['url'] )->text() + ); + } else { + wfSetupSession( $this->params['sessionId'] ); + $this->storeResultInSession( 'Warning', + 'warnings', $warnings ); + session_write_close(); + } + + return true; + } + } + + # Perform the upload + $status = $this->upload->performUpload( + $this->params['comment'], + $this->params['pageText'], + $this->params['watch'], + $this->user + ); + $this->leaveMessage( $status ); + return true; + + } + + /** + * Leave a message on the user talk page or in the session according to + * $params['leaveMessage']. + * + * @param $status Status + */ + protected function leaveMessage( $status ) { + if ( $this->params['leaveMessage'] ) { + if ( $status->isGood() ) { + $this->user->leaveUserMessage( wfMessage( 'upload-success-subj' )->text(), + wfMessage( 'upload-success-msg', + $this->upload->getTitle()->getText(), + $this->params['url'] + )->text() ); + } else { + $this->user->leaveUserMessage( wfMessage( 'upload-failure-subj' )->text(), + wfMessage( 'upload-failure-msg', + $status->getWikiText(), + $this->params['url'] + )->text() ); + } + } else { + wfSetupSession( $this->params['sessionId'] ); + if ( $status->isOk() ) { + $this->storeResultInSession( 'Success', + 'filename', $this->upload->getLocalFile()->getName() ); + } else { + $this->storeResultInSession( 'Failure', + 'errors', $status->getErrorsArray() ); + } + session_write_close(); + } + } + + /** + * Store a result in the session data. Note that the caller is responsible + * for appropriate session_start and session_write_close calls. + * + * @param string $result the result (Success|Warning|Failure) + * @param string $dataKey the key of the extra data + * @param $dataValue Mixed: the extra data itself + */ + protected function storeResultInSession( $result, $dataKey, $dataValue ) { + $session =& self::getSessionData( $this->params['sessionKey'] ); + $session['result'] = $result; + $session[$dataKey] = $dataValue; + } + + /** + * Initialize the session data. Sets the intial result to queued. + */ + public function initializeSessionData() { + $session =& self::getSessionData( $this->params['sessionKey'] ); + $$session['result'] = 'Queued'; + } + + /** + * @param $key + * @return mixed + */ + public static function &getSessionData( $key ) { + if ( !isset( $_SESSION[self::SESSION_KEYNAME][$key] ) ) { + $_SESSION[self::SESSION_KEYNAME][$key] = array(); + } + return $_SESSION[self::SESSION_KEYNAME][$key]; + } +} |