Merge commit 'fa89acd'

# Conflicts:
#	.gitignore
#	extensions/ArchInterWiki.sql

5 files changed, 153 insertions, 122 deletions

diff --git a/includes/deferred/DeferredUpdates.php b/includes/deferred/DeferredUpdates.php
index 42816ddc..cd0266f6 100644
--- a/includes/deferred/DeferredUpdates.php
+++ b/includes/deferred/DeferredUpdates.php
@@ -34,13 +34,18 @@ interface DeferrableUpdate {
 }
 
 /**
- * Class for managing the deferred updates.
+ * Class for managing the deferred updates
+ *
+ * Deferred updates can be run at the end of the request,
+ * after the HTTP response has been sent. In CLI mode, updates
+ * are only deferred until there is no local master DB transaction.
+ * When updates are deferred, they go into a simple FIFO queue.
  *
  * @since 1.19
  */
 class DeferredUpdates {
 	/**
-	 * Store of updates to be deferred until the end of the request.
+	 * @var array Updates to be deferred until the end of the request.
 	 */
 	private static $updates = array();
 
@@ -49,18 +54,28 @@ class DeferredUpdates {
 	 * @param DeferrableUpdate $update Some object that implements doUpdate()
 	 */
 	public static function addUpdate( DeferrableUpdate $update ) {
+		global $wgCommandLineMode;
+
 		array_push( self::$updates, $update );
-	}
 
-	/**
-	 * HTMLCacheUpdates are the most common deferred update people use. This
-	 * is a shortcut method for that.
-	 * @see HTMLCacheUpdate::__construct()
-	 * @param Title $title
-	 * @param string $table
-	 */
-	public static function addHTMLCacheUpdate( $title, $table ) {
-		self::addUpdate( new HTMLCacheUpdate( $title, $table ) );
+		// CLI scripts may forget to periodically flush these updates,
+		// so try to handle that rather than OOMing and losing them.
+		// Try to run the updates as soon as there is no local transaction.
+		static $waitingOnTrx = false; // de-duplicate callback
+		if ( $wgCommandLineMode && !$waitingOnTrx ) {
+			$lb = wfGetLB();
+			$dbw = $lb->getAnyOpenConnection( $lb->getWriterIndex() );
+			// Do the update as soon as there is no transaction
+			if ( $dbw && $dbw->trxLevel() ) {
+				$waitingOnTrx = true;
+				$dbw->onTransactionIdle( function() use ( &$waitingOnTrx ) {
+					DeferredUpdates::doUpdates();
+					$waitingOnTrx = false;
+				} );
+			} else {
+				self::doUpdates();
+			}
+		}
 	}
 
 	/**
@@ -80,23 +95,9 @@ class DeferredUpdates {
 	 *   prevent lock contention
 	 */
 	public static function doUpdates( $commit = '' ) {
-		global $wgDeferredUpdateList;
-
-		$updates = array_merge( $wgDeferredUpdateList, self::$updates );
+		$updates = self::$updates;
 
-		// No need to get master connections in case of empty updates array
-		if ( !count( $updates ) ) {
-
-			return;
-		}
-
-		$dbw = false;
-		$doCommit = $commit == 'commit';
-		if ( $doCommit ) {
-			$dbw = wfGetDB( DB_MASTER );
-		}
-
-		while ( $updates ) {
+		while ( count( $updates ) ) {
 			self::clearPendingUpdates();
 
 			/** @var DeferrableUpdate $update */
@@ -104,8 +105,8 @@ class DeferredUpdates {
 				try {
 					$update->doUpdate();
 
-					if ( $doCommit && $dbw->trxLevel() ) {
-						$dbw->commit( __METHOD__, 'flush' );
+					if ( $commit === 'commit' ) {
+						wfGetLBFactory()->commitMasterChanges();
 					}
 				} catch ( Exception $e ) {
 					// We don't want exceptions thrown during deferred updates to
@@ -116,9 +117,9 @@ class DeferredUpdates {
 					}
 				}
 			}
-			$updates = array_merge( $wgDeferredUpdateList, self::$updates );
-		}
 
+			$updates = self::$updates;
+		}
 	}
 
 	/**
@@ -126,7 +127,6 @@ class DeferredUpdates {
 	 * want or need to call this. Unit tests need it though.
 	 */
 	public static function clearPendingUpdates() {
-		global $wgDeferredUpdateList;
-		$wgDeferredUpdateList = self::$updates = array();
+		self::$updates = array();
 	}
 }
diff --git a/includes/deferred/HTMLCacheUpdate.php b/includes/deferred/HTMLCacheUpdate.php
index 79a10e68..a480aec8 100644
--- a/includes/deferred/HTMLCacheUpdate.php
+++ b/includes/deferred/HTMLCacheUpdate.php
@@ -55,8 +55,7 @@ class HTMLCacheUpdate implements DeferrableUpdate {
 
 		$count = $this->mTitle->getBacklinkCache()->getNumLinks( $this->mTable, 100 );
 		if ( $count >= 100 ) { // many backlinks
-			JobQueueGroup::singleton()->push( $job );
-			JobQueueGroup::singleton()->deduplicateRootJob( $job );
+			JobQueueGroup::singleton()->lazyPush( $job );
 		} else { // few backlinks ($count might be off even if 0)
 			$dbw = wfGetDB( DB_MASTER );
 			$dbw->onTransactionIdle( function () use ( $job ) {
diff --git a/includes/deferred/LinksDeletionUpdate.php b/includes/deferred/LinksDeletionUpdate.php
new file mode 100644
index 00000000..bbdfcf17
--- /dev/null
+++ b/includes/deferred/LinksDeletionUpdate.php
@@ -0,0 +1,105 @@
+<?php
+/**
+ * Updater for link tracking tables after a page edit.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+/**
+ * Update object handling the cleanup of links tables after a page was deleted.
+ **/
+class LinksDeletionUpdate extends SqlDataUpdate {
+	/** @var WikiPage The WikiPage that was deleted */
+	protected $mPage;
+
+	/**
+	 * Constructor
+	 *
+	 * @param WikiPage $page Page we are updating
+	 * @throws MWException
+	 */
+	function __construct( WikiPage $page ) {
+		parent::__construct( false ); // no implicit transaction
+
+		$this->mPage = $page;
+
+		if ( !$page->exists() ) {
+			throw new MWException( "Page ID not known, perhaps the page doesn't exist?" );
+		}
+	}
+
+	/**
+	 * Do some database updates after deletion
+	 */
+	public function doUpdate() {
+		$title = $this->mPage->getTitle();
+		$id = $this->mPage->getId();
+
+		# Delete restrictions for it
+		$this->mDb->delete( 'page_restrictions', array( 'pr_page' => $id ), __METHOD__ );
+
+		# Fix category table counts
+		$cats = array();
+		$res = $this->mDb->select( 'categorylinks', 'cl_to', array( 'cl_from' => $id ), __METHOD__ );
+
+		foreach ( $res as $row ) {
+			$cats[] = $row->cl_to;
+		}
+
+		$this->mPage->updateCategoryCounts( array(), $cats );
+
+		# If using cascading deletes, we can skip some explicit deletes
+		if ( !$this->mDb->cascadingDeletes() ) {
+			# Delete outgoing links
+			$this->mDb->delete( 'pagelinks', array( 'pl_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'imagelinks', array( 'il_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'categorylinks', array( 'cl_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'templatelinks', array( 'tl_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'externallinks', array( 'el_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'langlinks', array( 'll_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'iwlinks', array( 'iwl_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'redirect', array( 'rd_from' => $id ), __METHOD__ );
+			$this->mDb->delete( 'page_props', array( 'pp_page' => $id ), __METHOD__ );
+		}
+
+		# If using cleanup triggers, we can skip some manual deletes
+		if ( !$this->mDb->cleanupTriggers() ) {
+			# Find recentchanges entries to clean up...
+			$rcIdsForTitle = $this->mDb->selectFieldValues( 'recentchanges',
+				'rc_id',
+				array(
+					'rc_type != ' . RC_LOG,
+					'rc_namespace' => $title->getNamespace(),
+					'rc_title' => $title->getDBkey()
+				),
+				__METHOD__
+			);
+			$rcIdsForPage = $this->mDb->selectFieldValues( 'recentchanges',
+				'rc_id',
+				array( 'rc_type != ' . RC_LOG, 'rc_cur_id' => $id ),
+				__METHOD__
+			);
+
+			# T98706: delete PK to avoid lock contention with RC delete log insertions
+			$rcIds = array_merge( $rcIdsForTitle, $rcIdsForPage );
+			if ( $rcIds ) {
+				$this->mDb->delete( 'recentchanges', array( 'rc_id' => $rcIds ), __METHOD__ );
+			}
+		}
+	}
+}
\ No newline at end of file
diff --git a/includes/deferred/LinksUpdate.php b/includes/deferred/LinksUpdate.php
index e4f00e75..be5aff3b 100644
--- a/includes/deferred/LinksUpdate.php
+++ b/includes/deferred/LinksUpdate.php
@@ -267,7 +267,6 @@ class LinksUpdate extends SqlDataUpdate {
 			);
 
 			JobQueueGroup::singleton()->push( $job );
-			JobQueueGroup::singleton()->deduplicateRootJob( $job );
 		}
 	}
 
@@ -936,87 +935,3 @@ class LinksUpdate extends SqlDataUpdate {
 		}
 	}
 }
-
-/**
- * Update object handling the cleanup of links tables after a page was deleted.
- **/
-class LinksDeletionUpdate extends SqlDataUpdate {
-	/** @var WikiPage The WikiPage that was deleted */
-	protected $mPage;
-
-	/**
-	 * Constructor
-	 *
-	 * @param WikiPage $page Page we are updating
-	 * @throws MWException
-	 */
-	function __construct( WikiPage $page ) {
-		parent::__construct( false ); // no implicit transaction
-
-		$this->mPage = $page;
-
-		if ( !$page->exists() ) {
-			throw new MWException( "Page ID not known, perhaps the page doesn't exist?" );
-		}
-	}
-
-	/**
-	 * Do some database updates after deletion
-	 */
-	public function doUpdate() {
-		$title = $this->mPage->getTitle();
-		$id = $this->mPage->getId();
-
-		# Delete restrictions for it
-		$this->mDb->delete( 'page_restrictions', array( 'pr_page' => $id ), __METHOD__ );
-
-		# Fix category table counts
-		$cats = array();
-		$res = $this->mDb->select( 'categorylinks', 'cl_to', array( 'cl_from' => $id ), __METHOD__ );
-
-		foreach ( $res as $row ) {
-			$cats[] = $row->cl_to;
-		}
-
-		$this->mPage->updateCategoryCounts( array(), $cats );
-
-		# If using cascading deletes, we can skip some explicit deletes
-		if ( !$this->mDb->cascadingDeletes() ) {
-			# Delete outgoing links
-			$this->mDb->delete( 'pagelinks', array( 'pl_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'imagelinks', array( 'il_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'categorylinks', array( 'cl_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'templatelinks', array( 'tl_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'externallinks', array( 'el_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'langlinks', array( 'll_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'iwlinks', array( 'iwl_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'redirect', array( 'rd_from' => $id ), __METHOD__ );
-			$this->mDb->delete( 'page_props', array( 'pp_page' => $id ), __METHOD__ );
-		}
-
-		# If using cleanup triggers, we can skip some manual deletes
-		if ( !$this->mDb->cleanupTriggers() ) {
-			# Clean up recentchanges entries...
-			$this->mDb->delete( 'recentchanges',
-				array( 'rc_type != ' . RC_LOG,
-					'rc_namespace' => $title->getNamespace(),
-					'rc_title' => $title->getDBkey() ),
-				__METHOD__ );
-			$this->mDb->delete( 'recentchanges',
-				array( 'rc_type != ' . RC_LOG, 'rc_cur_id' => $id ),
-				__METHOD__ );
-		}
-	}
-
-	/**
-	 * Update all the appropriate counts in the category table.
-	 * @param array $added Associative array of category name => sort key
-	 * @param array $deleted Associative array of category name => sort key
-	 */
-	function updateCategoryCounts( $added, $deleted ) {
-		$a = WikiPage::factory( $this->mTitle );
-		$a->updateCategoryCounts(
-			array_keys( $added ), array_keys( $deleted )
-		);
-	}
-}
diff --git a/includes/deferred/SiteStatsUpdate.php b/includes/deferred/SiteStatsUpdate.php
index 97a17c39..ae75a754 100644
--- a/includes/deferred/SiteStatsUpdate.php
+++ b/includes/deferred/SiteStatsUpdate.php
@@ -65,6 +65,8 @@ class SiteStatsUpdate implements DeferrableUpdate {
 	public function doUpdate() {
 		global $wgSiteStatsAsyncFactor;
 
+		$this->doUpdateContextStats();
+
 		$rate = $wgSiteStatsAsyncFactor; // convenience
 		// If set to do so, only do actual DB updates 1 every $rate times.
 		// The other times, just update "pending delta" values in memcached.
@@ -128,7 +130,7 @@ class SiteStatsUpdate implements DeferrableUpdate {
 	 */
 	public static function cacheUpdate( $dbw ) {
 		global $wgActiveUserDays;
-		$dbr = wfGetDB( DB_SLAVE, array( 'SpecialStatistics', 'vslow' ) );
+		$dbr = wfGetDB( DB_SLAVE, 'vslow' );
 		# Get non-bot users than did some recent action other than making accounts.
 		# If account creation is included, the number gets inflated ~20+ fold on enwiki.
 		$activeUsers = $dbr->selectField(
@@ -153,6 +155,16 @@ class SiteStatsUpdate implements DeferrableUpdate {
 		return $activeUsers;
 	}
 
+	protected function doUpdateContextStats() {
+		$stats = RequestContext::getMain()->getStats();
+		foreach ( array( 'edits', 'articles', 'pages', 'users', 'images' ) as $type ) {
+			$delta = $this->$type;
+			if ( $delta !== 0 ) {
+				$stats->updateCount( "site.$type", $delta );
+			}
+		}
+	}
+
 	protected function doUpdatePendingDeltas() {
 		$this->adjustPending( 'ss_total_edits', $this->edits );
 		$this->adjustPending( 'ss_good_articles', $this->articles );