Merge commit 'fa89acd'

# Conflicts:
#	.gitignore
#	extensions/ArchInterWiki.sql

12 files changed, 874 insertions, 49 deletions

diff --git a/includes/utils/AutoloadGenerator.php b/includes/utils/AutoloadGenerator.php
index 9cf8cab5..7d631563 100644
--- a/includes/utils/AutoloadGenerator.php
+++ b/includes/utils/AutoloadGenerator.php
@@ -119,13 +119,49 @@ class AutoloadGenerator {
 	}
 
 	/**
-	 * Write out all known classes to autoload.php in
-	 * the provided basedir
+	 * Updates the AutoloadClasses field at the given
+	 * filename.
 	 *
-	 * @param string $commandName Value used in file comment to direct
-	 *  developers towards the appropriate way to update the autoload.
+	 * @param {string} $filename Filename of JSON
+	 *  extension/skin registration file
 	 */
-	public function generateAutoload( $commandName = 'AutoloadGenerator' ) {
+	protected function generateJsonAutoload( $filename ) {
+		require_once __DIR__ . '/../../includes/json/FormatJson.php';
+		$key = 'AutoloadClasses';
+		$json = FormatJson::decode( file_get_contents( $filename ), true );
+		unset( $json[$key] );
+		// Inverting the key-value pairs so that they become of the
+		// format class-name : path when they get converted into json.
+		foreach ( $this->classes as $path => $contained ) {
+			foreach ( $contained as $fqcn ) {
+
+				// Using substr to remove the leading '/'
+				$json[$key][$fqcn] = substr( $path, 1 );
+			}
+		}
+		foreach ( $this->overrides as $path => $fqcn ) {
+
+			// Using substr to remove the leading '/'
+			$json[$key][$fqcn] = substr( $path, 1 );
+		}
+
+		// Sorting the list of autoload classes.
+		ksort( $json[$key] );
+
+		// Update file, using constants for the required
+		// formatting.
+		file_put_contents( $filename,
+			FormatJson::encode( $json, true ) . "\n" );
+	}
+
+	/**
+	 * Generates a PHP file setting up autoload information.
+	 *
+	 * @param {string} $commandName Command name to include in comment
+	 * @param {string} $filename of PHP file to put autoload information in.
+	 */
+	protected function generatePHPAutoload( $commandName, $filename ) {
+		// No existing JSON file found; update/generate PHP file
 		$content = array();
 
 		// We need to generate a line each rather than exporting the
@@ -163,7 +199,7 @@ class AutoloadGenerator {
 
 		$output = implode( "\n\t", $content );
 		file_put_contents(
-			$this->basepath . '/autoload.php',
+			$filename,
 			<<<EOD
 <?php
 // This file is generated by $commandName, do not adjust manually
@@ -176,9 +212,35 @@ global \${$this->variableName};
 
 EOD
 		);
+
 	}
 
 	/**
+	 * Write out all known classes to autoload.php, extension.json, or skin.json in
+	 * the provided basedir
+	 *
+	 * @param string $commandName Value used in file comment to direct
+	 *  developers towards the appropriate way to update the autoload.
+	 */
+	public function generateAutoload( $commandName = 'AutoloadGenerator' ) {
+
+		// We need to check whether an extenson.json or skin.json exists or not, and
+		// incase it doesn't, update the autoload.php file.
+
+		$jsonFilename = null;
+		if ( file_exists( $this->basepath . "/extension.json" ) ) {
+			$jsonFilename = $this->basepath . "/extension.json";
+		} elseif ( file_exists( $this->basepath . "/skin.json" ) ) {
+			$jsonFilename = $this->basepath . "/skin.json";
+		}
+
+		if ( $jsonFilename !== null ) {
+			$this->generateJsonAutoload( $jsonFilename );
+		} else {
+			$this->generatePHPAutoload( $commandName, $this->basepath . '/autoload.php' );
+		}
+	}
+	/**
 	 * Ensure that Unix-style path separators ("/") are used in the path.
 	 *
 	 * @param string $path
diff --git a/includes/utils/AvroValidator.php b/includes/utils/AvroValidator.php
new file mode 100644
index 00000000..4f8e0b17
--- /dev/null
+++ b/includes/utils/AvroValidator.php
@@ -0,0 +1,184 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+/**
+ * Generate error strings for data that doesn't match the specified
+ * Avro schema. This is very similar to AvroSchema::is_valid_datum(),
+ * but returns error messages instead of a boolean.
+ *
+ * @since 1.26
+ * @author Erik Bernhardson <ebernhardson@wikimedia.org>
+ * @copyright © 2015 Erik Bernhardson and Wikimedia Foundation.
+ */
+class AvroValidator {
+	/**
+	 * @param AvroSchema $schema The rules to conform to.
+	 * @param mixed $datum The value to validate against $schema.
+	 * @return string|string[] An error or list of errors in the
+	 *  provided $datum. When no errors exist the empty array is
+	 *  returned.
+	 */
+	public static function getErrors( AvroSchema $schema, $datum ) {
+		switch ( $schema->type) {
+		case AvroSchema::NULL_TYPE:
+			if ( !is_null($datum) ) {
+				return self::wrongType( 'null', $datum );
+			}
+			return array();
+		case AvroSchema::BOOLEAN_TYPE:
+			if ( !is_bool($datum) ) {
+				return self::wrongType( 'boolean', $datum );
+			}
+			return array();
+		case AvroSchema::STRING_TYPE:
+		case AvroSchema::BYTES_TYPE:
+			if ( !is_string($datum) ) {
+				return self::wrongType( 'string', $datum );
+			}
+			return array();
+		case AvroSchema::INT_TYPE:
+			if ( !is_int($datum) ) {
+				return self::wrongType( 'integer', $datum );
+			}
+			if ( AvroSchema::INT_MIN_VALUE > $datum
+				|| $datum > AvroSchema::INT_MAX_VALUE
+			) {
+				return self::outOfRange(
+					AvroSchema::INT_MIN_VALUE,
+					AvroSchema::INT_MAX_VALUE,
+					$datum
+				);
+			}
+			return array();
+		case AvroSchema::LONG_TYPE:
+			if ( !is_int($datum) ) {
+				return self::wrongType( 'integer', $datum );
+			}
+			if ( AvroSchema::LONG_MIN_VALUE > $datum
+				|| $datum > AvroSchema::LONG_MAX_VALUE
+			) {
+				return self::outOfRange(
+					AvroSchema::LONG_MIN_VALUE,
+					AvroSchema::LONG_MAX_VALUE,
+					$datum
+				);
+			}
+			return array();
+		case AvroSchema::FLOAT_TYPE:
+		case AvroSchema::DOUBLE_TYPE:
+			if ( !is_float($datum) && !is_int($datum) ) {
+				return self::wrongType( 'float or integer', $datum );
+			}
+			return array();
+		case AvroSchema::ARRAY_SCHEMA:
+			if (!is_array($datum)) {
+				return self::wrongType( 'array', $datum );
+			}
+			$errors = array();
+			foreach ($datum as $d) {
+				$result = $this->validate( $schema->items(), $d );
+				if ( $result ) {
+					$errors[] = $result;
+				}
+			}
+			if ( $errors ) {
+				return $errors;
+			}
+			return array();
+		case AvroSchema::MAP_SCHEMA:
+			if (!is_array($datum)) {
+				return self::wrongType( 'array', $datum );
+			}
+			$errors = array();
+			foreach ($datum as $k => $v) {
+			if ( !is_string($k) ) {
+					$errors[] = self::wrongType( 'string key', $k );
+				}
+				$result = self::getErrors( $schema->values(), $v );
+				if ( $result ) {
+					$errors[$k] = $result;
+				}
+			}
+			return $errors;
+		case AvroSchema::UNION_SCHEMA:
+			$errors = array();
+			foreach ($schema->schemas() as $schema) {
+				$result = self::getErrors( $schema, $datum );
+				if ( !$result ) {
+					return array();
+				}
+				$errors[] = $result;
+			}
+			if ( $errors ) {
+				return array( "Expected any one of these to be true", $errors );
+			}
+			return "No schemas provided to union";
+		case AvroSchema::ENUM_SCHEMA:
+			if ( !in_array( $datum, $schema->symbols() ) ) {
+				$symbols = implode( ', ', $schema->symbols );
+				return "Expected one of $symbols but recieved $datum";
+			}
+			return array();
+		case AvroSchema::FIXED_SCHEMA:
+			if ( !is_string( $datum ) ) {
+				return self::wrongType( 'string', $datum );
+			}
+			$len = strlen( $datum );
+			if ( $len !== $schema->size() ) {
+				return "Expected string of length {$schema->size()}, "
+					. "but recieved one of length $len";
+			}
+			return array();
+		case AvroSchema::RECORD_SCHEMA:
+		case AvroSchema::ERROR_SCHEMA:
+		case AvroSchema::REQUEST_SCHEMA:
+			if ( !is_array( $datum ) ) {
+				return self::wrongType( 'array', $datum );
+			}
+			$errors = array();
+			foreach ( $schema->fields() as $field ) {
+				$name = $field->name();
+				if ( !array_key_exists( $name, $datum ) ) {
+					$errors[$name] = 'Missing expected field';
+					continue;
+				}
+				$result = self::getErrors( $field->type(), $datum[$name] );
+				if ( $result ) {
+					$errors[$name] = $result;
+				}
+			}
+			return $errors;
+		default:
+			return "Unknown avro schema type: {$schema->type}";
+		}
+	}
+
+	public static function typeOf( $datum ) {
+		return is_object( $datum ) ? get_class( $datum ) : gettype( $datum );
+	}
+
+	public static function wrongType( $expected, $datum ) {
+		return "Expected $expected, but recieved " . self::typeOf( $datum );
+	}
+
+	public static function outOfRange( $min, $max, $datum ) {
+		return "Expected value between $min and $max, but recieved $datum";
+	}
+}
diff --git a/includes/utils/BatchRowIterator.php b/includes/utils/BatchRowIterator.php
new file mode 100644
index 00000000..9441608a
--- /dev/null
+++ b/includes/utils/BatchRowIterator.php
@@ -0,0 +1,278 @@
+<?php
+/**
+ * Allows iterating a large number of rows in batches transparently.
+ * By default when iterated over returns the full query result as an
+ * array of rows.  Can be wrapped in RecursiveIteratorIterator to
+ * collapse those arrays into a single stream of rows queried in batches.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+class BatchRowIterator implements RecursiveIterator {
+
+	/**
+	 * @var DatabaseBase $db The database to read from
+	 */
+	protected $db;
+
+	/**
+	 * @var string $table The name of the table to read from
+	 */
+	protected $table;
+
+	/**
+	 * @var array $primaryKey The name of the primary key(s)
+	 */
+	protected $primaryKey;
+
+	/**
+	 * @var integer $batchSize The number of rows to fetch per iteration
+	 */
+	protected $batchSize;
+
+	/**
+	 * @var array $conditions Array of strings containing SQL conditions
+	 *  to add to the query
+	 */
+	protected $conditions = array();
+
+	/**
+	 * @var array $joinConditions
+	 */
+	protected $joinConditions = array();
+
+	/**
+	 * @var array $fetchColumns List of column names to select from the
+	 *  table suitable for use with DatabaseBase::select()
+	 */
+	protected $fetchColumns;
+
+	/**
+	 * @var string $orderBy SQL Order by condition generated from $this->primaryKey
+	 */
+	protected $orderBy;
+
+	/**
+	 * @var array $current The current iterator value
+	 */
+	private $current = array();
+
+	/**
+	 * @var integer key 0-indexed number of pages fetched since self::reset()
+	 */
+	private $key;
+
+	/**
+	 * @param DatabaseBase $db         The database to read from
+	 * @param string       $table      The name of the table to read from
+	 * @param string|array $primaryKey The name or names of the primary key columns
+	 * @param integer      $batchSize  The number of rows to fetch per iteration
+	 * @throws MWException
+	 */
+	public function __construct( DatabaseBase $db, $table, $primaryKey, $batchSize ) {
+		if ( $batchSize < 1 ) {
+			throw new MWException( 'Batch size must be at least 1 row.' );
+		}
+		$this->db = $db;
+		$this->table = $table;
+		$this->primaryKey = (array) $primaryKey;
+		$this->fetchColumns = $this->primaryKey;
+		$this->orderBy = implode( ' ASC,', $this->primaryKey ) . ' ASC';
+		$this->batchSize = $batchSize;
+	}
+
+	/**
+	 * @param array $condition Query conditions suitable for use with
+	 *  DatabaseBase::select
+	 */
+	public function addConditions( array $conditions ) {
+		$this->conditions = array_merge( $this->conditions, $conditions );
+	}
+
+	/**
+	 * @param array $condition Query join conditions suitable for use
+	 *  with DatabaseBase::select
+	 */
+	public function addJoinConditions( array $conditions ) {
+		$this->joinConditions = array_merge( $this->joinConditions, $conditions );
+	}
+
+	/**
+	 * @param array $columns List of column names to select from the
+	 *  table suitable for use with DatabaseBase::select()
+	 */
+	public function setFetchColumns( array $columns ) {
+		// If it's not the all column selector merge in the primary keys we need
+		if ( count( $columns ) === 1 && reset( $columns ) === '*' ) {
+			$this->fetchColumns = $columns;
+		} else {
+			$this->fetchColumns = array_unique( array_merge(
+				$this->primaryKey,
+				$columns
+			) );
+		}
+	}
+
+	/**
+	 * Extracts the primary key(s) from a database row.
+	 *
+	 * @param stdClass $row An individual database row from this iterator
+	 * @return array Map of primary key column to value within the row
+	 */
+	public function extractPrimaryKeys( $row ) {
+		$pk = array();
+		foreach ( $this->primaryKey as $column ) {
+			$pk[$column] = $row->$column;
+		}
+		return $pk;
+	}
+
+	/**
+	 * @return array The most recently fetched set of rows from the database
+	 */
+	public function current() {
+		return $this->current;
+	}
+
+	/**
+	 * @return integer 0-indexed count of the page number fetched
+	 */
+	public function key() {
+		return $this->key;
+	}
+
+	/**
+	 * Reset the iterator to the begining of the table.
+	 */
+	public function rewind() {
+		$this->key = -1; // self::next() will turn this into 0
+		$this->current = array();
+		$this->next();
+	}
+
+	/**
+	 * @return boolean True when the iterator is in a valid state
+	 */
+	public function valid() {
+		return (bool) $this->current;
+	}
+
+	/**
+	 * @return boolean True when this result set has rows
+	 */
+	public function hasChildren() {
+		return $this->current && count( $this->current );
+	}
+
+	/**
+	 * @return RecursiveIterator
+	 */
+	public function getChildren() {
+		return new NotRecursiveIterator( new ArrayIterator( $this->current ) );
+	}
+
+	/**
+	 * Fetch the next set of rows from the database.
+	 */
+	public function next() {
+		$res = $this->db->select(
+			$this->table,
+			$this->fetchColumns,
+			$this->buildConditions(),
+			__METHOD__,
+			array(
+				'LIMIT' => $this->batchSize,
+				'ORDER BY' => $this->orderBy,
+			),
+			$this->joinConditions
+		);
+
+		// The iterator is converted to an array because in addition to
+		// returning it in self::current() we need to use the end value
+		// in self::buildConditions()
+		$this->current = iterator_to_array( $res );
+		$this->key++;
+	}
+
+	/**
+	 * Uses the primary key list and the maximal result row from the
+	 * previous iteration to build an SQL condition sufficient for
+	 * selecting the next page of results.  All except the final key use
+	 * `=` conditions while the final key uses a `>` condition
+	 *
+	 * Example output:
+	 * 	  array( '( foo = 42 AND bar > 7 ) OR ( foo > 42 )' )
+	 *
+	 * @return array The SQL conditions necessary to select the next set
+	 *  of rows in the batched query
+	 */
+	protected function buildConditions() {
+		if ( !$this->current ) {
+			return $this->conditions;
+		}
+
+		$maxRow = end( $this->current );
+		$maximumValues = array();
+		foreach ( $this->primaryKey as $column ) {
+			$maximumValues[$column] = $this->db->addQuotes( $maxRow->$column );
+		}
+
+		$pkConditions = array();
+		// For example: If we have 3 primary keys
+		// first run through will generate
+		//   col1 = 4 AND col2 = 7 AND col3 > 1
+		// second run through will generate
+		//   col1 = 4 AND col2 > 7
+		// and the final run through will generate
+		//   col1 > 4
+		while ( $maximumValues ) {
+			$pkConditions[] = $this->buildGreaterThanCondition( $maximumValues );
+			array_pop( $maximumValues );
+		}
+
+		$conditions = $this->conditions;
+		$conditions[] = sprintf( '( %s )', implode( ' ) OR ( ', $pkConditions ) );
+
+		return $conditions;
+	}
+
+	/**
+	 * Given an array of column names and their maximum value  generate
+	 * an SQL condition where all keys except the last match $quotedMaximumValues
+	 * exactly and the last column is greater than the matching value in
+	 * $quotedMaximumValues
+	 *
+	 * @param array $quotedMaximumValues The maximum values quoted with
+	 *  $this->db->addQuotes()
+	 * @return string An SQL condition that will select rows where all
+	 *  columns match the maximum value exactly except the last column
+	 *  which must be greater than the provided maximum value
+	 */
+	protected function buildGreaterThanCondition( array $quotedMaximumValues ) {
+		$keys = array_keys( $quotedMaximumValues );
+		$lastColumn = end( $keys );
+		$lastValue = array_pop( $quotedMaximumValues );
+		$conditions = array();
+		foreach ( $quotedMaximumValues as $column => $value ) {
+			$conditions[] = "$column = $value";
+		}
+		$conditions[] = "$lastColumn > $lastValue";
+
+		return implode( ' AND ', $conditions );
+	}
+}
diff --git a/includes/utils/BatchRowUpdate.php b/includes/utils/BatchRowUpdate.php
new file mode 100644
index 00000000..a4257a53
--- /dev/null
+++ b/includes/utils/BatchRowUpdate.php
@@ -0,0 +1,133 @@
+<?php
+/*
+ * Ties together the batch update components to provide a composable
+ * method of batch updating rows in a database. To use create a class
+ * implementing the RowUpdateGenerator interface and configure the
+ * BatchRowIterator and BatchRowWriter for access to the correct table.
+ * The components will handle reading, writing, and waiting for slaves
+ * while the generator implementation handles generating update arrays
+ * for singular rows.
+ *
+ * Instantiate:
+ *   $updater = new BatchRowUpdate(
+ *       new BatchRowIterator( $dbr, 'some_table', 'primary_key_column', 500 ),
+ *       new BatchRowWriter( $dbw, 'some_table', 'clusterName' ),
+ *       new MyImplementationOfRowUpdateGenerator
+ *   );
+ *
+ * Run:
+ *   $updater->execute();
+ *
+ * An example maintenance script utilizing the BatchRowUpdate can be
+ * located in the Echo extension file maintenance/updateSchema.php
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+class BatchRowUpdate {
+	/**
+	 * @var BatchRowIterator $reader Iterator that returns an array of
+	 *  database rows
+	 */
+	protected $reader;
+
+	/**
+	 * @var BatchRowWriter $writer Writer capable of pushing row updates
+	 *  to the database
+	 */
+	protected $writer;
+
+	/**
+	 * @var RowUpdateGenerator $generator Generates single row updates
+	 *  based on the rows content
+	 */
+	protected $generator;
+
+	/**
+	 * @var callable $output Output callback
+	 */
+	protected $output;
+
+	/**
+	 * @param BatchRowIterator   $reader    Iterator that returns an
+	 *  array of database rows
+	 * @param BatchRowWriter     $writer    Writer capable of pushing
+	 *  row updates to the database
+	 * @param RowUpdateGenerator $generator Generates single row updates
+	 *  based on the rows content
+	 */
+	public function __construct( BatchRowIterator $reader, BatchRowWriter $writer, RowUpdateGenerator $generator ) {
+		$this->reader = $reader;
+		$this->writer = $writer;
+		$this->generator = $generator;
+		$this->output = function() {
+		}; // nop
+	}
+
+	/**
+	 * Runs the batch update process
+	 */
+	public function execute() {
+		foreach ( $this->reader as $rows ) {
+			$updates = array();
+			foreach ( $rows as $row ) {
+				$update = $this->generator->update( $row );
+				if ( $update ) {
+					$updates[] = array(
+						'primaryKey' => $this->reader->extractPrimaryKeys( $row ),
+						'changes' => $update,
+					);
+				}
+			}
+
+			if ( $updates ) {
+				$this->output( "Processing " . count( $updates ) . " rows\n" );
+				$this->writer->write( $updates );
+			}
+		}
+
+		$this->output( "Completed\n" );
+	}
+
+	/**
+	 * Accepts a callable which will receive a single parameter
+	 * containing string status updates
+	 *
+	 * @param callable $output A callback taking a single string
+	 *  parameter to output
+	 *
+	 * @throws MWException
+	 */
+	public function setOutput( $output ) {
+		if ( !is_callable( $output ) ) {
+			throw new MWException(
+				'Provided $output param is required to be callable.'
+			);
+		}
+		$this->output = $output;
+	}
+
+	/**
+	 * Write out a status update
+	 *
+	 * @param string $text The value to print
+	 */
+	protected function output( $text ) {
+		call_user_func( $this->output, $text );
+	}
+}
diff --git a/includes/utils/BatchRowWriter.php b/includes/utils/BatchRowWriter.php
new file mode 100644
index 00000000..04c00a3d
--- /dev/null
+++ b/includes/utils/BatchRowWriter.php
@@ -0,0 +1,71 @@
+<?php
+/**
+ * Updates database rows by primary key in batches.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+class BatchRowWriter {
+	/**
+	 * @var DatabaseBase $db The database to write to
+	 */
+	protected $db;
+
+	/**
+	 * @var string $table The name of the table to update
+	 */
+	protected $table;
+
+	/**
+	 * @var string $clusterName A cluster name valid for use with LBFactory
+	 */
+	protected $clusterName;
+
+	/**
+	 * @param DatabaseBase $db          The database to write to
+	 * @param string       $table       The name of the table to update
+	 * @param string|bool  $clusterName A cluster name valid for use with LBFactory
+	 */
+	public function __construct( DatabaseBase $db, $table, $clusterName = false ) {
+		$this->db = $db;
+		$this->table = $table;
+		$this->clusterName = $clusterName;
+	}
+
+	/**
+	 * @param array $updates Array of arrays each containing two keys, 'primaryKey'
+	 *  and 'changes'. primaryKey must contain a map of column names to values
+	 *  sufficient to uniquely identify the row changes must contain a map of column
+	 *  names to update values to apply to the row.
+	 */
+	public function write( array $updates ) {
+		$this->db->begin();
+
+		foreach ( $updates as $update ) {
+			$this->db->update(
+				$this->table,
+				$update['changes'],
+				$update['primaryKey'],
+				__METHOD__
+			);
+		}
+
+		$this->db->commit();
+		wfWaitForSlaves( false, false, $this->clusterName );
+	}
+}
diff --git a/includes/utils/IP.php b/includes/utils/IP.php
index 4441236d..13586f3c 100644
--- a/includes/utils/IP.php
+++ b/includes/utils/IP.php
@@ -21,6 +21,8 @@
  * @author Antoine Musso "<hashar at free dot fr>", Aaron Schulz
  */
 
+use IPSet\IPSet;
+
 // Some regex definition to "play" with IP address and IP address blocks
 
 // An IPv4 address is made of 4 bytes from x00 to xFF which is d0 to d255
@@ -130,8 +132,9 @@ class IP {
 
 	/**
 	 * Convert an IP into a verbose, uppercase, normalized form.
-	 * IPv6 addresses in octet notation are expanded to 8 words.
-	 * IPv4 addresses are just trimmed.
+	 * Both IPv4 and IPv6 addresses are trimmed. Additionally,
+	 * IPv6 addresses in octet notation are expanded to 8 words;
+	 * IPv4 addresses have leading zeros, in each octet, removed.
 	 *
 	 * @param string $ip IP address in quad or octet form (CIDR or not).
 	 * @return string
@@ -141,8 +144,16 @@ class IP {
 		if ( $ip === '' ) {
 			return null;
 		}
-		if ( self::isIPv4( $ip ) || !self::isIPv6( $ip ) ) {
-			return $ip; // nothing else to do for IPv4 addresses or invalid ones
+		/* If not an IP, just return trimmed value, since sanitizeIP() is called
+		 * in a number of contexts where usernames are supplied as input.
+		 */
+		if ( !self::isIPAddress($ip) ) {
+			return $ip;
+		}
+		if ( self::isIPv4( $ip ) ) {
+			// Remove leading 0's from octet representation of IPv4 address
+			$ip = preg_replace( '/(?:^|(?<=\.))0+(?=[1-9]|0\.|0$)/', '', $ip );
+			return $ip;
 		}
 		// Remove any whitespaces, convert to upper case
 		$ip = strtoupper( $ip );
@@ -240,7 +251,7 @@ class IP {
 	 * A bare IPv6 address is accepted despite the lack of square brackets.
 	 *
 	 * @param string $both The string with the host and port
-	 * @return array
+	 * @return array|false Array normally, false on certain failures
 	 */
 	public static function splitHostAndPort( $both ) {
 		if ( substr( $both, 0, 1 ) === '[' ) {
@@ -375,6 +386,8 @@ class IP {
 				'127.0.0.0/8', # loopback
 				'fc00::/7', # RFC 4193 (local)
 				'0:0:0:0:0:0:0:1', # loopback
+				'169.254.0.0/16', # link-local
+				'fe80::/10', # link-local
 			) );
 		}
 		return !$privateSet->match( $ip );
@@ -395,8 +408,9 @@ class IP {
 		if ( self::isIPv6( $ip ) ) {
 			$n = 'v6-' . self::IPv6ToRawHex( $ip );
 		} elseif ( self::isIPv4( $ip ) ) {
-			// Bug 60035: an IP with leading 0's fails in ip2long sometimes (e.g. *.08)
-			$ip = preg_replace( '/(?<=\.)0+(?=[1-9])/', '', $ip );
+			// T62035/T97897: An IP with leading 0's fails in ip2long sometimes (e.g. *.08),
+			// also double/triple 0 needs to be changed to just a single 0 for ip2long.
+			$ip = self::sanitizeIP( $ip );
 			$n = ip2long( $ip );
 			if ( $n < 0 ) {
 				$n += pow( 2, 32 );
diff --git a/includes/utils/MWCryptHKDF.php b/includes/utils/MWCryptHKDF.php
index 950dd846..740df922 100644
--- a/includes/utils/MWCryptHKDF.php
+++ b/includes/utils/MWCryptHKDF.php
@@ -161,7 +161,7 @@ class MWCryptHKDF {
 	 * @throws MWException
 	 */
 	protected static function singleton() {
-		global $wgHKDFAlgorithm, $wgHKDFSecret, $wgSecretKey;
+		global $wgHKDFAlgorithm, $wgHKDFSecret, $wgSecretKey, $wgMainCacheType;
 
 		$secret = $wgHKDFSecret ?: $wgSecretKey;
 		if ( !$secret ) {
@@ -176,11 +176,7 @@ class MWCryptHKDF {
 		$context[] = gethostname();
 
 		// Setup salt cache. Use APC, or fallback to the main cache if it isn't setup
-		try {
-			$cache = ObjectCache::newAccelerator( array() );
-		} catch ( Exception $e ) {
-			$cache = wfGetMainCache();
-		}
+		$cache = ObjectCache::newAccelerator( $wgMainCacheType );
 
 		if ( is_null( self::$singleton ) ) {
 			self::$singleton = new self( $secret, $wgHKDFAlgorithm, $cache, $context );
diff --git a/includes/utils/MWCryptRand.php b/includes/utils/MWCryptRand.php
index e6c0e784..f2237909 100644
--- a/includes/utils/MWCryptRand.php
+++ b/includes/utils/MWCryptRand.php
@@ -96,9 +96,9 @@ class MWCryptRand {
 		}
 
 		foreach ( $files as $file ) {
-			wfSuppressWarnings();
+			MediaWiki\suppressWarnings();
 			$stat = stat( $file );
-			wfRestoreWarnings();
+			MediaWiki\restoreWarnings();
 			if ( $stat ) {
 				// stat() duplicates data into numeric and string keys so kill off all the numeric ones
 				foreach ( $stat as $k => $v ) {
@@ -363,9 +363,9 @@ class MWCryptRand {
 			}
 			// /dev/urandom is generally considered the best possible commonly
 			// available random source, and is available on most *nix systems.
-			wfSuppressWarnings();
+			MediaWiki\suppressWarnings();
 			$urandom = fopen( "/dev/urandom", "rb" );
-			wfRestoreWarnings();
+			MediaWiki\restoreWarnings();
 
 			// Attempt to read all our random data from urandom
 			// php's fread always does buffered reads based on the stream's chunk_size
diff --git a/includes/utils/RowUpdateGenerator.php b/includes/utils/RowUpdateGenerator.php
new file mode 100644
index 00000000..6a4792cb
--- /dev/null
+++ b/includes/utils/RowUpdateGenerator.php
@@ -0,0 +1,39 @@
+<?php
+/**
+ * Interface for generating updates to single rows in the database.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+interface RowUpdateGenerator {
+	/**
+	 * Given a database row, generates an array mapping column names to
+	 * updated value within the database row.
+	 *
+	 * Sample Response:
+	 *   return array(
+	 *       'some_col' => 'new value',
+	 *       'other_col' => 99,
+	 *   );
+	 *
+	 * @param stdClass $row A row from the database
+	 * @return array Map of column names to updated value within the
+	 *  database row. When no update is required returns an empty array.
+	 */
+	public function update( $row );
+}
diff --git a/includes/utils/UIDGenerator.php b/includes/utils/UIDGenerator.php
index 92415877..04c8e194 100644
--- a/includes/utils/UIDGenerator.php
+++ b/includes/utils/UIDGenerator.php
@@ -20,6 +20,7 @@
  * @file
  * @author Aaron Schulz
  */
+use Wikimedia\Assert\Assert;
 
 /**
  * Class for getting statistically unique IDs
@@ -51,7 +52,7 @@ class UIDGenerator {
 		}
 		// Try to get some ID that uniquely identifies this machine (RFC 4122)...
 		if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) {
-			wfSuppressWarnings();
+			MediaWiki\suppressWarnings();
 			if ( wfIsWindows() ) {
 				// http://technet.microsoft.com/en-us/library/bb490913.aspx
 				$csv = trim( wfShellExec( 'getmac /NH /FO CSV' ) );
@@ -65,7 +66,7 @@ class UIDGenerator {
 					wfShellExec( '/sbin/ifconfig -a' ), $m );
 				$nodeId = isset( $m[1] ) ? str_replace( ':', '', $m[1] ) : '';
 			}
-			wfRestoreWarnings();
+			MediaWiki\restoreWarnings();
 			if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) {
 				$nodeId = MWCryptRand::generateHex( 12, true );
 				$nodeId[1] = dechex( hexdec( $nodeId[1] ) | 0x1 ); // set multicast bit
@@ -107,9 +108,10 @@ class UIDGenerator {
 	 * @throws MWException
 	 */
 	public static function newTimestampedUID88( $base = 10 ) {
-		if ( !is_integer( $base ) || $base > 36 || $base < 2 ) {
-			throw new MWException( "Base must an integer be between 2 and 36" );
-		}
+		Assert::parameterType( 'integer', $base, '$base' );
+		Assert::parameter( $base <= 36, '$base', 'must be <= 36' );
+		Assert::parameter( $base >= 2, '$base', 'must be >= 2' );
+
 		$gen = self::singleton();
 		$time = $gen->getTimestampAndDelay( 'lockFile88', 1, 1024 );
 
@@ -152,9 +154,10 @@ class UIDGenerator {
 	 * @throws MWException
 	 */
 	public static function newTimestampedUID128( $base = 10 ) {
-		if ( !is_integer( $base ) || $base > 36 || $base < 2 ) {
-			throw new MWException( "Base must be an integer between 2 and 36" );
-		}
+		Assert::parameterType( 'integer', $base, '$base' );
+		Assert::parameter( $base <= 36, '$base', 'must be <= 36' );
+		Assert::parameter( $base >= 2, '$base', 'must be >= 2' );
+
 		$gen = self::singleton();
 		$time = $gen->getTimestampAndDelay( 'lockFile128', 16384, 1048576 );
 
@@ -280,7 +283,7 @@ class UIDGenerator {
 		$cache = null;
 		if ( ( $flags & self::QUICK_VOLATILE ) && PHP_SAPI !== 'cli' ) {
 			try {
-				$cache = ObjectCache::newAccelerator( array() );
+				$cache = ObjectCache::newAccelerator();
 			} catch ( Exception $e ) {
 				// not supported
 			}
diff --git a/includes/utils/iterators/IteratorDecorator.php b/includes/utils/iterators/IteratorDecorator.php
new file mode 100644
index 00000000..c1b50207
--- /dev/null
+++ b/includes/utils/iterators/IteratorDecorator.php
@@ -0,0 +1,50 @@
+<?php
+/**
+ * Allows extending classes to decorate an Iterator with
+ * reduced boilerplate.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+abstract class IteratorDecorator implements Iterator {
+	protected $iterator;
+
+	public function __construct( Iterator $iterator ) {
+		$this->iterator = $iterator;
+	}
+
+	public function current() {
+		return $this->iterator->current();
+	}
+
+	public function key() {
+		return $this->iterator->key();
+	}
+
+	public function next() {
+		$this->iterator->next();
+	}
+
+	public function rewind() {
+		$this->iterator->rewind();
+	}
+
+	public function valid() {
+		return $this->iterator->valid();
+	}
+}
diff --git a/includes/utils/MWFunction.php b/includes/utils/iterators/NotRecursiveIterator.php
index fa7eebe8..52ca61b4 100644
--- a/includes/utils/MWFunction.php
+++ b/includes/utils/iterators/NotRecursiveIterator.php
@@ -1,6 +1,10 @@
 <?php
 /**
- * Helper methods to call functions and instance objects.
+ * Wraps a non-recursive iterator with methods to be recursive
+ * without children.
+ *
+ * Alternatively wraps a recursive iterator to prevent recursing deeper
+ * than the wrapped iterator.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -18,23 +22,14 @@
  * http://www.gnu.org/copyleft/gpl.html
  *
  * @file
+ * @ingroup Maintenance
  */
+class NotRecursiveIterator extends IteratorDecorator implements RecursiveIterator {
+	public function hasChildren() {
+		return false;
+	}
 
-class MWFunction {
-
-	/**
-	 * @param string $class
-	 * @param array $args
-	 * @return object
-	 * @deprecated 1.25 Use ObjectFactory::getObjectFromSpec() instead
-	 */
-	public static function newObj( $class, $args = array() ) {
-		wfDeprecated( __METHOD__, '1.25' );
-
-		return ObjectFactory::getObjectFromSpec( array(
-			'class' => $class,
-			'args' => $args,
-			'closure_expansion' => false,
-		) );
+	public function getChildren() {
+		return null;
 	}
 }