diff options
Diffstat (limited to 'includes/utils')
| -rw-r--r-- | includes/utils/AutoloadGenerator.php | 74 | ||||
| -rw-r--r-- | includes/utils/AvroValidator.php | 184 | ||||
| -rw-r--r-- | includes/utils/BatchRowIterator.php | 278 | ||||
| -rw-r--r-- | includes/utils/BatchRowUpdate.php | 133 | ||||
| -rw-r--r-- | includes/utils/BatchRowWriter.php | 71 | ||||
| -rw-r--r-- | includes/utils/IP.php | 28 | ||||
| -rw-r--r-- | includes/utils/MWCryptHKDF.php | 8 | ||||
| -rw-r--r-- | includes/utils/MWCryptRand.php | 8 | ||||
| -rw-r--r-- | includes/utils/RowUpdateGenerator.php | 39 | ||||
| -rw-r--r-- | includes/utils/UIDGenerator.php | 21 | ||||
| -rw-r--r-- | includes/utils/iterators/IteratorDecorator.php | 50 | ||||
| -rw-r--r-- | includes/utils/iterators/NotRecursiveIterator.php (renamed from includes/utils/MWFunction.php) | 29 |
12 files changed, 874 insertions, 49 deletions
diff --git a/includes/utils/AutoloadGenerator.php b/includes/utils/AutoloadGenerator.php index 9cf8cab5..7d631563 100644 --- a/includes/utils/AutoloadGenerator.php +++ b/includes/utils/AutoloadGenerator.php @@ -119,13 +119,49 @@ class AutoloadGenerator { } /** - * Write out all known classes to autoload.php in - * the provided basedir + * Updates the AutoloadClasses field at the given + * filename. * - * @param string $commandName Value used in file comment to direct - * developers towards the appropriate way to update the autoload. + * @param {string} $filename Filename of JSON + * extension/skin registration file */ - public function generateAutoload( $commandName = 'AutoloadGenerator' ) { + protected function generateJsonAutoload( $filename ) { + require_once __DIR__ . '/../../includes/json/FormatJson.php'; + $key = 'AutoloadClasses'; + $json = FormatJson::decode( file_get_contents( $filename ), true ); + unset( $json[$key] ); + // Inverting the key-value pairs so that they become of the + // format class-name : path when they get converted into json. + foreach ( $this->classes as $path => $contained ) { + foreach ( $contained as $fqcn ) { + + // Using substr to remove the leading '/' + $json[$key][$fqcn] = substr( $path, 1 ); + } + } + foreach ( $this->overrides as $path => $fqcn ) { + + // Using substr to remove the leading '/' + $json[$key][$fqcn] = substr( $path, 1 ); + } + + // Sorting the list of autoload classes. + ksort( $json[$key] ); + + // Update file, using constants for the required + // formatting. + file_put_contents( $filename, + FormatJson::encode( $json, true ) . "\n" ); + } + + /** + * Generates a PHP file setting up autoload information. + * + * @param {string} $commandName Command name to include in comment + * @param {string} $filename of PHP file to put autoload information in. + */ + protected function generatePHPAutoload( $commandName, $filename ) { + // No existing JSON file found; update/generate PHP file $content = array(); // We need to generate a line each rather than exporting the @@ -163,7 +199,7 @@ class AutoloadGenerator { $output = implode( "\n\t", $content ); file_put_contents( - $this->basepath . '/autoload.php', + $filename, <<<EOD <?php // This file is generated by $commandName, do not adjust manually @@ -176,9 +212,35 @@ global \${$this->variableName}; EOD ); + } /** + * Write out all known classes to autoload.php, extension.json, or skin.json in + * the provided basedir + * + * @param string $commandName Value used in file comment to direct + * developers towards the appropriate way to update the autoload. + */ + public function generateAutoload( $commandName = 'AutoloadGenerator' ) { + + // We need to check whether an extenson.json or skin.json exists or not, and + // incase it doesn't, update the autoload.php file. + + $jsonFilename = null; + if ( file_exists( $this->basepath . "/extension.json" ) ) { + $jsonFilename = $this->basepath . "/extension.json"; + } elseif ( file_exists( $this->basepath . "/skin.json" ) ) { + $jsonFilename = $this->basepath . "/skin.json"; + } + + if ( $jsonFilename !== null ) { + $this->generateJsonAutoload( $jsonFilename ); + } else { + $this->generatePHPAutoload( $commandName, $this->basepath . '/autoload.php' ); + } + } + /** * Ensure that Unix-style path separators ("/") are used in the path. * * @param string $path diff --git a/includes/utils/AvroValidator.php b/includes/utils/AvroValidator.php new file mode 100644 index 00000000..4f8e0b17 --- /dev/null +++ b/includes/utils/AvroValidator.php @@ -0,0 +1,184 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Generate error strings for data that doesn't match the specified + * Avro schema. This is very similar to AvroSchema::is_valid_datum(), + * but returns error messages instead of a boolean. + * + * @since 1.26 + * @author Erik Bernhardson <ebernhardson@wikimedia.org> + * @copyright © 2015 Erik Bernhardson and Wikimedia Foundation. + */ +class AvroValidator { + /** + * @param AvroSchema $schema The rules to conform to. + * @param mixed $datum The value to validate against $schema. + * @return string|string[] An error or list of errors in the + * provided $datum. When no errors exist the empty array is + * returned. + */ + public static function getErrors( AvroSchema $schema, $datum ) { + switch ( $schema->type) { + case AvroSchema::NULL_TYPE: + if ( !is_null($datum) ) { + return self::wrongType( 'null', $datum ); + } + return array(); + case AvroSchema::BOOLEAN_TYPE: + if ( !is_bool($datum) ) { + return self::wrongType( 'boolean', $datum ); + } + return array(); + case AvroSchema::STRING_TYPE: + case AvroSchema::BYTES_TYPE: + if ( !is_string($datum) ) { + return self::wrongType( 'string', $datum ); + } + return array(); + case AvroSchema::INT_TYPE: + if ( !is_int($datum) ) { + return self::wrongType( 'integer', $datum ); + } + if ( AvroSchema::INT_MIN_VALUE > $datum + || $datum > AvroSchema::INT_MAX_VALUE + ) { + return self::outOfRange( + AvroSchema::INT_MIN_VALUE, + AvroSchema::INT_MAX_VALUE, + $datum + ); + } + return array(); + case AvroSchema::LONG_TYPE: + if ( !is_int($datum) ) { + return self::wrongType( 'integer', $datum ); + } + if ( AvroSchema::LONG_MIN_VALUE > $datum + || $datum > AvroSchema::LONG_MAX_VALUE + ) { + return self::outOfRange( + AvroSchema::LONG_MIN_VALUE, + AvroSchema::LONG_MAX_VALUE, + $datum + ); + } + return array(); + case AvroSchema::FLOAT_TYPE: + case AvroSchema::DOUBLE_TYPE: + if ( !is_float($datum) && !is_int($datum) ) { + return self::wrongType( 'float or integer', $datum ); + } + return array(); + case AvroSchema::ARRAY_SCHEMA: + if (!is_array($datum)) { + return self::wrongType( 'array', $datum ); + } + $errors = array(); + foreach ($datum as $d) { + $result = $this->validate( $schema->items(), $d ); + if ( $result ) { + $errors[] = $result; + } + } + if ( $errors ) { + return $errors; + } + return array(); + case AvroSchema::MAP_SCHEMA: + if (!is_array($datum)) { + return self::wrongType( 'array', $datum ); + } + $errors = array(); + foreach ($datum as $k => $v) { + if ( !is_string($k) ) { + $errors[] = self::wrongType( 'string key', $k ); + } + $result = self::getErrors( $schema->values(), $v ); + if ( $result ) { + $errors[$k] = $result; + } + } + return $errors; + case AvroSchema::UNION_SCHEMA: + $errors = array(); + foreach ($schema->schemas() as $schema) { + $result = self::getErrors( $schema, $datum ); + if ( !$result ) { + return array(); + } + $errors[] = $result; + } + if ( $errors ) { + return array( "Expected any one of these to be true", $errors ); + } + return "No schemas provided to union"; + case AvroSchema::ENUM_SCHEMA: + if ( !in_array( $datum, $schema->symbols() ) ) { + $symbols = implode( ', ', $schema->symbols ); + return "Expected one of $symbols but recieved $datum"; + } + return array(); + case AvroSchema::FIXED_SCHEMA: + if ( !is_string( $datum ) ) { + return self::wrongType( 'string', $datum ); + } + $len = strlen( $datum ); + if ( $len !== $schema->size() ) { + return "Expected string of length {$schema->size()}, " + . "but recieved one of length $len"; + } + return array(); + case AvroSchema::RECORD_SCHEMA: + case AvroSchema::ERROR_SCHEMA: + case AvroSchema::REQUEST_SCHEMA: + if ( !is_array( $datum ) ) { + return self::wrongType( 'array', $datum ); + } + $errors = array(); + foreach ( $schema->fields() as $field ) { + $name = $field->name(); + if ( !array_key_exists( $name, $datum ) ) { + $errors[$name] = 'Missing expected field'; + continue; + } + $result = self::getErrors( $field->type(), $datum[$name] ); + if ( $result ) { + $errors[$name] = $result; + } + } + return $errors; + default: + return "Unknown avro schema type: {$schema->type}"; + } + } + + public static function typeOf( $datum ) { + return is_object( $datum ) ? get_class( $datum ) : gettype( $datum ); + } + + public static function wrongType( $expected, $datum ) { + return "Expected $expected, but recieved " . self::typeOf( $datum ); + } + + public static function outOfRange( $min, $max, $datum ) { + return "Expected value between $min and $max, but recieved $datum"; + } +} diff --git a/includes/utils/BatchRowIterator.php b/includes/utils/BatchRowIterator.php new file mode 100644 index 00000000..9441608a --- /dev/null +++ b/includes/utils/BatchRowIterator.php @@ -0,0 +1,278 @@ +<?php +/** + * Allows iterating a large number of rows in batches transparently. + * By default when iterated over returns the full query result as an + * array of rows. Can be wrapped in RecursiveIteratorIterator to + * collapse those arrays into a single stream of rows queried in batches. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +class BatchRowIterator implements RecursiveIterator { + + /** + * @var DatabaseBase $db The database to read from + */ + protected $db; + + /** + * @var string $table The name of the table to read from + */ + protected $table; + + /** + * @var array $primaryKey The name of the primary key(s) + */ + protected $primaryKey; + + /** + * @var integer $batchSize The number of rows to fetch per iteration + */ + protected $batchSize; + + /** + * @var array $conditions Array of strings containing SQL conditions + * to add to the query + */ + protected $conditions = array(); + + /** + * @var array $joinConditions + */ + protected $joinConditions = array(); + + /** + * @var array $fetchColumns List of column names to select from the + * table suitable for use with DatabaseBase::select() + */ + protected $fetchColumns; + + /** + * @var string $orderBy SQL Order by condition generated from $this->primaryKey + */ + protected $orderBy; + + /** + * @var array $current The current iterator value + */ + private $current = array(); + + /** + * @var integer key 0-indexed number of pages fetched since self::reset() + */ + private $key; + + /** + * @param DatabaseBase $db The database to read from + * @param string $table The name of the table to read from + * @param string|array $primaryKey The name or names of the primary key columns + * @param integer $batchSize The number of rows to fetch per iteration + * @throws MWException + */ + public function __construct( DatabaseBase $db, $table, $primaryKey, $batchSize ) { + if ( $batchSize < 1 ) { + throw new MWException( 'Batch size must be at least 1 row.' ); + } + $this->db = $db; + $this->table = $table; + $this->primaryKey = (array) $primaryKey; + $this->fetchColumns = $this->primaryKey; + $this->orderBy = implode( ' ASC,', $this->primaryKey ) . ' ASC'; + $this->batchSize = $batchSize; + } + + /** + * @param array $condition Query conditions suitable for use with + * DatabaseBase::select + */ + public function addConditions( array $conditions ) { + $this->conditions = array_merge( $this->conditions, $conditions ); + } + + /** + * @param array $condition Query join conditions suitable for use + * with DatabaseBase::select + */ + public function addJoinConditions( array $conditions ) { + $this->joinConditions = array_merge( $this->joinConditions, $conditions ); + } + + /** + * @param array $columns List of column names to select from the + * table suitable for use with DatabaseBase::select() + */ + public function setFetchColumns( array $columns ) { + // If it's not the all column selector merge in the primary keys we need + if ( count( $columns ) === 1 && reset( $columns ) === '*' ) { + $this->fetchColumns = $columns; + } else { + $this->fetchColumns = array_unique( array_merge( + $this->primaryKey, + $columns + ) ); + } + } + + /** + * Extracts the primary key(s) from a database row. + * + * @param stdClass $row An individual database row from this iterator + * @return array Map of primary key column to value within the row + */ + public function extractPrimaryKeys( $row ) { + $pk = array(); + foreach ( $this->primaryKey as $column ) { + $pk[$column] = $row->$column; + } + return $pk; + } + + /** + * @return array The most recently fetched set of rows from the database + */ + public function current() { + return $this->current; + } + + /** + * @return integer 0-indexed count of the page number fetched + */ + public function key() { + return $this->key; + } + + /** + * Reset the iterator to the begining of the table. + */ + public function rewind() { + $this->key = -1; // self::next() will turn this into 0 + $this->current = array(); + $this->next(); + } + + /** + * @return boolean True when the iterator is in a valid state + */ + public function valid() { + return (bool) $this->current; + } + + /** + * @return boolean True when this result set has rows + */ + public function hasChildren() { + return $this->current && count( $this->current ); + } + + /** + * @return RecursiveIterator + */ + public function getChildren() { + return new NotRecursiveIterator( new ArrayIterator( $this->current ) ); + } + + /** + * Fetch the next set of rows from the database. + */ + public function next() { + $res = $this->db->select( + $this->table, + $this->fetchColumns, + $this->buildConditions(), + __METHOD__, + array( + 'LIMIT' => $this->batchSize, + 'ORDER BY' => $this->orderBy, + ), + $this->joinConditions + ); + + // The iterator is converted to an array because in addition to + // returning it in self::current() we need to use the end value + // in self::buildConditions() + $this->current = iterator_to_array( $res ); + $this->key++; + } + + /** + * Uses the primary key list and the maximal result row from the + * previous iteration to build an SQL condition sufficient for + * selecting the next page of results. All except the final key use + * `=` conditions while the final key uses a `>` condition + * + * Example output: + * array( '( foo = 42 AND bar > 7 ) OR ( foo > 42 )' ) + * + * @return array The SQL conditions necessary to select the next set + * of rows in the batched query + */ + protected function buildConditions() { + if ( !$this->current ) { + return $this->conditions; + } + + $maxRow = end( $this->current ); + $maximumValues = array(); + foreach ( $this->primaryKey as $column ) { + $maximumValues[$column] = $this->db->addQuotes( $maxRow->$column ); + } + + $pkConditions = array(); + // For example: If we have 3 primary keys + // first run through will generate + // col1 = 4 AND col2 = 7 AND col3 > 1 + // second run through will generate + // col1 = 4 AND col2 > 7 + // and the final run through will generate + // col1 > 4 + while ( $maximumValues ) { + $pkConditions[] = $this->buildGreaterThanCondition( $maximumValues ); + array_pop( $maximumValues ); + } + + $conditions = $this->conditions; + $conditions[] = sprintf( '( %s )', implode( ' ) OR ( ', $pkConditions ) ); + + return $conditions; + } + + /** + * Given an array of column names and their maximum value generate + * an SQL condition where all keys except the last match $quotedMaximumValues + * exactly and the last column is greater than the matching value in + * $quotedMaximumValues + * + * @param array $quotedMaximumValues The maximum values quoted with + * $this->db->addQuotes() + * @return string An SQL condition that will select rows where all + * columns match the maximum value exactly except the last column + * which must be greater than the provided maximum value + */ + protected function buildGreaterThanCondition( array $quotedMaximumValues ) { + $keys = array_keys( $quotedMaximumValues ); + $lastColumn = end( $keys ); + $lastValue = array_pop( $quotedMaximumValues ); + $conditions = array(); + foreach ( $quotedMaximumValues as $column => $value ) { + $conditions[] = "$column = $value"; + } + $conditions[] = "$lastColumn > $lastValue"; + + return implode( ' AND ', $conditions ); + } +} diff --git a/includes/utils/BatchRowUpdate.php b/includes/utils/BatchRowUpdate.php new file mode 100644 index 00000000..a4257a53 --- /dev/null +++ b/includes/utils/BatchRowUpdate.php @@ -0,0 +1,133 @@ +<?php +/* + * Ties together the batch update components to provide a composable + * method of batch updating rows in a database. To use create a class + * implementing the RowUpdateGenerator interface and configure the + * BatchRowIterator and BatchRowWriter for access to the correct table. + * The components will handle reading, writing, and waiting for slaves + * while the generator implementation handles generating update arrays + * for singular rows. + * + * Instantiate: + * $updater = new BatchRowUpdate( + * new BatchRowIterator( $dbr, 'some_table', 'primary_key_column', 500 ), + * new BatchRowWriter( $dbw, 'some_table', 'clusterName' ), + * new MyImplementationOfRowUpdateGenerator + * ); + * + * Run: + * $updater->execute(); + * + * An example maintenance script utilizing the BatchRowUpdate can be + * located in the Echo extension file maintenance/updateSchema.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +class BatchRowUpdate { + /** + * @var BatchRowIterator $reader Iterator that returns an array of + * database rows + */ + protected $reader; + + /** + * @var BatchRowWriter $writer Writer capable of pushing row updates + * to the database + */ + protected $writer; + + /** + * @var RowUpdateGenerator $generator Generates single row updates + * based on the rows content + */ + protected $generator; + + /** + * @var callable $output Output callback + */ + protected $output; + + /** + * @param BatchRowIterator $reader Iterator that returns an + * array of database rows + * @param BatchRowWriter $writer Writer capable of pushing + * row updates to the database + * @param RowUpdateGenerator $generator Generates single row updates + * based on the rows content + */ + public function __construct( BatchRowIterator $reader, BatchRowWriter $writer, RowUpdateGenerator $generator ) { + $this->reader = $reader; + $this->writer = $writer; + $this->generator = $generator; + $this->output = function() { + }; // nop + } + + /** + * Runs the batch update process + */ + public function execute() { + foreach ( $this->reader as $rows ) { + $updates = array(); + foreach ( $rows as $row ) { + $update = $this->generator->update( $row ); + if ( $update ) { + $updates[] = array( + 'primaryKey' => $this->reader->extractPrimaryKeys( $row ), + 'changes' => $update, + ); + } + } + + if ( $updates ) { + $this->output( "Processing " . count( $updates ) . " rows\n" ); + $this->writer->write( $updates ); + } + } + + $this->output( "Completed\n" ); + } + + /** + * Accepts a callable which will receive a single parameter + * containing string status updates + * + * @param callable $output A callback taking a single string + * parameter to output + * + * @throws MWException + */ + public function setOutput( $output ) { + if ( !is_callable( $output ) ) { + throw new MWException( + 'Provided $output param is required to be callable.' + ); + } + $this->output = $output; + } + + /** + * Write out a status update + * + * @param string $text The value to print + */ + protected function output( $text ) { + call_user_func( $this->output, $text ); + } +} diff --git a/includes/utils/BatchRowWriter.php b/includes/utils/BatchRowWriter.php new file mode 100644 index 00000000..04c00a3d --- /dev/null +++ b/includes/utils/BatchRowWriter.php @@ -0,0 +1,71 @@ +<?php +/** + * Updates database rows by primary key in batches. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +class BatchRowWriter { + /** + * @var DatabaseBase $db The database to write to + */ + protected $db; + + /** + * @var string $table The name of the table to update + */ + protected $table; + + /** + * @var string $clusterName A cluster name valid for use with LBFactory + */ + protected $clusterName; + + /** + * @param DatabaseBase $db The database to write to + * @param string $table The name of the table to update + * @param string|bool $clusterName A cluster name valid for use with LBFactory + */ + public function __construct( DatabaseBase $db, $table, $clusterName = false ) { + $this->db = $db; + $this->table = $table; + $this->clusterName = $clusterName; + } + + /** + * @param array $updates Array of arrays each containing two keys, 'primaryKey' + * and 'changes'. primaryKey must contain a map of column names to values + * sufficient to uniquely identify the row changes must contain a map of column + * names to update values to apply to the row. + */ + public function write( array $updates ) { + $this->db->begin(); + + foreach ( $updates as $update ) { + $this->db->update( + $this->table, + $update['changes'], + $update['primaryKey'], + __METHOD__ + ); + } + + $this->db->commit(); + wfWaitForSlaves( false, false, $this->clusterName ); + } +} diff --git a/includes/utils/IP.php b/includes/utils/IP.php index 4441236d..13586f3c 100644 --- a/includes/utils/IP.php +++ b/includes/utils/IP.php @@ -21,6 +21,8 @@ * @author Antoine Musso "<hashar at free dot fr>", Aaron Schulz */ +use IPSet\IPSet; + // Some regex definition to "play" with IP address and IP address blocks // An IPv4 address is made of 4 bytes from x00 to xFF which is d0 to d255 @@ -130,8 +132,9 @@ class IP { /** * Convert an IP into a verbose, uppercase, normalized form. - * IPv6 addresses in octet notation are expanded to 8 words. - * IPv4 addresses are just trimmed. + * Both IPv4 and IPv6 addresses are trimmed. Additionally, + * IPv6 addresses in octet notation are expanded to 8 words; + * IPv4 addresses have leading zeros, in each octet, removed. * * @param string $ip IP address in quad or octet form (CIDR or not). * @return string @@ -141,8 +144,16 @@ class IP { if ( $ip === '' ) { return null; } - if ( self::isIPv4( $ip ) || !self::isIPv6( $ip ) ) { - return $ip; // nothing else to do for IPv4 addresses or invalid ones + /* If not an IP, just return trimmed value, since sanitizeIP() is called + * in a number of contexts where usernames are supplied as input. + */ + if ( !self::isIPAddress($ip) ) { + return $ip; + } + if ( self::isIPv4( $ip ) ) { + // Remove leading 0's from octet representation of IPv4 address + $ip = preg_replace( '/(?:^|(?<=\.))0+(?=[1-9]|0\.|0$)/', '', $ip ); + return $ip; } // Remove any whitespaces, convert to upper case $ip = strtoupper( $ip ); @@ -240,7 +251,7 @@ class IP { * A bare IPv6 address is accepted despite the lack of square brackets. * * @param string $both The string with the host and port - * @return array + * @return array|false Array normally, false on certain failures */ public static function splitHostAndPort( $both ) { if ( substr( $both, 0, 1 ) === '[' ) { @@ -375,6 +386,8 @@ class IP { '127.0.0.0/8', # loopback 'fc00::/7', # RFC 4193 (local) '0:0:0:0:0:0:0:1', # loopback + '169.254.0.0/16', # link-local + 'fe80::/10', # link-local ) ); } return !$privateSet->match( $ip ); @@ -395,8 +408,9 @@ class IP { if ( self::isIPv6( $ip ) ) { $n = 'v6-' . self::IPv6ToRawHex( $ip ); } elseif ( self::isIPv4( $ip ) ) { - // Bug 60035: an IP with leading 0's fails in ip2long sometimes (e.g. *.08) - $ip = preg_replace( '/(?<=\.)0+(?=[1-9])/', '', $ip ); + // T62035/T97897: An IP with leading 0's fails in ip2long sometimes (e.g. *.08), + // also double/triple 0 needs to be changed to just a single 0 for ip2long. + $ip = self::sanitizeIP( $ip ); $n = ip2long( $ip ); if ( $n < 0 ) { $n += pow( 2, 32 ); diff --git a/includes/utils/MWCryptHKDF.php b/includes/utils/MWCryptHKDF.php index 950dd846..740df922 100644 --- a/includes/utils/MWCryptHKDF.php +++ b/includes/utils/MWCryptHKDF.php @@ -161,7 +161,7 @@ class MWCryptHKDF { * @throws MWException */ protected static function singleton() { - global $wgHKDFAlgorithm, $wgHKDFSecret, $wgSecretKey; + global $wgHKDFAlgorithm, $wgHKDFSecret, $wgSecretKey, $wgMainCacheType; $secret = $wgHKDFSecret ?: $wgSecretKey; if ( !$secret ) { @@ -176,11 +176,7 @@ class MWCryptHKDF { $context[] = gethostname(); // Setup salt cache. Use APC, or fallback to the main cache if it isn't setup - try { - $cache = ObjectCache::newAccelerator( array() ); - } catch ( Exception $e ) { - $cache = wfGetMainCache(); - } + $cache = ObjectCache::newAccelerator( $wgMainCacheType ); if ( is_null( self::$singleton ) ) { self::$singleton = new self( $secret, $wgHKDFAlgorithm, $cache, $context ); diff --git a/includes/utils/MWCryptRand.php b/includes/utils/MWCryptRand.php index e6c0e784..f2237909 100644 --- a/includes/utils/MWCryptRand.php +++ b/includes/utils/MWCryptRand.php @@ -96,9 +96,9 @@ class MWCryptRand { } foreach ( $files as $file ) { - wfSuppressWarnings(); + MediaWiki\suppressWarnings(); $stat = stat( $file ); - wfRestoreWarnings(); + MediaWiki\restoreWarnings(); if ( $stat ) { // stat() duplicates data into numeric and string keys so kill off all the numeric ones foreach ( $stat as $k => $v ) { @@ -363,9 +363,9 @@ class MWCryptRand { } // /dev/urandom is generally considered the best possible commonly // available random source, and is available on most *nix systems. - wfSuppressWarnings(); + MediaWiki\suppressWarnings(); $urandom = fopen( "/dev/urandom", "rb" ); - wfRestoreWarnings(); + MediaWiki\restoreWarnings(); // Attempt to read all our random data from urandom // php's fread always does buffered reads based on the stream's chunk_size diff --git a/includes/utils/RowUpdateGenerator.php b/includes/utils/RowUpdateGenerator.php new file mode 100644 index 00000000..6a4792cb --- /dev/null +++ b/includes/utils/RowUpdateGenerator.php @@ -0,0 +1,39 @@ +<?php +/** + * Interface for generating updates to single rows in the database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +interface RowUpdateGenerator { + /** + * Given a database row, generates an array mapping column names to + * updated value within the database row. + * + * Sample Response: + * return array( + * 'some_col' => 'new value', + * 'other_col' => 99, + * ); + * + * @param stdClass $row A row from the database + * @return array Map of column names to updated value within the + * database row. When no update is required returns an empty array. + */ + public function update( $row ); +} diff --git a/includes/utils/UIDGenerator.php b/includes/utils/UIDGenerator.php index 92415877..04c8e194 100644 --- a/includes/utils/UIDGenerator.php +++ b/includes/utils/UIDGenerator.php @@ -20,6 +20,7 @@ * @file * @author Aaron Schulz */ +use Wikimedia\Assert\Assert; /** * Class for getting statistically unique IDs @@ -51,7 +52,7 @@ class UIDGenerator { } // Try to get some ID that uniquely identifies this machine (RFC 4122)... if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) { - wfSuppressWarnings(); + MediaWiki\suppressWarnings(); if ( wfIsWindows() ) { // http://technet.microsoft.com/en-us/library/bb490913.aspx $csv = trim( wfShellExec( 'getmac /NH /FO CSV' ) ); @@ -65,7 +66,7 @@ class UIDGenerator { wfShellExec( '/sbin/ifconfig -a' ), $m ); $nodeId = isset( $m[1] ) ? str_replace( ':', '', $m[1] ) : ''; } - wfRestoreWarnings(); + MediaWiki\restoreWarnings(); if ( !preg_match( '/^[0-9a-f]{12}$/i', $nodeId ) ) { $nodeId = MWCryptRand::generateHex( 12, true ); $nodeId[1] = dechex( hexdec( $nodeId[1] ) | 0x1 ); // set multicast bit @@ -107,9 +108,10 @@ class UIDGenerator { * @throws MWException */ public static function newTimestampedUID88( $base = 10 ) { - if ( !is_integer( $base ) || $base > 36 || $base < 2 ) { - throw new MWException( "Base must an integer be between 2 and 36" ); - } + Assert::parameterType( 'integer', $base, '$base' ); + Assert::parameter( $base <= 36, '$base', 'must be <= 36' ); + Assert::parameter( $base >= 2, '$base', 'must be >= 2' ); + $gen = self::singleton(); $time = $gen->getTimestampAndDelay( 'lockFile88', 1, 1024 ); @@ -152,9 +154,10 @@ class UIDGenerator { * @throws MWException */ public static function newTimestampedUID128( $base = 10 ) { - if ( !is_integer( $base ) || $base > 36 || $base < 2 ) { - throw new MWException( "Base must be an integer between 2 and 36" ); - } + Assert::parameterType( 'integer', $base, '$base' ); + Assert::parameter( $base <= 36, '$base', 'must be <= 36' ); + Assert::parameter( $base >= 2, '$base', 'must be >= 2' ); + $gen = self::singleton(); $time = $gen->getTimestampAndDelay( 'lockFile128', 16384, 1048576 ); @@ -280,7 +283,7 @@ class UIDGenerator { $cache = null; if ( ( $flags & self::QUICK_VOLATILE ) && PHP_SAPI !== 'cli' ) { try { - $cache = ObjectCache::newAccelerator( array() ); + $cache = ObjectCache::newAccelerator(); } catch ( Exception $e ) { // not supported } diff --git a/includes/utils/iterators/IteratorDecorator.php b/includes/utils/iterators/IteratorDecorator.php new file mode 100644 index 00000000..c1b50207 --- /dev/null +++ b/includes/utils/iterators/IteratorDecorator.php @@ -0,0 +1,50 @@ +<?php +/** + * Allows extending classes to decorate an Iterator with + * reduced boilerplate. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +abstract class IteratorDecorator implements Iterator { + protected $iterator; + + public function __construct( Iterator $iterator ) { + $this->iterator = $iterator; + } + + public function current() { + return $this->iterator->current(); + } + + public function key() { + return $this->iterator->key(); + } + + public function next() { + $this->iterator->next(); + } + + public function rewind() { + $this->iterator->rewind(); + } + + public function valid() { + return $this->iterator->valid(); + } +} diff --git a/includes/utils/MWFunction.php b/includes/utils/iterators/NotRecursiveIterator.php index fa7eebe8..52ca61b4 100644 --- a/includes/utils/MWFunction.php +++ b/includes/utils/iterators/NotRecursiveIterator.php @@ -1,6 +1,10 @@ <?php /** - * Helper methods to call functions and instance objects. + * Wraps a non-recursive iterator with methods to be recursive + * without children. + * + * Alternatively wraps a recursive iterator to prevent recursing deeper + * than the wrapped iterator. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,23 +22,14 @@ * http://www.gnu.org/copyleft/gpl.html * * @file + * @ingroup Maintenance */ +class NotRecursiveIterator extends IteratorDecorator implements RecursiveIterator { + public function hasChildren() { + return false; + } -class MWFunction { - - /** - * @param string $class - * @param array $args - * @return object - * @deprecated 1.25 Use ObjectFactory::getObjectFromSpec() instead - */ - public static function newObj( $class, $args = array() ) { - wfDeprecated( __METHOD__, '1.25' ); - - return ObjectFactory::getObjectFromSpec( array( - 'class' => $class, - 'args' => $args, - 'closure_expansion' => false, - ) ); + public function getChildren() { + return null; } } |