Update to MediaWiki 1.26.0

5 files changed, 589 insertions, 2 deletions

diff --git a/includes/debug/logger/monolog/AvroFormatter.php b/includes/debug/logger/monolog/AvroFormatter.php
new file mode 100644
index 00000000..b6adab4a
--- /dev/null
+++ b/includes/debug/logger/monolog/AvroFormatter.php
@@ -0,0 +1,139 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace MediaWiki\Logger\Monolog;
+
+use AvroIODatumWriter;
+use AvroIOBinaryEncoder;
+use AvroIOTypeException;
+use AvroNamedSchemata;
+use AvroSchema;
+use AvroStringIO;
+use AvroValidator;
+use Monolog\Formatter\FormatterInterface;
+
+/**
+ * Log message formatter that uses the apache Avro format.
+ *
+ * @since 1.26
+ * @author Erik Bernhardson <ebernhardson@wikimedia.org>
+ * @copyright © 2015 Erik Bernhardson and Wikimedia Foundation.
+ */
+class AvroFormatter implements FormatterInterface {
+	/**
+	 * @var array Map from schema name to schema definition
+	 */
+	protected $schemas;
+
+	/**
+	 * @var AvroStringIO
+	 */
+	protected $io;
+
+	/**
+	 * @var AvroIOBinaryEncoder
+	 */
+	protected $encoder;
+
+	/**
+	 * @var AvroIODatumWriter
+	 */
+	protected $writer;
+
+	/**
+	 * @var array $schemas Map from Monolog channel to Avro schema.
+	 *  Each schema can be either the JSON string or decoded into PHP
+	 *  arrays.
+	 */
+	public function __construct( array $schemas ) {
+		$this->schemas = $schemas;
+		$this->io = new AvroStringIO( '' );
+		$this->encoder = new AvroIOBinaryEncoder( $this->io );
+		$this->writer = new AvroIODatumWriter();
+	}
+
+	/**
+	 * Formats the record context into a binary string per the
+	 * schema configured for the records channel.
+	 *
+	 * @param array $record
+	 * @return string|null The serialized record, or null if
+	 *  the record is not valid for the selected schema.
+	 */
+	public function format( array $record ) {
+		$this->io->truncate();
+		$schema = $this->getSchema( $record['channel'] );
+		if ( $schema === null ) {
+			trigger_error( "The schema for channel '{$record['channel']}' is not available" );
+			return null;
+		}
+		try {
+			$this->writer->write_data( $schema, $record['context'], $this->encoder );
+		} catch ( AvroIOTypeException $e ) {
+			$errors = AvroValidator::getErrors( $schema, $record['context'] );
+			$json = json_encode( $errors );
+			trigger_error( "Avro failed to serialize record for {$record['channel']} : {$json}" );
+			return null;
+		}
+		return $this->io->string();
+	}
+
+	/**
+	 * Format a set of records into a list of binary strings
+	 * conforming to the configured schema.
+	 *
+	 * @param array $records
+	 * @return string[]
+	 */
+	public function formatBatch( array $records ) {
+		$result = array();
+		foreach ( $records as $record ) {
+			$message = $this->format( $record );
+			if ( $message !== null ) {
+				$result[] = $message;
+			}
+		}
+		return $result;
+	}
+
+	/**
+	 * Get the writer for the named channel
+	 *
+	 * @var string $channel Name of the schema to fetch
+	 * @return AvroSchema|null
+	 */
+	protected function getSchema( $channel ) {
+		if ( !isset( $this->schemas[$channel] ) ) {
+			return null;
+		}
+		if ( !$this->schemas[$channel] instanceof AvroSchema ) {
+			if ( is_string( $this->schemas[$channel] ) ) {
+				$this->schemas[$channel] = AvroSchema::parse( $this->schemas[$channel] );
+			} else {
+				$this->schemas[$channel] = AvroSchema::real_parse(
+					$this->schemas[$channel],
+					null,
+					new AvroNamedSchemata()
+				);
+			}
+		}
+		return $this->schemas[$channel];
+	}
+}
diff --git a/includes/debug/logger/monolog/BufferHandler.php b/includes/debug/logger/monolog/BufferHandler.php
new file mode 100644
index 00000000..3ebd0b1f
--- /dev/null
+++ b/includes/debug/logger/monolog/BufferHandler.php
@@ -0,0 +1,47 @@
+<?php
+/**
+ * Helper class for the index.php entry point.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace MediaWiki\Logger\Monolog;
+
+use DeferredUpdates;
+use Monolog\Handler\BufferHandler as BaseBufferHandler;
+
+/**
+ * Updates the Monolog BufferHandler to use DeferredUpdates rather
+ * than register_shutdown_function. On supported platforms this will
+ * use register_postsend_function or fastcgi_finish_request() to delay
+ * until after the request has shutdown and we are no longer delaying
+ * the web request.
+ */
+class BufferHandler extends BaseBufferHandler {
+	/**
+	 * {@inheritDoc}
+	 */
+	public function handle( array $record ) {
+		if (!$this->initialized) {
+			DeferredUpdates::addCallableUpdate( array( $this, 'close' ) );
+			$this->initialized = true;
+		}
+		return parent::handle( $record );
+	}
+}
+
diff --git a/includes/debug/logger/monolog/KafkaHandler.php b/includes/debug/logger/monolog/KafkaHandler.php
new file mode 100644
index 00000000..59d7764a
--- /dev/null
+++ b/includes/debug/logger/monolog/KafkaHandler.php
@@ -0,0 +1,224 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace MediaWiki\Logger\Monolog;
+
+use Kafka\MetaDataFromKafka;
+use Kafka\Produce;
+use MediaWiki\Logger\LoggerFactory;
+use Monolog\Handler\AbstractProcessingHandler;
+use Monolog\Logger;
+use Psr\Log\LoggerInterface;
+
+/**
+ * Log handler sends log events to a kafka server.
+ *
+ * Constructor options array arguments:
+ * * alias: map from monolog channel to kafka topic name. When no
+ *	  alias exists the topic "monolog_$channel" will be used.
+ * * swallowExceptions: Swallow exceptions that occur while talking to
+ *    kafka. Defaults to false.
+ * * logExceptions: Log exceptions talking to kafka here. Either null,
+ *    the name of a channel to log to, or an object implementing
+ *    FormatterInterface. Defaults to null.
+ *
+ * Requires the nmred/kafka-php library, version >= 1.3.0
+ *
+ * @since 1.26
+ * @author Erik Bernhardson <ebernhardson@wikimedia.org>
+ * @copyright © 2015 Erik Bernhardson and Wikimedia Foundation.
+ */
+class KafkaHandler extends AbstractProcessingHandler {
+	/**
+	 * @var Produce Sends requests to kafka
+	 */
+	protected $produce;
+
+	/**
+	 * @var array Optional handler configuration
+	 */
+	protected $options;
+
+	/**
+	 * @var array Map from topic name to partition this request produces to
+	 */
+	protected $partitions = array();
+
+	/**
+	 * @var array defaults for constructor options
+	 */
+	private static $defaultOptions = array(
+		'alias' => array(), // map from monolog channel to kafka topic
+		'swallowExceptions' => false, // swallow exceptions sending records
+		'logExceptions' => null, // A PSR3 logger to inform about errors
+	);
+
+	/**
+	 * @param Produce $produce Kafka instance to produce through
+	 * @param array $options optional handler configuration
+	 * @param int $level The minimum logging level at which this handler will be triggered
+	 * @param bool $bubble Whether the messages that are handled can bubble up the stack or not
+	 */
+	public function __construct( Produce $produce, array $options, $level = Logger::DEBUG, $bubble = true ) {
+		parent::__construct( $level, $bubble );
+		$this->produce = $produce;
+		$this->options = array_merge( self::$defaultOptions, $options );
+	}
+
+	/**
+	 * Constructs the necessary support objects and returns a KafkaHandler
+	 * instance.
+	 *
+	 * @param string[] $kafkaServers
+	 * @param array $options
+	 * @param int $level The minimum logging level at which this handle will be triggered
+	 * @param bool $bubble Whether the messages that are handled can bubble the stack or not
+	 * @return KafkaHandler
+	 */
+	public static function factory( $kafkaServers, array $options = array(), $level = Logger::DEBUG, $bubble = true ) {
+		$metadata = new MetaDataFromKafka( $kafkaServers );
+		$produce = new Produce( $metadata );
+		if ( isset( $options['logExceptions'] ) && is_string( $options['logExceptions'] ) ) {
+			$options['logExceptions'] = LoggerFactory::getInstance( $options['logExceptions'] );
+		}
+		return new self( $produce, $options, $level, $bubble );
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	protected function write( array $record ) {
+		if ( $record['formatted'] !== null ) {
+			$this->addMessages( $record['channel'], array( $record['formatted'] ) );
+			$this->send();
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function handleBatch( array $batch ) {
+		$channels = array();
+		foreach ( $batch as $record ) {
+			if ( $record['level'] < $this->level ) {
+				continue;
+			}
+			$channels[$record['channel']][] = $this->processRecord( $record );
+		}
+
+		$formatter = $this->getFormatter();
+		foreach ( $channels as $channel => $records ) {
+			$messages = array();
+			foreach ( $records as $idx => $record ) {
+				$message = $formatter->format( $record );
+				if ( $message !== null ) {
+					$messages[] = $message;
+				}
+			}
+			if ( $messages ) {
+				$this->addMessages($channel, $messages);
+			}
+		}
+
+		$this->send();
+	}
+
+	/**
+	 * Send any records in the kafka client internal queue.
+	 */
+	protected function send() {
+		try {
+			$this->produce->send();
+		} catch ( \Kafka\Exception $e ) {
+			$ignore = $this->warning(
+				'Error sending records to kafka: {exception}',
+				array( 'exception' => $e ) );
+			if ( !$ignore ) {
+				throw $e;
+			}
+		}
+	}
+
+	/**
+	 * @param string $topic Name of topic to get partition for
+	 * @return int|null The random partition to produce to for this request,
+	 *  or null if a partition could not be determined.
+	 */
+	protected function getRandomPartition( $topic ) {
+		if ( !array_key_exists( $topic, $this->partitions ) ) {
+			try {
+				$partitions = $this->produce->getAvailablePartitions( $topic );
+			} catch ( \Kafka\Exception $e ) {
+				$ignore = $this->warning(
+					'Error getting metadata for kafka topic {topic}: {exception}',
+					array( 'topic' => $topic, 'exception' => $e ) );
+				if ( $ignore ) {
+					return null;
+				}
+				throw $e;
+			}
+			if ( $partitions ) {
+				$key = array_rand( $partitions );
+				$this->partitions[$topic] = $partitions[$key];
+			} else {
+				$details = $this->produce->getClient()->getTopicDetail( $topic );
+				$ignore = $this->warning(
+					'No partitions available for kafka topic {topic}',
+					array( 'topic' => $topic, 'kafka' => $details )
+				);
+				if ( !$ignore ) {
+					throw new \RuntimeException( "No partitions available for kafka topic $topic" );
+				}
+				$this->partitions[$topic] = null;
+			}
+		}
+		return $this->partitions[$topic];
+	}
+
+	/**
+	 * Adds records for a channel to the Kafka client internal queue.
+	 *
+	 * @param string $channel Name of Monolog channel records belong to
+	 * @param array $records List of records to append
+	 */
+	protected function addMessages( $channel, array $records ) {
+		if ( isset( $this->options['alias'][$channel] ) ) {
+			$topic = $this->options['alias'][$channel];
+		} else {
+			$topic = "monolog_$channel";
+		}
+		$partition = $this->getRandomPartition( $topic );
+		if ( $partition !== null ) {
+			$this->produce->setMessages( $topic, $partition, $records );
+		}
+	}
+
+	/**
+	 * @param string $message PSR3 compatible message string
+	 * @param array $context PSR3 compatible log context
+	 * @return bool true if caller should ignore warning
+	 */
+	protected function warning( $message, array $context = array() ) {
+		if ( $this->options['logExceptions'] instanceof LoggerInterface ) {
+			$this->options['logExceptions']->warning( $message, $context );
+		}
+		return $this->options['swallowExceptions'];
+	}
+}
diff --git a/includes/debug/logger/monolog/LegacyFormatter.php b/includes/debug/logger/monolog/LegacyFormatter.php
index 9ec15cb8..42e7caba 100644
--- a/includes/debug/logger/monolog/LegacyFormatter.php
+++ b/includes/debug/logger/monolog/LegacyFormatter.php
@@ -26,12 +26,12 @@ use Monolog\Formatter\NormalizerFormatter;
 /**
  * Log message formatter that mimics the legacy log message formatting of
  * `wfDebug`, `wfDebugLog`, `wfLogDBError` and `wfErrorLog` global functions by
- * delegating the formatting to \MediaWiki\Logger\LegacyLogger.
+ * delegating the formatting to \\MediaWiki\\Logger\\LegacyLogger.
  *
  * @since 1.25
  * @author Bryan Davis <bd808@wikimedia.org>
  * @copyright © 2013 Bryan Davis and Wikimedia Foundation.
- * @see \MediaWiki\Logger\LegacyLogger
+ * @see \\MediaWiki\\Logger\\LegacyLogger
  */
 class LegacyFormatter extends NormalizerFormatter {
 
diff --git a/includes/debug/logger/monolog/LineFormatter.php b/includes/debug/logger/monolog/LineFormatter.php
new file mode 100644
index 00000000..2ba7a53c
--- /dev/null
+++ b/includes/debug/logger/monolog/LineFormatter.php
@@ -0,0 +1,177 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace MediaWiki\Logger\Monolog;
+
+use Exception;
+use Monolog\Formatter\LineFormatter as MonologLineFormatter;
+use MWExceptionHandler;
+
+/**
+ * Formats incoming records into a one-line string.
+ *
+ * An 'exeception' in the log record's context will be treated specially.
+ * It will be output for an '%exception%' placeholder in the format and
+ * excluded from '%context%' output if the '%exception%' placeholder is
+ * present.
+ *
+ * Exceptions that are logged with this formatter will optional have their
+ * stack traces appended. If that is done, MWExceptionHandler::redactedTrace()
+ * will be used to redact the trace information.
+ *
+ * @since 1.26
+ * @author Bryan Davis <bd808@wikimedia.org>
+ * @copyright © 2015 Bryan Davis and Wikimedia Foundation.
+ */
+class LineFormatter extends MonologLineFormatter {
+
+	/**
+	 * @param string $format The format of the message
+	 * @param string $dateFormat The format of the timestamp: one supported by DateTime::format
+	 * @param bool $allowInlineLineBreaks Whether to allow inline line breaks in log entries
+	 * @param bool $ignoreEmptyContextAndExtra
+	 * @param bool $includeStacktraces
+	 */
+	public function __construct(
+		$format = null, $dateFormat = null, $allowInlineLineBreaks = false,
+		$ignoreEmptyContextAndExtra = false, $includeStacktraces = false
+	) {
+		parent::__construct(
+			$format, $dateFormat, $allowInlineLineBreaks,
+			$ignoreEmptyContextAndExtra
+		);
+		$this->includeStacktraces( $includeStacktraces );
+	}
+
+
+	/**
+	 * {@inheritdoc}
+	 */
+	public function format( array $record ) {
+		// Drop the 'private' flag from the context
+		unset( $record['context']['private'] );
+
+		// Handle exceptions specially: pretty format and remove from context
+		// Will be output for a '%exception%' placeholder in format
+		$prettyException = '';
+		if ( isset( $record['context']['exception'] ) &&
+			strpos( $this->format, '%exception%' ) !== false
+		) {
+			$e = $record['context']['exception'];
+			unset( $record['context']['exception'] );
+
+			if ( $e instanceof Exception ) {
+				$prettyException = $this->normalizeException( $e );
+			} elseif ( is_array( $e ) ) {
+				$prettyException = $this->normalizeExceptionArray( $e );
+			} else {
+				$prettyException = $this->stringify( $e );
+			}
+		}
+
+		$output = parent::format( $record );
+
+		if ( strpos( $output, '%exception%' ) !== false ) {
+			$output = str_replace( '%exception%', $prettyException, $output );
+		}
+		return $output;
+	}
+
+
+	/**
+	 * Convert an Exception to a string.
+	 *
+	 * @param Exception $e
+	 * @return string
+	 */
+	protected function normalizeException( Exception $e ) {
+		return $this->normalizeExceptionArray( $this->exceptionAsArray( $e ) );
+	}
+
+
+	/**
+	 * Convert an exception to an array of structured data.
+	 *
+	 * @param Exception $e
+	 * @return array
+	 */
+	protected function exceptionAsArray( Exception $e ) {
+		$out = array(
+			'class' => get_class( $e ),
+			'message' => $e->getMessage(),
+			'code' => $e->getCode(),
+			'file' => $e->getFile(),
+			'line' => $e->getLine(),
+			'trace' => MWExceptionHandler::redactTrace( $e->getTrace() ),
+		);
+
+		$prev = $e->getPrevious();
+		if ( $prev ) {
+			$out['previous'] = $this->exceptionAsArray( $prev );
+		}
+
+		return $out;
+	}
+
+
+	/**
+	 * Convert an array of Exception data to a string.
+	 *
+	 * @param array $e
+	 * @return string
+	 */
+	protected function normalizeExceptionArray( array $e ) {
+		$defaults = array(
+			'class' => 'Unknown',
+			'file' => 'unknown',
+			'line' => null,
+			'message' => 'unknown',
+			'trace' => array(),
+		);
+		$e = array_merge( $defaults, $e );
+
+		$str = "\n[Exception {$e['class']}] (" .
+			"{$e['file']}:{$e['line']}) {$e['message']}";
+
+		if ( $this->includeStacktraces && $e['trace'] ) {
+			$str .= "\n" .
+				MWExceptionHandler::prettyPrintTrace( $e['trace'], '  ' );
+		}
+
+		if ( isset( $e['previous'] ) ) {
+			$prev = $e['previous'];
+			while ( $prev ) {
+				$prev = array_merge( $defaults, $prev );
+				$str .= "\nCaused by: [Exception {$prev['class']}] (" .
+					"{$prev['file']}:{$prev['line']}) {$prev['message']}";
+
+				if ( $this->includeStacktraces && $prev['trace'] ) {
+					$str .= "\n" .
+						MWExceptionHandler::prettyPrintTrace(
+							$prev['trace'], '  '
+						);
+				}
+
+				$prev = isset( $prev['previous'] ) ? $prev['previous'] : null;
+			}
+		}
+		return $str;
+	}
+}