diff options
author | Pierre Schmitz <pierre@archlinux.de> | 2014-12-27 15:41:37 +0100 |
---|---|---|
committer | Pierre Schmitz <pierre@archlinux.de> | 2014-12-31 11:43:28 +0100 |
commit | c1f9b1f7b1b77776192048005dcc66dcf3df2bfb (patch) | |
tree | 2b38796e738dd74cb42ecd9bfd151803108386bc /includes/libs | |
parent | b88ab0086858470dd1f644e64cb4e4f62bb2be9b (diff) |
Update to MediaWiki 1.24.1
Diffstat (limited to 'includes/libs')
-rw-r--r-- | includes/libs/CSSJanus.php | 246 | ||||
-rw-r--r-- | includes/libs/CSSMin.php | 300 | ||||
-rw-r--r-- | includes/libs/GenericArrayObject.php | 5 | ||||
-rw-r--r-- | includes/libs/HashRing.php | 239 | ||||
-rw-r--r-- | includes/libs/HttpStatus.php | 8 | ||||
-rw-r--r-- | includes/libs/IEContentAnalyzer.php | 7 | ||||
-rw-r--r-- | includes/libs/IPSet.php | 277 | ||||
-rw-r--r-- | includes/libs/JavaScriptMinifier.php | 1 | ||||
-rw-r--r-- | includes/libs/MWMessagePack.php | 189 | ||||
-rw-r--r-- | includes/libs/MappedIterator.php | 117 | ||||
-rw-r--r-- | includes/libs/MultiHttpClient.php | 389 | ||||
-rw-r--r-- | includes/libs/ProcessCacheLRU.php | 148 | ||||
-rw-r--r-- | includes/libs/RunningStat.php | 176 | ||||
-rw-r--r-- | includes/libs/ScopedCallback.php | 73 | ||||
-rw-r--r-- | includes/libs/ScopedPHPTimeout.php | 84 | ||||
-rw-r--r-- | includes/libs/XmlTypeCheck.php | 264 | ||||
-rw-r--r-- | includes/libs/jsminplus.php | 1 | ||||
-rw-r--r-- | includes/libs/lessc.inc.php | 88 | ||||
-rw-r--r-- | includes/libs/virtualrest/SwiftVirtualRESTService.php | 175 | ||||
-rw-r--r-- | includes/libs/virtualrest/VirtualRESTService.php | 107 | ||||
-rw-r--r-- | includes/libs/virtualrest/VirtualRESTServiceClient.php | 289 |
21 files changed, 2958 insertions, 225 deletions
diff --git a/includes/libs/CSSJanus.php b/includes/libs/CSSJanus.php index 5a52fc7c..07a83a54 100644 --- a/includes/libs/CSSJanus.php +++ b/includes/libs/CSSJanus.php @@ -60,7 +60,7 @@ class CSSJanus { 'lookahead_not_letter' => '(?![a-zA-Z])', 'lookbehind_not_letter' => '(?<![a-zA-Z])', 'chars_within_selector' => '[^\}]*?', - 'noflip_annotation' => '\/\*\s*@noflip\s*\*\/', + 'noflip_annotation' => '\/\*\!?\s*@noflip\s*\*\/', 'noflip_single' => null, 'noflip_class' => null, 'comment' => '/\/\*[^*]*\*+([^\/*][^*]*\*+)*\//', @@ -88,11 +88,12 @@ class CSSJanus { * Build patterns we can't define above because they depend on other patterns. */ private static function buildPatterns() { - if ( !is_null( self::$patterns['escape'] ) ) { + if (!is_null(self::$patterns['escape'])) { // Patterns have already been built return; } + // @codingStandardsIgnoreStart Generic.Files.LineLength.TooLong $patterns =& self::$patterns; $patterns['escape'] = "(?:{$patterns['unicode']}|\\[^\r\n\f0-9a-f])"; $patterns['nmstart'] = "(?:[_a-z]|{$patterns['nonAscii']}|{$patterns['escape']})"; @@ -102,7 +103,7 @@ class CSSJanus { $patterns['possibly_negative_quantity'] = "((?:-?{$patterns['quantity']})|(?:inherit|auto))"; $patterns['color'] = "(#?{$patterns['nmchar']}+|(?:rgba?|hsla?)\([ \d.,%-]+\))"; $patterns['url_chars'] = "(?:{$patterns['url_special_chars']}|{$patterns['nonAscii']}|{$patterns['escape']})*"; - $patterns['lookahead_not_open_brace'] = "(?!({$patterns['nmchar']}|\r?\n|\s|#|\:|\.|\,|\+|>|\(|\))*?{)"; + $patterns['lookahead_not_open_brace'] = "(?!({$patterns['nmchar']}|\r?\n|\s|#|\:|\.|\,|\+|>|\(|\)|\[|\]|=|\*=|~=|\^=|'[^']*'])*?{)"; $patterns['lookahead_not_closing_paren'] = "(?!{$patterns['url_chars']}?{$patterns['valid_after_uri_chars']}\))"; $patterns['lookahead_for_closing_paren'] = "(?={$patterns['url_chars']}?{$patterns['valid_after_uri_chars']}\))"; $patterns['noflip_single'] = "/({$patterns['noflip_annotation']}{$patterns['lookahead_not_open_brace']}[^;}]+;?)/i"; @@ -117,16 +118,17 @@ class CSSJanus { $patterns['rtl_in_url'] = "/{$patterns['lookbehind_not_letter']}(rtl){$patterns['lookahead_for_closing_paren']}/i"; $patterns['cursor_east'] = "/{$patterns['lookbehind_not_letter']}([ns]?)e-resize/"; $patterns['cursor_west'] = "/{$patterns['lookbehind_not_letter']}([ns]?)w-resize/"; - $patterns['four_notation_quantity'] = "/(:\s*){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s*[;}])/i"; - $patterns['four_notation_color'] = "/(-color\s*:\s*){$patterns['color']}(\s+){$patterns['color']}(\s+){$patterns['color']}(\s+){$patterns['color']}(\s*[;}])/i"; - $patterns['border_radius'] = "/(border-radius\s*:\s*){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s*[;}])/i"; + $patterns['four_notation_quantity_props'] = "((?:margin|padding|border-width)\s*:\s*)"; + $patterns['four_notation_quantity'] = "/{$patterns['four_notation_quantity_props']}{$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s+){$patterns['possibly_negative_quantity']}(\s*[;}])/i"; + $patterns['four_notation_color'] = "/((?:-color|border-style)\s*:\s*){$patterns['color']}(\s+){$patterns['color']}(\s+){$patterns['color']}(\s+){$patterns['color']}(\s*[;}])/i"; + $patterns['border_radius'] = "/(border-radius\s*:\s*)([^;}]*)/"; $patterns['box_shadow'] = "/(box-shadow\s*:\s*(?:inset\s*)?){$patterns['possibly_negative_quantity']}/i"; $patterns['text_shadow1'] = "/(text-shadow\s*:\s*){$patterns['color']}(\s*){$patterns['possibly_negative_quantity']}/i"; $patterns['text_shadow2'] = "/(text-shadow\s*:\s*){$patterns['possibly_negative_quantity']}/i"; - // The two regexes below are parenthesized differently then in the original implementation to make the - // callback's job more straightforward - $patterns['bg_horizontal_percentage'] = "/(background(?:-position)?\s*:\s*[^%]*?)(-?{$patterns['num']})(%\s*(?:{$patterns['quantity']}|{$patterns['ident']}))/"; - $patterns['bg_horizontal_percentage_x'] = "/(background-position-x\s*:\s*)(-?{$patterns['num']})(%)/"; + $patterns['bg_horizontal_percentage'] = "/(background(?:-position)?\s*:\s*(?:[^:;}\s]+\s+)*?)({$patterns['quantity']})/i"; + $patterns['bg_horizontal_percentage_x'] = "/(background-position-x\s*:\s*)(-?{$patterns['num']}%)/i"; + // @codingStandardsIgnoreEnd + } /** @@ -136,46 +138,46 @@ class CSSJanus { * @param $swapLeftRightInURL Boolean: If true, swap 'left' and 'right' in URLs * @return string Transformed stylesheet */ - public static function transform( $css, $swapLtrRtlInURL = false, $swapLeftRightInURL = false ) { + public static function transform($css, $swapLtrRtlInURL = false, $swapLeftRightInURL = false) { // We wrap tokens in ` , not ~ like the original implementation does. // This was done because ` is not a legal character in CSS and can only // occur in URLs, where we escape it to %60 before inserting our tokens. - $css = str_replace( '`', '%60', $css ); + $css = str_replace('`', '%60', $css); self::buildPatterns(); // Tokenize single line rules with /* @noflip */ - $noFlipSingle = new CSSJanus_Tokenizer( self::$patterns['noflip_single'], '`NOFLIP_SINGLE`' ); - $css = $noFlipSingle->tokenize( $css ); + $noFlipSingle = new CSSJanusTokenizer(self::$patterns['noflip_single'], '`NOFLIP_SINGLE`'); + $css = $noFlipSingle->tokenize($css); // Tokenize class rules with /* @noflip */ - $noFlipClass = new CSSJanus_Tokenizer( self::$patterns['noflip_class'], '`NOFLIP_CLASS`' ); - $css = $noFlipClass->tokenize( $css ); + $noFlipClass = new CSSJanusTokenizer(self::$patterns['noflip_class'], '`NOFLIP_CLASS`'); + $css = $noFlipClass->tokenize($css); // Tokenize comments - $comments = new CSSJanus_Tokenizer( self::$patterns['comment'], '`C`' ); - $css = $comments->tokenize( $css ); + $comments = new CSSJanusTokenizer(self::$patterns['comment'], '`C`'); + $css = $comments->tokenize($css); // LTR->RTL fixes start here - $css = self::fixDirection( $css ); - if ( $swapLtrRtlInURL ) { - $css = self::fixLtrRtlInURL( $css ); + $css = self::fixDirection($css); + if ($swapLtrRtlInURL) { + $css = self::fixLtrRtlInURL($css); } - if ( $swapLeftRightInURL ) { - $css = self::fixLeftRightInURL( $css ); + if ($swapLeftRightInURL) { + $css = self::fixLeftRightInURL($css); } - $css = self::fixLeftAndRight( $css ); - $css = self::fixCursorProperties( $css ); - $css = self::fixFourPartNotation( $css ); - $css = self::fixBorderRadius( $css ); - $css = self::fixBackgroundPosition( $css ); - $css = self::fixShadows( $css ); + $css = self::fixLeftAndRight($css); + $css = self::fixCursorProperties($css); + $css = self::fixFourPartNotation($css); + $css = self::fixBorderRadius($css); + $css = self::fixBackgroundPosition($css); + $css = self::fixShadows($css); // Detokenize stuff we tokenized before - $css = $comments->detokenize( $css ); - $css = $noFlipClass->detokenize( $css ); - $css = $noFlipSingle->detokenize( $css ); + $css = $comments->detokenize($css); + $css = $noFlipClass->detokenize($css); + $css = $noFlipSingle->detokenize($css); return $css; } @@ -187,16 +189,19 @@ class CSSJanus { * and misses "body\n{\ndirection:ltr;\n}". This function does not have * these problems. * - * See http://code.google.com/p/cssjanus/issues/detail?id=15 and - * TODO: URL + * See https://code.google.com/p/cssjanus/issues/detail?id=15 + * * @param $css string * @return string */ - private static function fixDirection( $css ) { - $css = preg_replace( self::$patterns['direction_ltr'], - '$1' . self::$patterns['tmpToken'], $css ); - $css = preg_replace( self::$patterns['direction_rtl'], '$1ltr', $css ); - $css = str_replace( self::$patterns['tmpToken'], 'rtl', $css ); + private static function fixDirection($css) { + $css = preg_replace( + self::$patterns['direction_ltr'], + '$1' . self::$patterns['tmpToken'], + $css + ); + $css = preg_replace(self::$patterns['direction_rtl'], '$1ltr', $css); + $css = str_replace(self::$patterns['tmpToken'], 'rtl', $css); return $css; } @@ -206,10 +211,10 @@ class CSSJanus { * @param $css string * @return string */ - private static function fixLtrRtlInURL( $css ) { - $css = preg_replace( self::$patterns['ltr_in_url'], self::$patterns['tmpToken'], $css ); - $css = preg_replace( self::$patterns['rtl_in_url'], 'ltr', $css ); - $css = str_replace( self::$patterns['tmpToken'], 'rtl', $css ); + private static function fixLtrRtlInURL($css) { + $css = preg_replace(self::$patterns['ltr_in_url'], self::$patterns['tmpToken'], $css); + $css = preg_replace(self::$patterns['rtl_in_url'], 'ltr', $css); + $css = str_replace(self::$patterns['tmpToken'], 'rtl', $css); return $css; } @@ -219,10 +224,10 @@ class CSSJanus { * @param $css string * @return string */ - private static function fixLeftRightInURL( $css ) { - $css = preg_replace( self::$patterns['left_in_url'], self::$patterns['tmpToken'], $css ); - $css = preg_replace( self::$patterns['right_in_url'], 'left', $css ); - $css = str_replace( self::$patterns['tmpToken'], 'right', $css ); + private static function fixLeftRightInURL($css) { + $css = preg_replace(self::$patterns['left_in_url'], self::$patterns['tmpToken'], $css); + $css = preg_replace(self::$patterns['right_in_url'], 'left', $css); + $css = str_replace(self::$patterns['tmpToken'], 'right', $css); return $css; } @@ -232,10 +237,10 @@ class CSSJanus { * @param $css string * @return string */ - private static function fixLeftAndRight( $css ) { - $css = preg_replace( self::$patterns['left'], self::$patterns['tmpToken'], $css ); - $css = preg_replace( self::$patterns['right'], 'left', $css ); - $css = str_replace( self::$patterns['tmpToken'], 'right', $css ); + private static function fixLeftAndRight($css) { + $css = preg_replace(self::$patterns['left'], self::$patterns['tmpToken'], $css); + $css = preg_replace(self::$patterns['right'], 'left', $css); + $css = str_replace(self::$patterns['tmpToken'], 'right', $css); return $css; } @@ -245,11 +250,14 @@ class CSSJanus { * @param $css string * @return string */ - private static function fixCursorProperties( $css ) { - $css = preg_replace( self::$patterns['cursor_east'], - '$1' . self::$patterns['tmpToken'], $css ); - $css = preg_replace( self::$patterns['cursor_west'], '$1e-resize', $css ); - $css = str_replace( self::$patterns['tmpToken'], 'w-resize', $css ); + private static function fixCursorProperties($css) { + $css = preg_replace( + self::$patterns['cursor_east'], + '$1' . self::$patterns['tmpToken'], + $css + ); + $css = preg_replace(self::$patterns['cursor_west'], '$1e-resize', $css); + $css = str_replace(self::$patterns['tmpToken'], 'w-resize', $css); return $css; } @@ -262,28 +270,38 @@ class CSSJanus { * the bug where whitespace is not preserved when flipping four-part rules * and four-part color rules with multiple whitespace characters between * colors are not recognized. - * See http://code.google.com/p/cssjanus/issues/detail?id=16 + * See https://code.google.com/p/cssjanus/issues/detail?id=16 * @param $css string * @return string */ - private static function fixFourPartNotation( $css ) { - $css = preg_replace( self::$patterns['four_notation_quantity'], '$1$2$3$8$5$6$7$4$9', $css ); - $css = preg_replace( self::$patterns['four_notation_color'], '$1$2$3$8$5$6$7$4$9', $css ); + private static function fixFourPartNotation($css) { + $css = preg_replace(self::$patterns['four_notation_quantity'], '$1$2$3$8$5$6$7$4$9', $css); + $css = preg_replace(self::$patterns['four_notation_color'], '$1$2$3$8$5$6$7$4$9', $css); return $css; } /** - * Swaps appropriate corners in four-part border-radius rules. - * Needs to undo the effect of fixFourPartNotation() on those rules, too. + * Swaps appropriate corners in border-radius values. * * @param $css string * @return string */ - private static function fixBorderRadius( $css ) { - // Undo four_notation_quantity - $css = preg_replace( self::$patterns['border_radius'], '$1$2$3$8$5$6$7$4$9', $css ); - // Do the real thing - $css = preg_replace( self::$patterns['border_radius'], '$1$4$3$2$5$8$7$6$9', $css ); + private static function fixBorderRadius($css) { + $css = preg_replace_callback(self::$patterns['border_radius'], function ($matches) { + $pre = $matches[1]; + $values = $matches[2]; + $numValues = count(preg_split('/\s+/', trim($values))); + switch ($numValues) { + case 4: + $values = preg_replace('/^(\S+)(\s*)(\S+)(\s*)(\S+)(\s*)(\S+)/', '$3$2$1$4$7$6$5', $values); + break; + case 3: + case 2: + $values = preg_replace('/^(\S+)(\s*)(\S+)/', '$3$2$1', $values); + break; + } + return $pre . $values; + }, $css); return $css; } @@ -294,31 +312,31 @@ class CSSJanus { * @param $css string * @return string */ - private static function fixShadows( $css ) { + private static function fixShadows($css) { // Flips the sign of a CSS value, possibly with a unit. // (We can't just negate the value with unary minus due to the units.) - $flipSign = function ( $cssValue ) { + $flipSign = function ($cssValue) { // Don't mangle zeroes - if ( intval( $cssValue ) === 0 ) { + if (floatval($cssValue) === 0.0) { return $cssValue; - } elseif ( $cssValue[0] === '-' ) { - return substr( $cssValue, 1 ); + } elseif ($cssValue[0] === '-') { + return substr($cssValue, 1); } else { return "-" . $cssValue; } }; - $css = preg_replace_callback( self::$patterns['box_shadow'], function ( $matches ) use ( $flipSign ) { - return $matches[1] . $flipSign( $matches[2] ); - }, $css ); + $css = preg_replace_callback(self::$patterns['box_shadow'], function ($matches) use ($flipSign) { + return $matches[1] . $flipSign($matches[2]); + }, $css); - $css = preg_replace_callback( self::$patterns['text_shadow1'], function ( $matches ) use ( $flipSign ) { - return $matches[1] . $matches[2] . $matches[3] . $flipSign( $matches[4] ); - }, $css ); + $css = preg_replace_callback(self::$patterns['text_shadow1'], function ($matches) use ($flipSign) { + return $matches[1] . $matches[2] . $matches[3] . $flipSign($matches[4]); + }, $css); - $css = preg_replace_callback( self::$patterns['text_shadow2'], function ( $matches ) use ( $flipSign ) { - return $matches[1] . $flipSign( $matches[2] ); - }, $css ); + $css = preg_replace_callback(self::$patterns['text_shadow2'], function ($matches) use ($flipSign) { + return $matches[1] . $flipSign($matches[2]); + }, $css); return $css; } @@ -328,16 +346,22 @@ class CSSJanus { * @param $css string * @return string */ - private static function fixBackgroundPosition( $css ) { - $replaced = preg_replace_callback( self::$patterns['bg_horizontal_percentage'], - array( 'self', 'calculateNewBackgroundPosition' ), $css ); - if ( $replaced !== null ) { - // Check for null; sometimes preg_replace_callback() returns null here for some weird reason + private static function fixBackgroundPosition($css) { + $replaced = preg_replace_callback( + self::$patterns['bg_horizontal_percentage'], + array('self', 'calculateNewBackgroundPosition'), + $css + ); + if ($replaced !== null) { + // preg_replace_callback() sometimes returns null $css = $replaced; } - $replaced = preg_replace_callback( self::$patterns['bg_horizontal_percentage_x'], - array( 'self', 'calculateNewBackgroundPosition' ), $css ); - if ( $replaced !== null ) { + $replaced = preg_replace_callback( + self::$patterns['bg_horizontal_percentage_x'], + array('self', 'calculateNewBackgroundPosition'), + $css + ); + if ($replaced !== null) { $css = $replaced; } @@ -345,12 +369,22 @@ class CSSJanus { } /** - * Callback for calculateNewBackgroundPosition() + * Callback for fixBackgroundPosition() * @param $matches array * @return string */ - private static function calculateNewBackgroundPosition( $matches ) { - return $matches[1] . ( 100 - $matches[2] ) . $matches[3]; + private static function calculateNewBackgroundPosition($matches) { + $value = $matches[2]; + if (substr($value, -1) === '%') { + $idx = strpos($value, '.'); + if ($idx !== false) { + $len = strlen($value) - $idx - 2; + $value = number_format(100 - $value, $len) . '%'; + } else { + $value = (100 - $value) . '%'; + } + } + return $matches[1] . $value; } } @@ -359,8 +393,9 @@ class CSSJanus { * to protect from being janused. * @author Roan Kattouw */ -class CSSJanus_Tokenizer { - private $regex, $token; +class CSSJanusTokenizer { + private $regex; + private $token; private $originals; /** @@ -368,7 +403,7 @@ class CSSJanus_Tokenizer { * @param string $regex Regular expression whose matches to replace by a token. * @param string $token Token */ - public function __construct( $regex, $token ) { + public function __construct($regex, $token) { $this->regex = $regex; $this->token = $token; $this->originals = array(); @@ -380,15 +415,15 @@ class CSSJanus_Tokenizer { * @param string $str to tokenize * @return string Tokenized string */ - public function tokenize( $str ) { - return preg_replace_callback( $this->regex, array( $this, 'tokenizeCallback' ), $str ); + public function tokenize($str) { + return preg_replace_callback($this->regex, array($this, 'tokenizeCallback'), $str); } /** * @param $matches array * @return string */ - private function tokenizeCallback( $matches ) { + private function tokenizeCallback($matches) { $this->originals[] = $matches[0]; return $this->token; } @@ -399,21 +434,24 @@ class CSSJanus_Tokenizer { * @param string $str previously run through tokenize() * @return string Original string */ - public function detokenize( $str ) { + public function detokenize($str) { // PHP has no function to replace only the first occurrence or to // replace occurrences of the same string with different values, // so we use preg_replace_callback() even though we don't really need a regex - return preg_replace_callback( '/' . preg_quote( $this->token, '/' ) . '/', - array( $this, 'detokenizeCallback' ), $str ); + return preg_replace_callback( + '/' . preg_quote($this->token, '/') . '/', + array($this, 'detokenizeCallback'), + $str + ); } /** * @param $matches * @return mixed */ - private function detokenizeCallback( $matches ) { - $retval = current( $this->originals ); - next( $this->originals ); + private function detokenizeCallback($matches) { + $retval = current($this->originals); + next($this->originals); return $retval; } diff --git a/includes/libs/CSSMin.php b/includes/libs/CSSMin.php index 4f142fc7..c69e79f5 100644 --- a/includes/libs/CSSMin.php +++ b/includes/libs/CSSMin.php @@ -38,11 +38,13 @@ class CSSMin { * which when base64 encoded will result in a 1/3 increase in size. */ const EMBED_SIZE_LIMIT = 24576; - const URL_REGEX = 'url\(\s*[\'"]?(?P<file>[^\?\)\'"]*)(?P<query>\??[^\)\'"]*)[\'"]?\s*\)'; + const URL_REGEX = 'url\(\s*[\'"]?(?P<file>[^\?\)\'"]*?)(?P<query>\?[^\)\'"]*?|)[\'"]?\s*\)'; + const EMBED_REGEX = '\/\*\s*\@embed\s*\*\/'; + const COMMENT_REGEX = '\/\*.*?\*\/'; /* Protected Static Members */ - /** @var array List of common image files extensions and mime-types */ + /** @var array List of common image files extensions and MIME-types */ protected static $mimeTypes = array( 'gif' => 'image/gif', 'jpe' => 'image/jpeg', @@ -52,6 +54,7 @@ class CSSMin { 'tif' => 'image/tiff', 'tiff' => 'image/tiff', 'xbm' => 'image/x-xbitmap', + 'svg' => 'image/svg+xml', ); /* Static Methods */ @@ -59,23 +62,38 @@ class CSSMin { /** * Gets a list of local file paths which are referenced in a CSS style sheet * + * This function will always return an empty array if the second parameter is not given or null + * for backwards-compatibility. + * * @param string $source CSS data to remap * @param string $path File path where the source was read from (optional) * @return array List of local file references */ public static function getLocalFileReferences( $source, $path = null ) { + if ( $path === null ) { + return array(); + } + + $path = rtrim( $path, '/' ) . '/'; $files = array(); + $rFlags = PREG_OFFSET_CAPTURE | PREG_SET_ORDER; if ( preg_match_all( '/' . self::URL_REGEX . '/', $source, $matches, $rFlags ) ) { foreach ( $matches as $match ) { - $file = ( isset( $path ) - ? rtrim( $path, '/' ) . '/' - : '' ) . "{$match['file'][0]}"; + $url = $match['file'][0]; - // Only proceed if we can access the file - if ( !is_null( $path ) && file_exists( $file ) ) { - $files[] = $file; + // Skip fully-qualified and protocol-relative URLs and data URIs + if ( substr( $url, 0, 2 ) === '//' || parse_url( $url, PHP_URL_SCHEME ) ) { + break; } + + $file = $path . $url; + // Skip non-existent files + if ( file_exists( $file ) ) { + break; + } + + $files[] = $file; } } return $files; @@ -95,7 +113,9 @@ class CSSMin { * instead. If $sizeLimit is false, no limit is enforced. * @return string|bool: Image contents encoded as a data URI or false. */ - public static function encodeImageAsDataURI( $file, $type = null, $sizeLimit = self::EMBED_SIZE_LIMIT ) { + public static function encodeImageAsDataURI( $file, $type = null, + $sizeLimit = self::EMBED_SIZE_LIMIT + ) { if ( $sizeLimit !== false && filesize( $file ) >= $sizeLimit ) { return false; } @@ -115,124 +135,214 @@ class CSSMin { */ public static function getMimeType( $file ) { $realpath = realpath( $file ); - // Try a couple of different ways to get the mime-type of a file, in order of - // preference if ( $realpath && function_exists( 'finfo_file' ) && function_exists( 'finfo_open' ) && defined( 'FILEINFO_MIME_TYPE' ) ) { - // As of PHP 5.3, this is how you get the mime-type of a file; it uses the Fileinfo - // PECL extension return finfo_file( finfo_open( FILEINFO_MIME_TYPE ), $realpath ); - } elseif ( function_exists( 'mime_content_type' ) ) { - // Before this was deprecated in PHP 5.3, this was how you got the mime-type of a file - return mime_content_type( $file ); - } else { - // Worst-case scenario has happened, use the file extension to infer the mime-type - $ext = strtolower( pathinfo( $file, PATHINFO_EXTENSION ) ); - if ( isset( self::$mimeTypes[$ext] ) ) { - return self::$mimeTypes[$ext]; - } } + + // Infer the MIME-type from the file extension + $ext = strtolower( pathinfo( $file, PATHINFO_EXTENSION ) ); + if ( isset( self::$mimeTypes[$ext] ) ) { + return self::$mimeTypes[$ext]; + } + return false; } /** - * Remaps CSS URL paths and automatically embeds data URIs for URL rules - * preceded by an /* @embed * / comment + * Build a CSS 'url()' value for the given URL, quoting parentheses (and other funny characters) + * and escaping quotes as necessary. + * + * See http://www.w3.org/TR/css-syntax-3/#consume-a-url-token + * + * @param string $url URL to process + * @return string 'url()' value, usually just `"url($url)"`, quoted/escaped if necessary + */ + public static function buildUrlValue( $url ) { + // The list below has been crafted to match URLs such as: + // scheme://user@domain:port/~user/fi%20le.png?query=yes&really=y+s + // data:image/png;base64,R0lGODlh/+== + if ( preg_match( '!^[\w\d:@/~.%+;,?&=-]+$!', $url ) ) { + return "url($url)"; + } else { + return 'url("' . strtr( $url, array( '\\' => '\\\\', '"' => '\\"' ) ) . '")'; + } + } + + /** + * Remaps CSS URL paths and automatically embeds data URIs for CSS rules + * or url() values preceded by an / * @embed * / comment. * * @param string $source CSS data to remap * @param string $local File path where the source was read from * @param string $remote URL path to the file - * @param bool $embedData If false, never do any data URI embedding, even if / * @embed * / is found + * @param bool $embedData If false, never do any data URI embedding, + * even if / * @embed * / is found. * @return string Remapped CSS data */ public static function remap( $source, $local, $remote, $embedData = true ) { - $pattern = '/((?P<embed>\s*\/\*\s*\@embed\s*\*\/)(?P<pre>[^\;\}]*))?' . - self::URL_REGEX . '(?P<post>[^;]*)[\;]?/'; - $offset = 0; - while ( preg_match( $pattern, $source, $match, PREG_OFFSET_CAPTURE, $offset ) ) { - // Skip fully-qualified URLs and data URIs - $urlScheme = parse_url( $match['file'][0], PHP_URL_SCHEME ); - if ( $urlScheme ) { - // Move the offset to the end of the match, leaving it alone - $offset = $match[0][1] + strlen( $match[0][0] ); - continue; - } - // URLs with absolute paths like /w/index.php need to be expanded - // to absolute URLs but otherwise left alone - if ( $match['file'][0] !== '' && $match['file'][0][0] === '/' ) { - // Replace the file path with an expanded (possibly protocol-relative) URL - // ...but only if wfExpandUrl() is even available. - // This will not be the case if we're running outside of MW - $lengthIncrease = 0; - if ( function_exists( 'wfExpandUrl' ) ) { - $expanded = wfExpandUrl( $match['file'][0], PROTO_RELATIVE ); - $origLength = strlen( $match['file'][0] ); - $lengthIncrease = strlen( $expanded ) - $origLength; - $source = substr_replace( $source, $expanded, - $match['file'][1], $origLength + // High-level overview: + // * For each CSS rule in $source that includes at least one url() value: + // * Check for an @embed comment at the start indicating that all URIs should be embedded + // * For each url() value: + // * Check for an @embed comment directly preceding the value + // * If either @embed comment exists: + // * Embedding the URL as data: URI, if it's possible / allowed + // * Otherwise remap the URL to work in generated stylesheets + + // Guard against trailing slashes, because "some/remote/../foo.png" + // resolves to "some/remote/foo.png" on (some?) clients (bug 27052). + if ( substr( $remote, -1 ) == '/' ) { + $remote = substr( $remote, 0, -1 ); + } + + // Replace all comments by a placeholder so they will not interfere with the remapping. + // Warning: This will also catch on anything looking like the start of a comment between + // quotation marks (e.g. "foo /* bar"). + $comments = array(); + $placeholder = uniqid( '', true ); + + $pattern = '/(?!' . CSSMin::EMBED_REGEX . ')(' . CSSMin::COMMENT_REGEX . ')/s'; + + $source = preg_replace_callback( + $pattern, + function ( $match ) use ( &$comments, $placeholder ) { + $comments[] = $match[ 0 ]; + return $placeholder . ( count( $comments ) - 1 ) . 'x'; + }, + $source + ); + + // Note: This will not correctly handle cases where ';', '{' or '}' + // appears in the rule itself, e.g. in a quoted string. You are advised + // not to use such characters in file names. We also match start/end of + // the string to be consistent in edge-cases ('@import url(…)'). + $pattern = '/(?:^|[;{])\K[^;{}]*' . CSSMin::URL_REGEX . '[^;}]*(?=[;}]|$)/'; + + $source = preg_replace_callback( + $pattern, + function ( $matchOuter ) use ( $local, $remote, $embedData, $placeholder ) { + $rule = $matchOuter[0]; + + // Check for global @embed comment and remove it. Allow other comments to be present + // before @embed (they have been replaced with placeholders at this point). + $embedAll = false; + $rule = preg_replace( '/^((?:\s+|' . $placeholder . '(\d+)x)*)' . CSSMin::EMBED_REGEX . '\s*/', '$1', $rule, 1, $embedAll ); + + // Build two versions of current rule: with remapped URLs + // and with embedded data: URIs (where possible). + $pattern = '/(?P<embed>' . CSSMin::EMBED_REGEX . '\s*|)' . CSSMin::URL_REGEX . '/'; + + $ruleWithRemapped = preg_replace_callback( + $pattern, + function ( $match ) use ( $local, $remote ) { + $remapped = CSSMin::remapOne( $match['file'], $match['query'], $local, $remote, false ); + + return CSSMin::buildUrlValue( $remapped ); + }, + $rule + ); + + if ( $embedData ) { + $ruleWithEmbedded = preg_replace_callback( + $pattern, + function ( $match ) use ( $embedAll, $local, $remote ) { + $embed = $embedAll || $match['embed']; + $embedded = CSSMin::remapOne( + $match['file'], + $match['query'], + $local, + $remote, + $embed + ); + + return CSSMin::buildUrlValue( $embedded ); + }, + $rule ); } - // Move the offset to the end of the match, leaving it alone - $offset = $match[0][1] + strlen( $match[0][0] ) + $lengthIncrease; - continue; - } - // Guard against double slashes, because "some/remote/../foo.png" - // resolves to "some/remote/foo.png" on (some?) clients (bug 27052). - if ( substr( $remote, -1 ) == '/' ) { - $remote = substr( $remote, 0, -1 ); - } + if ( $embedData && $ruleWithEmbedded !== $ruleWithRemapped ) { + // Build 2 CSS properties; one which uses a base64 encoded data URI in place + // of the @embed comment to try and retain line-number integrity, and the + // other with a remapped an versioned URL and an Internet Explorer hack + // making it ignored in all browsers that support data URIs + return "$ruleWithEmbedded;$ruleWithRemapped!ie"; + } else { + // No reason to repeat twice + return $ruleWithRemapped; + } + }, $source ); - // Shortcuts - $embed = $match['embed'][0]; - $pre = $match['pre'][0]; - $post = $match['post'][0]; - $query = $match['query'][0]; - $url = "{$remote}/{$match['file'][0]}"; - $file = "{$local}/{$match['file'][0]}"; + // Re-insert comments + $pattern = '/' . $placeholder . '(\d+)x/'; + $source = preg_replace_callback( $pattern, function( $match ) use ( &$comments ) { + return $comments[ $match[1] ]; + }, $source ); - $replacement = false; + return $source; - if ( $local !== false && file_exists( $file ) ) { + } + + /** + * Remap or embed a CSS URL path. + * + * @param string $file URL to remap/embed + * @param string $query + * @param string $local File path where the source was read from + * @param string $remote URL path to the file + * @param bool $embed Whether to do any data URI embedding + * @return string Remapped/embedded URL data + */ + public static function remapOne( $file, $query, $local, $remote, $embed ) { + // The full URL possibly with query, as passed to the 'url()' value in CSS + $url = $file . $query; + + // Skip fully-qualified and protocol-relative URLs and data URIs + if ( substr( $url, 0, 2 ) === '//' || parse_url( $url, PHP_URL_SCHEME ) ) { + return $url; + } + + // URLs with absolute paths like /w/index.php need to be expanded + // to absolute URLs but otherwise left alone + if ( $url !== '' && $url[0] === '/' ) { + // Replace the file path with an expanded (possibly protocol-relative) URL + // ...but only if wfExpandUrl() is even available. + // This will not be the case if we're running outside of MW + if ( function_exists( 'wfExpandUrl' ) ) { + return wfExpandUrl( $url, PROTO_RELATIVE ); + } else { + return $url; + } + } + + if ( $local === false ) { + // Assume that all paths are relative to $remote, and make them absolute + return $remote . '/' . $url; + } else { + // We drop the query part here and instead make the path relative to $remote + $url = "{$remote}/{$file}"; + // Path to the actual file on the filesystem + $localFile = "{$local}/{$file}"; + if ( file_exists( $localFile ) ) { // Add version parameter as a time-stamp in ISO 8601 format, // using Z for the timezone, meaning GMT - $url .= '?' . gmdate( 'Y-m-d\TH:i:s\Z', round( filemtime( $file ), -2 ) ); - // Embedding requires a bit of extra processing, so let's skip that if we can - if ( $embedData && $embed && $match['embed'][1] > 0 ) { - $data = self::encodeImageAsDataURI( $file ); + $url .= '?' . gmdate( 'Y-m-d\TH:i:s\Z', round( filemtime( $localFile ), -2 ) ); + if ( $embed ) { + $data = self::encodeImageAsDataURI( $localFile ); if ( $data !== false ) { - // Build 2 CSS properties; one which uses a base64 encoded data URI in place - // of the @embed comment to try and retain line-number integrity, and the - // other with a remapped an versioned URL and an Internet Explorer hack - // making it ignored in all browsers that support data URIs - $replacement = "{$pre}url({$data}){$post};{$pre}url({$url}){$post}!ie;"; + return $data; } } - if ( $replacement === false ) { - // Assume that all paths are relative to $remote, and make them absolute - $replacement = "{$embed}{$pre}url({$url}){$post};"; - } - } elseif ( $local === false ) { - // Assume that all paths are relative to $remote, and make them absolute - $replacement = "{$embed}{$pre}url({$url}{$query}){$post};"; } - if ( $replacement !== false ) { - // Perform replacement on the source - $source = substr_replace( - $source, $replacement, $match[0][1], strlen( $match[0][0] ) - ); - // Move the offset to the end of the replacement in the source - $offset = $match[0][1] + strlen( $replacement ); - continue; - } - // Move the offset to the end of the match, leaving it alone - $offset = $match[0][1] + strlen( $match[0][0] ); + // If any of these conditions failed (file missing, we don't want to embed it + // or it's not embeddable), return the URL (possibly with ?timestamp part) + return $url; } - return $source; } /** diff --git a/includes/libs/GenericArrayObject.php b/includes/libs/GenericArrayObject.php index d77d8ad6..db8a7ecf 100644 --- a/includes/libs/GenericArrayObject.php +++ b/includes/libs/GenericArrayObject.php @@ -33,7 +33,6 @@ * @author Jeroen De Dauw < jeroendedauw@gmail.com > */ abstract class GenericArrayObject extends ArrayObject { - /** * Returns the name of an interface/class that the element should implement/extend. * @@ -144,7 +143,8 @@ abstract class GenericArrayObject extends ArrayObject { protected function setElement( $index, $value ) { if ( !$this->hasValidType( $value ) ) { throw new InvalidArgumentException( - 'Can only add ' . $this->getObjectType() . ' implementing objects to ' . get_called_class() . '.' + 'Can only add ' . $this->getObjectType() . ' implementing objects to ' + . get_called_class() . '.' ); } @@ -237,5 +237,4 @@ abstract class GenericArrayObject extends ArrayObject { public function isEmpty() { return $this->count() === 0; } - } diff --git a/includes/libs/HashRing.php b/includes/libs/HashRing.php new file mode 100644 index 00000000..2022b225 --- /dev/null +++ b/includes/libs/HashRing.php @@ -0,0 +1,239 @@ +<?php +/** + * Convenience class for weighted consistent hash rings. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * Convenience class for weighted consistent hash rings + * + * @since 1.22 + */ +class HashRing { + /** @var Array (location => weight) */ + protected $sourceMap = array(); + /** @var Array (location => (start, end)) */ + protected $ring = array(); + + /** @var Array (location => (start, end)) */ + protected $liveRing; + /** @var Array (location => UNIX timestamp) */ + protected $ejectionExpiries = array(); + /** @var integer UNIX timestamp */ + protected $ejectionNextExpiry = INF; + + const RING_SIZE = 268435456; // 2^28 + + /** + * @param array $map (location => weight) + */ + public function __construct( array $map ) { + $map = array_filter( $map, function ( $w ) { + return $w > 0; + } ); + if ( !count( $map ) ) { + throw new UnexpectedValueException( "Ring is empty or all weights are zero." ); + } + $this->sourceMap = $map; + // Sort the locations based on the hash of their names + $hashes = array(); + foreach ( $map as $location => $weight ) { + $hashes[$location] = sha1( $location ); + } + uksort( $map, function ( $a, $b ) use ( $hashes ) { + return strcmp( $hashes[$a], $hashes[$b] ); + } ); + // Fit the map to weight-proportionate one with a space of size RING_SIZE + $sum = array_sum( $map ); + $standardMap = array(); + foreach ( $map as $location => $weight ) { + $standardMap[$location] = (int)floor( $weight / $sum * self::RING_SIZE ); + } + // Build a ring of RING_SIZE spots, with each location at a spot in location hash order + $index = 0; + foreach ( $standardMap as $location => $weight ) { + // Location covers half-closed interval [$index,$index + $weight) + $this->ring[$location] = array( $index, $index + $weight ); + $index += $weight; + } + // Make sure the last location covers what is left + end( $this->ring ); + $this->ring[key( $this->ring )][1] = self::RING_SIZE; + } + + /** + * Get the location of an item on the ring + * + * @param string $item + * @return string Location + */ + public function getLocation( $item ) { + $locations = $this->getLocations( $item, 1 ); + + return $locations[0]; + } + + /** + * Get the location of an item on the ring, as well as the next locations + * + * @param string $item + * @param integer $limit Maximum number of locations to return + * @return array List of locations + */ + public function getLocations( $item, $limit ) { + $locations = array(); + $primaryLocation = null; + $spot = hexdec( substr( sha1( $item ), 0, 7 ) ); // first 28 bits + foreach ( $this->ring as $location => $range ) { + if ( count( $locations ) >= $limit ) { + break; + } + // The $primaryLocation is the location the item spot is in. + // After that is reached, keep appending the next locations. + if ( ( $range[0] <= $spot && $spot < $range[1] ) || $primaryLocation !== null ) { + if ( $primaryLocation === null ) { + $primaryLocation = $location; + } + $locations[] = $location; + } + } + // If more locations are requested, wrap-around and keep adding them + reset( $this->ring ); + while ( count( $locations ) < $limit ) { + list( $location, ) = each( $this->ring ); + if ( $location === $primaryLocation ) { + break; // don't go in circles + } + $locations[] = $location; + } + + return $locations; + } + + /** + * Get the map of locations to weight (ignores 0-weight items) + * + * @return array + */ + public function getLocationWeights() { + return $this->sourceMap; + } + + /** + * Get a new hash ring with a location removed from the ring + * + * @param string $location + * @return HashRing|bool Returns false if no non-zero weighted spots are left + */ + public function newWithoutLocation( $location ) { + $map = $this->sourceMap; + unset( $map[$location] ); + + return count( $map ) ? new self( $map ) : false; + } + + /** + * Remove a location from the "live" hash ring + * + * @param string $location + * @param integer $ttl Seconds + * @return bool Whether some non-ejected locations are left + */ + public function ejectFromLiveRing( $location, $ttl ) { + if ( !isset( $this->sourceMap[$location] ) ) { + throw new UnexpectedValueException( "No location '$location' in the ring." ); + } + $expiry = time() + $ttl; + $this->liveRing = null; // stale + $this->ejectionExpiries[$location] = $expiry; + $this->ejectionNextExpiry = min( $expiry, $this->ejectionNextExpiry ); + + return ( count( $this->ejectionExpiries ) < count( $this->sourceMap ) ); + } + + /** + * Get the "live" hash ring (which does not include ejected locations) + * + * @return HashRing + * @throws UnexpectedValueException + */ + public function getLiveRing() { + $now = time(); + if ( $this->liveRing === null || $this->ejectionNextExpiry <= $now ) { + $this->ejectionExpiries = array_filter( + $this->ejectionExpiries, + function( $expiry ) use ( $now ) { + return ( $expiry > $now ); + } + ); + if ( count( $this->ejectionExpiries ) ) { + $map = array_diff_key( $this->sourceMap, $this->ejectionExpiries ); + $this->liveRing = count( $map ) ? new self( $map ) : false; + + $this->ejectionNextExpiry = min( $this->ejectionExpiries ); + } else { // common case; avoid recalculating ring + $this->liveRing = clone $this; + $this->liveRing->ejectionExpiries = array(); + $this->liveRing->ejectionNextExpiry = INF; + $this->liveRing->liveRing = null; + + $this->ejectionNextExpiry = INF; + } + } + if ( !$this->liveRing ) { + throw new UnexpectedValueException( "The live ring is currently empty." ); + } + + return $this->liveRing; + } + + /** + * Get the location of an item on the "live" ring + * + * @param string $item + * @return string Location + * @throws UnexpectedValueException + */ + public function getLiveLocation( $item ) { + return $this->getLiveRing()->getLocation( $item ); + } + + /** + * Get the location of an item on the "live" ring, as well as the next locations + * + * @param string $item + * @param integer $limit Maximum number of locations to return + * @return array List of locations + * @throws UnexpectedValueException + */ + public function getLiveLocations( $item ) { + return $this->getLiveRing()->getLocations( $item ); + } + + /** + * Get the map of "live" locations to weight (ignores 0-weight items) + * + * @return array + * @throws UnexpectedValueException + */ + public function getLiveLocationWeights() { + return $this->getLiveRing()->getLocationWeights(); + } +} diff --git a/includes/libs/HttpStatus.php b/includes/libs/HttpStatus.php index 4f626b23..809bfdf5 100644 --- a/includes/libs/HttpStatus.php +++ b/includes/libs/HttpStatus.php @@ -28,8 +28,6 @@ class HttpStatus { /** * Get the message associated with HTTP response code $code * - * Replace OutputPage::getStatusMessage( $code ) - * * @param $code Integer: status code * @return String or null: message or null if $code is not in the list of * messages @@ -75,13 +73,17 @@ class HttpStatus { 422 => 'Unprocessable Entity', 423 => 'Locked', 424 => 'Failed Dependency', + 428 => 'Precondition Required', + 429 => 'Too Many Requests', + 431 => 'Request Header Fields Too Large', 500 => 'Internal Server Error', 501 => 'Not Implemented', 502 => 'Bad Gateway', 503 => 'Service Unavailable', 504 => 'Gateway Timeout', 505 => 'HTTP Version Not Supported', - 507 => 'Insufficient Storage' + 507 => 'Insufficient Storage', + 511 => 'Network Authentication Required', ); return isset( $statusMessage[$code] ) ? $statusMessage[$code] : null; } diff --git a/includes/libs/IEContentAnalyzer.php b/includes/libs/IEContentAnalyzer.php index 7f461a03..c31a3527 100644 --- a/includes/libs/IEContentAnalyzer.php +++ b/includes/libs/IEContentAnalyzer.php @@ -333,7 +333,7 @@ class IEContentAnalyzer { * @param string $chunk the first 256 bytes of the file * @param string $proposed the MIME type proposed by the server * - * @return Array: map of IE version to detected mime type + * @return Array: map of IE version to detected MIME type */ public function getRealMimesFromData( $fileName, $chunk, $proposed ) { $types = $this->getMimesFromData( $fileName, $chunk, $proposed ); @@ -371,7 +371,7 @@ class IEContentAnalyzer { * @param string $chunk the first 256 bytes of the file * @param string $proposed the MIME type proposed by the server * - * @return Array: map of IE version to detected mime type + * @return Array: map of IE version to detected MIME type */ public function getMimesFromData( $fileName, $chunk, $proposed ) { $types = array(); @@ -712,8 +712,9 @@ class IEContentAnalyzer { $xbmMagic2 = '_width'; $xbmMagic3 = '_bits'; $binhexMagic = 'converted with BinHex'; + $chunkLength = strlen( $chunk ); - for ( $offset = 0; $offset < strlen( $chunk ); $offset++ ) { + for ( $offset = 0; $offset < $chunkLength; $offset++ ) { $curChar = $chunk[$offset]; if ( $curChar == "\x0a" ) { $counters['lf']++; diff --git a/includes/libs/IPSet.php b/includes/libs/IPSet.php new file mode 100644 index 00000000..ae593785 --- /dev/null +++ b/includes/libs/IPSet.php @@ -0,0 +1,277 @@ +<?php +/** + * @section LICENSE + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Brandon Black <blblack@gmail.com> + */ + +/** + * Matches IP addresses against a set of CIDR specifications + * + * Usage: + * // At startup, calculate the optimized data structure for the set: + * $ipset = new IPSet( $wgSquidServersNoPurge ); + * // runtime check against cached set (returns bool): + * $allowme = $ipset->match( $ip ); + * + * In rough benchmarking, this takes about 80% more time than + * in_array() checks on a short (a couple hundred at most) array + * of addresses. It's fast either way at those levels, though, + * and IPSet would scale better than in_array if the array were + * much larger. + * + * For mixed-family CIDR sets, however, this code gives well over + * 100x speedup vs iterating IP::isInRange() over an array + * of CIDR specs. + * + * The basic implementation is two separate binary trees + * (IPv4 and IPv6) as nested php arrays with keys named 0 and 1. + * The values false and true are terminal match-fail and match-success, + * otherwise the value is a deeper node in the tree. + * + * A simple depth-compression scheme is also implemented: whole-byte + * tree compression at whole-byte boundaries only, where no branching + * occurs during that whole byte of depth. A compressed node has + * keys 'comp' (the byte to compare) and 'next' (the next node to + * recurse into if 'comp' matched successfully). + * + * For example, given these inputs: + * 25.0.0.0/9 + * 25.192.0.0/10 + * + * The v4 tree would look like: + * root4 => array( + * 'comp' => 25, + * 'next' => array( + * 0 => true, + * 1 => array( + * 0 => false, + * 1 => true, + * ), + * ), + * ); + * + * (multi-byte compression nodes were attempted as well, but were + * a net loss in my test scenarios due to additional match complexity) + * + * @since 1.24 + */ +class IPSet { + /** @var array $root4: the root of the IPv4 matching tree */ + private $root4 = array( false, false ); + + /** @var array $root6: the root of the IPv6 matching tree */ + private $root6 = array( false, false ); + + /** + * __construct() instantiate the object from an array of CIDR specs + * + * @param array $cfg array of IPv[46] CIDR specs as strings + * @return IPSet new IPSet object + * + * Invalid input network/mask values in $cfg will result in issuing + * E_WARNING and/or E_USER_WARNING and the bad values being ignored. + */ + public function __construct( array $cfg ) { + foreach ( $cfg as $cidr ) { + $this->addCidr( $cidr ); + } + + self::recOptimize( $this->root4 ); + self::recCompress( $this->root4, 0, 24 ); + self::recOptimize( $this->root6 ); + self::recCompress( $this->root6, 0, 120 ); + } + + /** + * Add a single CIDR spec to the internal matching trees + * + * @param string $cidr string CIDR spec, IPv[46], optional /mask (def all-1's) + */ + private function addCidr( $cidr ) { + // v4 or v6 check + if ( strpos( $cidr, ':' ) === false ) { + $node =& $this->root4; + $defMask = '32'; + } else { + $node =& $this->root6; + $defMask = '128'; + } + + // Default to all-1's mask if no netmask in the input + if ( strpos( $cidr, '/' ) === false ) { + $net = $cidr; + $mask = $defMask; + } else { + list( $net, $mask ) = explode( '/', $cidr, 2 ); + if ( !ctype_digit( $mask ) || intval( $mask ) > $defMask ) { + trigger_error( "IPSet: Bad mask '$mask' from '$cidr', ignored", E_USER_WARNING ); + return; + } + } + $mask = intval( $mask ); // explicit integer convert, checked above + + // convert $net to an array of integer bytes, length 4 or 16: + $raw = inet_pton( $net ); + if ( $raw === false ) { + return; // inet_pton() sends an E_WARNING for us + } + $rawOrd = array_map( 'ord', str_split( $raw ) ); + + // special-case: zero mask overwrites the whole tree with a pair of terminal successes + if ( $mask == 0 ) { + $node = array( true, true ); + return; + } + + // iterate the bits of the address while walking the tree structure for inserts + $curBit = 0; + while ( 1 ) { + $maskShift = 7 - ( $curBit & 7 ); + $node =& $node[( $rawOrd[$curBit >> 3] & ( 1 << $maskShift ) ) >> $maskShift]; + ++$curBit; + if ( $node === true ) { + // already added a larger supernet, no need to go deeper + return; + } elseif ( $curBit == $mask ) { + // this may wipe out deeper subnets from earlier + $node = true; + return; + } elseif ( $node === false ) { + // create new subarray to go deeper + $node = array( false, false ); + } + } + } + + /** + * Match an IP address against the set + * + * @param string $ip string IPv[46] address + * @return boolean true is match success, false is match failure + * + * If $ip is unparseable, inet_pton may issue an E_WARNING to that effect + */ + public function match( $ip ) { + $raw = inet_pton( $ip ); + if ( $raw === false ) { + return false; // inet_pton() sends an E_WARNING for us + } + + $rawOrd = array_map( 'ord', str_split( $raw ) ); + if ( count( $rawOrd ) == 4 ) { + $node =& $this->root4; + } else { + $node =& $this->root6; + } + + $curBit = 0; + while ( 1 ) { + if ( isset( $node['comp'] ) ) { + // compressed node, matches 1 whole byte on a byte boundary + if ( $rawOrd[$curBit >> 3] != $node['comp'] ) { + return false; + } + $curBit += 8; + $node =& $node['next']; + } else { + // uncompressed node, walk in the correct direction for the current bit-value + $maskShift = 7 - ( $curBit & 7 ); + $node =& $node[( $rawOrd[$curBit >> 3] & ( 1 << $maskShift ) ) >> $maskShift]; + ++$curBit; + } + + if ( $node === true || $node === false ) { + return $node; + } + } + } + + /** + * Recursively merges adjacent nets into larger supernets + * + * @param array &$node Tree node to optimize, by-reference + * + * e.g.: 8.0.0.0/8 + 9.0.0.0/8 -> 8.0.0.0/7 + */ + private static function recOptimize( &$node ) { + if ( $node[0] !== false && $node[0] !== true && self::recOptimize( $node[0] ) ) { + $node[0] = true; + } + if ( $node[1] !== false && $node[1] !== true && self::recOptimize( $node[1] ) ) { + $node[1] = true; + } + if ( $node[0] === true && $node[1] === true ) { + return true; + } + return false; + } + + /** + * Recursively compresses a tree + * + * @param array &$node Tree node to compress, by-reference + * @param integer $curBit current depth in the tree + * @param integer $maxCompStart maximum depth at which compression can start, family-specific + * + * This is a very simplistic compression scheme: if we go through a whole + * byte of address starting at a byte boundary with no real branching + * other than immediate false-vs-(node|true), compress that subtree down to a single + * byte-matching node. + * The $maxCompStart check elides recursing the final 7 levels of depth (family-dependent) + */ + private static function recCompress( &$node, $curBit, $maxCompStart ) { + if ( !( $curBit & 7 ) ) { // byte boundary, check for depth-8 single path(s) + $byte = 0; + $cnode =& $node; + $i = 8; + while ( $i-- ) { + if ( $cnode[0] === false ) { + $byte |= 1 << $i; + $cnode =& $cnode[1]; + } elseif ( $cnode[1] === false ) { + $cnode =& $cnode[0]; + } else { + // partial-byte branching, give up + break; + } + } + if ( $i == -1 ) { // means we did not exit the while() via break + $node = array( + 'comp' => $byte, + 'next' => &$cnode, + ); + $curBit += 8; + if ( $cnode !== true ) { + self::recCompress( $cnode, $curBit, $maxCompStart ); + } + return; + } + } + + ++$curBit; + if ( $curBit <= $maxCompStart ) { + if ( $node[0] !== false && $node[0] !== true ) { + self::recCompress( $node[0], $curBit, $maxCompStart ); + } + if ( $node[1] !== false && $node[1] !== true ) { + self::recCompress( $node[1], $curBit, $maxCompStart ); + } + } + } +} diff --git a/includes/libs/JavaScriptMinifier.php b/includes/libs/JavaScriptMinifier.php index 998805ae..2990782c 100644 --- a/includes/libs/JavaScriptMinifier.php +++ b/includes/libs/JavaScriptMinifier.php @@ -1,4 +1,5 @@ <?php +// @codingStandardsIgnoreFile File external to MediaWiki. Ignore coding conventions checks. /** * JavaScript Minifier * diff --git a/includes/libs/MWMessagePack.php b/includes/libs/MWMessagePack.php new file mode 100644 index 00000000..cd9aad8f --- /dev/null +++ b/includes/libs/MWMessagePack.php @@ -0,0 +1,189 @@ +<?php +/** + * MessagePack serializer + * + * MessagePack is a space-efficient binary data interchange format. This + * class provides a pack() method that encodes native PHP values as MessagePack + * binary strings. The implementation is derived from msgpack-php. + * + * Copyright (c) 2013 Ori Livneh <ori@wikimedia.org> + * Copyright (c) 2011 OnlineCity <https://github.com/onlinecity/msgpack-php>. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * @see <http://msgpack.org/> + * @see <http://wiki.msgpack.org/display/MSGPACK/Format+specification> + * + * @since 1.23 + * @file + */ +class MWMessagePack { + /** @var boolean|null Whether current system is bigendian. **/ + public static $bigendian = null; + + /** + * Encode a value using MessagePack + * + * This method supports null, boolean, integer, float, string and array + * (both indexed and associative) types. Object serialization is not + * supported. + * + * @param mixed $value + * @return string + * @throws InvalidArgumentException if $value is an unsupported type or too long a string + */ + public static function pack( $value ) { + if ( self::$bigendian === null ) { + self::$bigendian = pack( 'S', 1 ) === pack( 'n', 1 ); + } + + switch ( gettype( $value ) ) { + case 'NULL': + return "\xC0"; + + case 'boolean': + return $value ? "\xC3" : "\xC2"; + + case 'double': + case 'float': + return self::$bigendian + ? "\xCB" . pack( 'd', $value ) + : "\xCB" . strrev( pack( 'd', $value ) ); + + case 'string': + $length = strlen( $value ); + if ( $length < 32 ) { + return pack( 'Ca*', 0xA0 | $length, $value ); + } elseif ( $length <= 0xFFFF ) { + return pack( 'Cna*', 0xDA, $length, $value ); + } elseif ( $length <= 0xFFFFFFFF ) { + return pack( 'CNa*', 0xDB, $length, $value ); + } + throw new InvalidArgumentException( __METHOD__ + . ": string too long (length: $length; max: 4294967295)" ); + + case 'integer': + if ( $value >= 0 ) { + if ( $value <= 0x7F ) { + // positive fixnum + return chr( $value ); + } + if ( $value <= 0xFF ) { + // uint8 + return pack( 'CC', 0xCC, $value ); + } + if ( $value <= 0xFFFF ) { + // uint16 + return pack( 'Cn', 0xCD, $value ); + } + if ( $value <= 0xFFFFFFFF ) { + // uint32 + return pack( 'CN', 0xCE, $value ); + } + if ( $value <= 0xFFFFFFFFFFFFFFFF ) { + // uint64 + $hi = ( $value & 0xFFFFFFFF00000000 ) >> 32; + $lo = $value & 0xFFFFFFFF; + return self::$bigendian + ? pack( 'CNN', 0xCF, $lo, $hi ) + : pack( 'CNN', 0xCF, $hi, $lo ); + } + } else { + if ( $value >= -32 ) { + // negative fixnum + return pack( 'c', $value ); + } + if ( $value >= -0x80 ) { + // int8 + return pack( 'Cc', 0xD0, $value ); + } + if ( $value >= -0x8000 ) { + // int16 + $p = pack( 's', $value ); + return self::$bigendian + ? pack( 'Ca2', 0xD1, $p ) + : pack( 'Ca2', 0xD1, strrev( $p ) ); + } + if ( $value >= -0x80000000 ) { + // int32 + $p = pack( 'l', $value ); + return self::$bigendian + ? pack( 'Ca4', 0xD2, $p ) + : pack( 'Ca4', 0xD2, strrev( $p ) ); + } + if ( $value >= -0x8000000000000000 ) { + // int64 + // pack() does not support 64-bit ints either so pack into two 32-bits + $p1 = pack( 'l', $value & 0xFFFFFFFF ); + $p2 = pack( 'l', ( $value >> 32 ) & 0xFFFFFFFF ); + return self::$bigendian + ? pack( 'Ca4a4', 0xD3, $p1, $p2 ) + : pack( 'Ca4a4', 0xD3, strrev( $p2 ), strrev( $p1 ) ); + } + } + throw new InvalidArgumentException( __METHOD__ . ": invalid integer '$value'" ); + + case 'array': + $buffer = ''; + $length = count( $value ); + if ( $length > 0xFFFFFFFF ) { + throw new InvalidArgumentException( __METHOD__ + . ": array too long (length: $length, max: 4294967295)" ); + } + + $index = 0; + foreach ( $value as $k => $v ) { + if ( $index !== $k || $index === $length ) { + break; + } else { + $index++; + } + } + $associative = $index !== $length; + + if ( $associative ) { + if ( $length < 16 ) { + $buffer .= pack( 'C', 0x80 | $length ); + } elseif ( $length <= 0xFFFF ) { + $buffer .= pack( 'Cn', 0xDE, $length ); + } else { + $buffer .= pack( 'CN', 0xDF, $length ); + } + foreach ( $value as $k => $v ) { + $buffer .= self::pack( $k ); + $buffer .= self::pack( $v ); + } + } else { + if ( $length < 16 ) { + $buffer .= pack( 'C', 0x90 | $length ); + } elseif ( $length <= 0xFFFF ) { + $buffer .= pack( 'Cn', 0xDC, $length ); + } else { + $buffer .= pack( 'CN', 0xDD, $length ); + } + foreach ( $value as $v ) { + $buffer .= self::pack( $v ); + } + } + return $buffer; + + default: + throw new InvalidArgumentException( __METHOD__ . ': unsupported type ' . gettype( $value ) ); + } + } +} diff --git a/includes/libs/MappedIterator.php b/includes/libs/MappedIterator.php new file mode 100644 index 00000000..7fdde8a8 --- /dev/null +++ b/includes/libs/MappedIterator.php @@ -0,0 +1,117 @@ +<?php +/** + * Convenience class for generating iterators from iterators. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * Convenience class for generating iterators from iterators. + * + * @since 1.21 + */ +class MappedIterator extends FilterIterator { + /** @var callable */ + protected $vCallback; + /** @var callable */ + protected $aCallback; + /** @var array */ + protected $cache = array(); + + protected $rewound = false; // boolean; whether rewind() has been called + + /** + * Build an new iterator from a base iterator by having the former wrap the + * later, returning the result of "value" callback for each current() invocation. + * The callback takes the result of current() on the base iterator as an argument. + * The keys of the base iterator are reused verbatim. + * + * An "accept" callback can also be provided which will be called for each value in + * the base iterator (post-callback) and will return true if that value should be + * included in iteration of the MappedIterator (otherwise it will be filtered out). + * + * @param Iterator|Array $iter + * @param callable $vCallback Value transformation callback + * @param array $options Options map (includes "accept") (since 1.22) + * @throws UnexpectedValueException + */ + public function __construct( $iter, $vCallback, array $options = array() ) { + if ( is_array( $iter ) ) { + $baseIterator = new ArrayIterator( $iter ); + } elseif ( $iter instanceof Iterator ) { + $baseIterator = $iter; + } else { + throw new UnexpectedValueException( "Invalid base iterator provided." ); + } + parent::__construct( $baseIterator ); + $this->vCallback = $vCallback; + $this->aCallback = isset( $options['accept'] ) ? $options['accept'] : null; + } + + public function next() { + $this->cache = array(); + parent::next(); + } + + public function rewind() { + $this->rewound = true; + $this->cache = array(); + parent::rewind(); + } + + public function accept() { + $value = call_user_func( $this->vCallback, $this->getInnerIterator()->current() ); + $ok = ( $this->aCallback ) ? call_user_func( $this->aCallback, $value ) : true; + if ( $ok ) { + $this->cache['current'] = $value; + } + + return $ok; + } + + public function key() { + $this->init(); + + return parent::key(); + } + + public function valid() { + $this->init(); + + return parent::valid(); + } + + public function current() { + $this->init(); + if ( parent::valid() ) { + return $this->cache['current']; + } else { + return null; // out of range + } + } + + /** + * Obviate the usual need for rewind() before using a FilterIterator in a manual loop + */ + protected function init() { + if ( !$this->rewound ) { + $this->rewind(); + } + } +} diff --git a/includes/libs/MultiHttpClient.php b/includes/libs/MultiHttpClient.php new file mode 100644 index 00000000..8c982c43 --- /dev/null +++ b/includes/libs/MultiHttpClient.php @@ -0,0 +1,389 @@ +<?php +/** + * HTTP service client + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Class to handle concurrent HTTP requests + * + * HTTP request maps are arrays that use the following format: + * - method : GET/HEAD/PUT/POST/DELETE + * - url : HTTP/HTTPS URL + * - query : <query parameter field/value associative array> (uses RFC 3986) + * - headers : <header name/value associative array> + * - body : source to get the HTTP request body from; + * this can simply be a string (always), a resource for + * PUT requests, and a field/value array for POST request; + * array bodies are encoded as multipart/form-data and strings + * use application/x-www-form-urlencoded (headers sent automatically) + * - stream : resource to stream the HTTP response body to + * Request maps can use integer index 0 instead of 'method' and 1 instead of 'url'. + * + * @author Aaron Schulz + * @since 1.23 + */ +class MultiHttpClient { + /** @var resource */ + protected $multiHandle = null; // curl_multi handle + /** @var string|null SSL certificates path */ + protected $caBundlePath; + /** @var integer */ + protected $connTimeout = 10; + /** @var integer */ + protected $reqTimeout = 300; + /** @var bool */ + protected $usePipelining = false; + /** @var integer */ + protected $maxConnsPerHost = 50; + + /** + * @param array $options + * - connTimeout : default connection timeout + * - reqTimeout : default request timeout + * - usePipelining : whether to use HTTP pipelining if possible (for all hosts) + * - maxConnsPerHost : maximum number of concurrent connections (per host) + */ + public function __construct( array $options ) { + if ( isset( $options['caBundlePath'] ) ) { + $this->caBundlePath = $options['caBundlePath']; + if ( !file_exists( $this->caBundlePath ) ) { + throw new Exception( "Cannot find CA bundle: " . $this->caBundlePath ); + } + } + static $opts = array( 'connTimeout', 'reqTimeout', 'usePipelining', 'maxConnsPerHost' ); + foreach ( $opts as $key ) { + if ( isset( $options[$key] ) ) { + $this->$key = $options[$key]; + } + } + } + + /** + * Execute an HTTP(S) request + * + * This method returns a response map of: + * - code : HTTP response code or 0 if there was a serious cURL error + * - reason : HTTP response reason (empty if there was a serious cURL error) + * - headers : <header name/value associative array> + * - body : HTTP response body or resource (if "stream" was set) + * - err : Any cURL error string + * The map also stores integer-indexed copies of these values. This lets callers do: + * <code> + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $http->run( $req ); + * </code> + * @param array $req HTTP request array + * @param array $opts + * - connTimeout : connection timeout per request + * - reqTimeout : post-connection timeout per request + * @return array Response array for request + */ + final public function run( array $req, array $opts = array() ) { + $req = $this->runMulti( array( $req ), $opts ); + return $req[0]['response']; + } + + /** + * Execute a set of HTTP(S) requests concurrently + * + * The maps are returned by this method with the 'response' field set to a map of: + * - code : HTTP response code or 0 if there was a serious cURL error + * - reason : HTTP response reason (empty if there was a serious cURL error) + * - headers : <header name/value associative array> + * - body : HTTP response body or resource (if "stream" was set) + * - err : Any cURL error string + * The map also stores integer-indexed copies of these values. This lets callers do: + * <code> + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $req['response']; + * </code> + * All headers in the 'headers' field are normalized to use lower case names. + * This is true for the request headers and the response headers. Integer-indexed + * method/URL entries will also be changed to use the corresponding string keys. + * + * @param array $reqs Map of HTTP request arrays + * @param array $opts + * - connTimeout : connection timeout per request + * - reqTimeout : post-connection timeout per request + * - usePipelining : whether to use HTTP pipelining if possible + * - maxConnsPerHost : maximum number of concurrent connections (per host) + * @return array $reqs With response array populated for each + */ + public function runMulti( array $reqs, array $opts = array() ) { + $chm = $this->getCurlMulti(); + + // Normalize $reqs and add all of the required cURL handles... + $handles = array(); + foreach ( $reqs as $index => &$req ) { + $req['response'] = array( + 'code' => 0, + 'reason' => '', + 'headers' => array(), + 'body' => '', + 'error' => '' + ); + if ( isset( $req[0] ) ) { + $req['method'] = $req[0]; // short-form + unset( $req[0] ); + } + if ( isset( $req[1] ) ) { + $req['url'] = $req[1]; // short-form + unset( $req[1] ); + } + if ( !isset( $req['method'] ) ) { + throw new Exception( "Request has no 'method' field set." ); + } elseif ( !isset( $req['url'] ) ) { + throw new Exception( "Request has no 'url' field set." ); + } + $req['query'] = isset( $req['query'] ) ? $req['query'] : array(); + $headers = array(); // normalized headers + if ( isset( $req['headers'] ) ) { + foreach ( $req['headers'] as $name => $value ) { + $headers[strtolower( $name )] = $value; + } + } + $req['headers'] = $headers; + if ( !isset( $req['body'] ) ) { + $req['body'] = ''; + $req['headers']['content-length'] = 0; + } + $handles[$index] = $this->getCurlHandle( $req, $opts ); + if ( count( $reqs ) > 1 ) { + // https://github.com/guzzle/guzzle/issues/349 + curl_setopt( $handles[$index], CURLOPT_FORBID_REUSE, true ); + } + } + unset( $req ); // don't assign over this by accident + + $indexes = array_keys( $reqs ); + if ( function_exists( 'curl_multi_setopt' ) ) { // PHP 5.5 + if ( isset( $opts['usePipelining'] ) ) { + curl_multi_setopt( $chm, CURLMOPT_PIPELINING, (int)$opts['usePipelining'] ); + } + if ( isset( $opts['maxConnsPerHost'] ) ) { + // Keep these sockets around as they may be needed later in the request + curl_multi_setopt( $chm, CURLMOPT_MAXCONNECTS, (int)$opts['maxConnsPerHost'] ); + } + } + + // @TODO: use a per-host rolling handle window (e.g. CURLMOPT_MAX_HOST_CONNECTIONS) + $batches = array_chunk( $indexes, $this->maxConnsPerHost ); + + foreach ( $batches as $batch ) { + // Attach all cURL handles for this batch + foreach ( $batch as $index ) { + curl_multi_add_handle( $chm, $handles[$index] ); + } + // Execute the cURL handles concurrently... + $active = null; // handles still being processed + do { + // Do any available work... + do { + $mrc = curl_multi_exec( $chm, $active ); + } while ( $mrc == CURLM_CALL_MULTI_PERFORM ); + // Wait (if possible) for available work... + if ( $active > 0 && $mrc == CURLM_OK ) { + if ( curl_multi_select( $chm, 10 ) == -1 ) { + // PHP bug 63411; http://curl.haxx.se/libcurl/c/curl_multi_fdset.html + usleep( 5000 ); // 5ms + } + } + } while ( $active > 0 && $mrc == CURLM_OK ); + } + + // Remove all of the added cURL handles and check for errors... + foreach ( $reqs as $index => &$req ) { + $ch = $handles[$index]; + curl_multi_remove_handle( $chm, $ch ); + if ( curl_errno( $ch ) !== 0 ) { + $req['response']['error'] = "(curl error: " . + curl_errno( $ch ) . ") " . curl_error( $ch ); + } + // For convenience with the list() operator + $req['response'][0] = $req['response']['code']; + $req['response'][1] = $req['response']['reason']; + $req['response'][2] = $req['response']['headers']; + $req['response'][3] = $req['response']['body']; + $req['response'][4] = $req['response']['error']; + curl_close( $ch ); + // Close any string wrapper file handles + if ( isset( $req['_closeHandle'] ) ) { + fclose( $req['_closeHandle'] ); + unset( $req['_closeHandle'] ); + } + } + unset( $req ); // don't assign over this by accident + + // Restore the default settings + if ( function_exists( 'curl_multi_setopt' ) ) { // PHP 5.5 + curl_multi_setopt( $chm, CURLMOPT_PIPELINING, (int)$this->usePipelining ); + curl_multi_setopt( $chm, CURLMOPT_MAXCONNECTS, (int)$this->maxConnsPerHost ); + } + + return $reqs; + } + + /** + * @param array $req HTTP request map + * @param array $opts + * - connTimeout : default connection timeout + * - reqTimeout : default request timeout + * @return resource + */ + protected function getCurlHandle( array &$req, array $opts = array() ) { + $ch = curl_init(); + + curl_setopt( $ch, CURLOPT_CONNECTTIMEOUT, + isset( $opts['connTimeout'] ) ? $opts['connTimeout'] : $this->connTimeout ); + curl_setopt( $ch, CURLOPT_TIMEOUT, + isset( $opts['reqTimeout'] ) ? $opts['reqTimeout'] : $this->reqTimeout ); + curl_setopt( $ch, CURLOPT_FOLLOWLOCATION, 1 ); + curl_setopt( $ch, CURLOPT_MAXREDIRS, 4 ); + curl_setopt( $ch, CURLOPT_HEADER, 0 ); + if ( !is_null( $this->caBundlePath ) ) { + curl_setopt( $ch, CURLOPT_SSL_VERIFYPEER, true ); + curl_setopt( $ch, CURLOPT_CAINFO, $this->caBundlePath ); + } + curl_setopt( $ch, CURLOPT_RETURNTRANSFER, 1 ); + + $url = $req['url']; + // PHP_QUERY_RFC3986 is PHP 5.4+ only + $query = str_replace( + array( '+', '%7E' ), + array( '%20', '~' ), + http_build_query( $req['query'], '', '&' ) + ); + if ( $query != '' ) { + $url .= strpos( $req['url'], '?' ) === false ? "?$query" : "&$query"; + } + curl_setopt( $ch, CURLOPT_URL, $url ); + + curl_setopt( $ch, CURLOPT_CUSTOMREQUEST, $req['method'] ); + if ( $req['method'] === 'HEAD' ) { + curl_setopt( $ch, CURLOPT_NOBODY, 1 ); + } + + if ( $req['method'] === 'PUT' ) { + curl_setopt( $ch, CURLOPT_PUT, 1 ); + if ( is_resource( $req['body'] ) ) { + curl_setopt( $ch, CURLOPT_INFILE, $req['body'] ); + if ( isset( $req['headers']['content-length'] ) ) { + curl_setopt( $ch, CURLOPT_INFILESIZE, $req['headers']['content-length'] ); + } elseif ( isset( $req['headers']['transfer-encoding'] ) && + $req['headers']['transfer-encoding'] === 'chunks' + ) { + curl_setopt( $ch, CURLOPT_UPLOAD, true ); + } else { + throw new Exception( "Missing 'Content-Length' or 'Transfer-Encoding' header." ); + } + } elseif ( $req['body'] !== '' ) { + $fp = fopen( "php://temp", "wb+" ); + fwrite( $fp, $req['body'], strlen( $req['body'] ) ); + rewind( $fp ); + curl_setopt( $ch, CURLOPT_INFILE, $fp ); + curl_setopt( $ch, CURLOPT_INFILESIZE, strlen( $req['body'] ) ); + $req['_closeHandle'] = $fp; // remember to close this later + } else { + curl_setopt( $ch, CURLOPT_INFILESIZE, 0 ); + } + curl_setopt( $ch, CURLOPT_READFUNCTION, + function ( $ch, $fd, $length ) { + $data = fread( $fd, $length ); + $len = strlen( $data ); + return $data; + } + ); + } elseif ( $req['method'] === 'POST' ) { + curl_setopt( $ch, CURLOPT_POST, 1 ); + curl_setopt( $ch, CURLOPT_POSTFIELDS, $req['body'] ); + } else { + if ( is_resource( $req['body'] ) || $req['body'] !== '' ) { + throw new Exception( "HTTP body specified for a non PUT/POST request." ); + } + $req['headers']['content-length'] = 0; + } + + $headers = array(); + foreach ( $req['headers'] as $name => $value ) { + if ( strpos( $name, ': ' ) ) { + throw new Exception( "Headers cannot have ':' in the name." ); + } + $headers[] = $name . ': ' . trim( $value ); + } + curl_setopt( $ch, CURLOPT_HTTPHEADER, $headers ); + + curl_setopt( $ch, CURLOPT_HEADERFUNCTION, + function ( $ch, $header ) use ( &$req ) { + $length = strlen( $header ); + $matches = array(); + if ( preg_match( "/^(HTTP\/1\.[01]) (\d{3}) (.*)/", $header, $matches ) ) { + $req['response']['code'] = (int)$matches[2]; + $req['response']['reason'] = trim( $matches[3] ); + return $length; + } + if ( strpos( $header, ":" ) === false ) { + return $length; + } + list( $name, $value ) = explode( ":", $header, 2 ); + $req['response']['headers'][strtolower( $name )] = trim( $value ); + return $length; + } + ); + + if ( isset( $req['stream'] ) ) { + // Don't just use CURLOPT_FILE as that might give: + // curl_setopt(): cannot represent a stream of type Output as a STDIO FILE* + // The callback here handles both normal files and php://temp handles. + curl_setopt( $ch, CURLOPT_WRITEFUNCTION, + function ( $ch, $data ) use ( &$req ) { + return fwrite( $req['stream'], $data ); + } + ); + } else { + curl_setopt( $ch, CURLOPT_WRITEFUNCTION, + function ( $ch, $data ) use ( &$req ) { + $req['response']['body'] .= $data; + return strlen( $data ); + } + ); + } + + return $ch; + } + + /** + * @return resource + */ + protected function getCurlMulti() { + if ( !$this->multiHandle ) { + $cmh = curl_multi_init(); + if ( function_exists( 'curl_multi_setopt' ) ) { // PHP 5.5 + curl_multi_setopt( $cmh, CURLMOPT_PIPELINING, (int)$this->usePipelining ); + curl_multi_setopt( $cmh, CURLMOPT_MAXCONNECTS, (int)$this->maxConnsPerHost ); + } + $this->multiHandle = $cmh; + } + return $this->multiHandle; + } + + function __destruct() { + if ( $this->multiHandle ) { + curl_multi_close( $this->multiHandle ); + } + } +} diff --git a/includes/libs/ProcessCacheLRU.php b/includes/libs/ProcessCacheLRU.php new file mode 100644 index 00000000..f988207a --- /dev/null +++ b/includes/libs/ProcessCacheLRU.php @@ -0,0 +1,148 @@ +<?php +/** + * Per-process memory cache for storing items. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * Handles per process caching of items + * @ingroup Cache + */ +class ProcessCacheLRU { + /** @var Array */ + protected $cache = array(); // (key => prop => value) + /** @var Array */ + protected $cacheTimes = array(); // (key => prop => UNIX timestamp) + + protected $maxCacheKeys; // integer; max entries + + /** + * @param $maxKeys integer Maximum number of entries allowed (min 1). + * @throws UnexpectedValueException When $maxCacheKeys is not an int or =< 0. + */ + public function __construct( $maxKeys ) { + $this->resize( $maxKeys ); + } + + /** + * Set a property field for a cache entry. + * This will prune the cache if it gets too large based on LRU. + * If the item is already set, it will be pushed to the top of the cache. + * + * @param $key string + * @param $prop string + * @param $value mixed + * @return void + */ + public function set( $key, $prop, $value ) { + if ( isset( $this->cache[$key] ) ) { + $this->ping( $key ); // push to top + } elseif ( count( $this->cache ) >= $this->maxCacheKeys ) { + reset( $this->cache ); + $evictKey = key( $this->cache ); + unset( $this->cache[$evictKey] ); + unset( $this->cacheTimes[$evictKey] ); + } + $this->cache[$key][$prop] = $value; + $this->cacheTimes[$key][$prop] = time(); + } + + /** + * Check if a property field exists for a cache entry. + * + * @param $key string + * @param $prop string + * @param $maxAge integer Ignore items older than this many seconds (since 1.21) + * @return bool + */ + public function has( $key, $prop, $maxAge = 0 ) { + if ( isset( $this->cache[$key][$prop] ) ) { + return ( $maxAge <= 0 || ( time() - $this->cacheTimes[$key][$prop] ) <= $maxAge ); + } + + return false; + } + + /** + * Get a property field for a cache entry. + * This returns null if the property is not set. + * If the item is already set, it will be pushed to the top of the cache. + * + * @param $key string + * @param $prop string + * @return mixed + */ + public function get( $key, $prop ) { + if ( isset( $this->cache[$key][$prop] ) ) { + $this->ping( $key ); // push to top + return $this->cache[$key][$prop]; + } else { + return null; + } + } + + /** + * Clear one or several cache entries, or all cache entries + * + * @param $keys string|Array + * @return void + */ + public function clear( $keys = null ) { + if ( $keys === null ) { + $this->cache = array(); + $this->cacheTimes = array(); + } else { + foreach ( (array)$keys as $key ) { + unset( $this->cache[$key] ); + unset( $this->cacheTimes[$key] ); + } + } + } + + /** + * Resize the maximum number of cache entries, removing older entries as needed + * + * @param $maxKeys integer + * @return void + */ + public function resize( $maxKeys ) { + if ( !is_int( $maxKeys ) || $maxKeys < 1 ) { + throw new UnexpectedValueException( __METHOD__ . " must be given an integer >= 1" ); + } + $this->maxCacheKeys = $maxKeys; + while ( count( $this->cache ) > $this->maxCacheKeys ) { + reset( $this->cache ); + $evictKey = key( $this->cache ); + unset( $this->cache[$evictKey] ); + unset( $this->cacheTimes[$evictKey] ); + } + } + + /** + * Push an entry to the top of the cache + * + * @param $key string + */ + protected function ping( $key ) { + $item = $this->cache[$key]; + unset( $this->cache[$key] ); + $this->cache[$key] = $item; + } +} diff --git a/includes/libs/RunningStat.php b/includes/libs/RunningStat.php new file mode 100644 index 00000000..dda5254e --- /dev/null +++ b/includes/libs/RunningStat.php @@ -0,0 +1,176 @@ +<?php +/** + * Compute running mean, variance, and extrema of a stream of numbers. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Profiler + */ + +// Needed due to PHP non-bug <https://bugs.php.net/bug.php?id=49828>. +define( 'NEGATIVE_INF', -INF ); + +/** + * Represents a running summary of a stream of numbers. + * + * RunningStat instances are accumulator-like objects that provide a set of + * continuously-updated summary statistics for a stream of numbers, without + * requiring that each value be stored. The measures it provides are the + * arithmetic mean, variance, standard deviation, and extrema (min and max); + * together they describe the central tendency and statistical dispersion of a + * set of values. + * + * One RunningStat instance can be merged into another; the resultant + * RunningStat has the state it would have had if it had accumulated each + * individual point. This allows data to be summarized in parallel and in + * stages without loss of fidelity. + * + * Based on a C++ implementation by John D. Cook: + * <http://www.johndcook.com/standard_deviation.html> + * <http://www.johndcook.com/skewness_kurtosis.html> + * + * The in-line documentation for this class incorporates content from the + * English Wikipedia articles "Variance", "Algorithms for calculating + * variance", and "Standard deviation". + * + * @since 1.23 + */ +class RunningStat implements Countable { + + /** @var int Number of samples. **/ + public $n = 0; + + /** @var float The first moment (or mean, or expected value). **/ + public $m1 = 0.0; + + /** @var float The second central moment (or variance). **/ + public $m2 = 0.0; + + /** @var float The least value in the the set. **/ + public $min = INF; + + /** @var float The most value in the set. **/ + public $max = NEGATIVE_INF; + + /** + * Count the number of accumulated values. + * @return int Number of values + */ + public function count() { + return $this->n; + } + + /** + * Add a number to the data set. + * @param int|float $x Value to add + */ + public function push( $x ) { + $x = (float) $x; + + $this->min = min( $this->min, $x ); + $this->max = max( $this->max, $x ); + + $n1 = $this->n; + $this->n += 1; + $delta = $x - $this->m1; + $delta_n = $delta / $this->n; + $this->m1 += $delta_n; + $this->m2 += $delta * $delta_n * $n1; + } + + /** + * Get the mean, or expected value. + * + * The arithmetic mean is the sum of all measurements divided by the number + * of observations in the data set. + * + * @return float Mean + */ + public function getMean() { + return $this->m1; + } + + /** + * Get the estimated variance. + * + * Variance measures how far a set of numbers is spread out. A small + * variance indicates that the data points tend to be very close to the + * mean (and hence to each other), while a high variance indicates that the + * data points are very spread out from the mean and from each other. + * + * @return float Estimated variance + */ + public function getVariance() { + if ( $this->n === 0 ) { + // The variance of the empty set is undefined. + return NAN; + } elseif ( $this->n === 1 ) { + return 0.0; + } else { + return $this->m2 / ( $this->n - 1.0 ); + } + } + + /** + * Get the estimated stanard deviation. + * + * The standard deviation of a statistical population is the square root of + * its variance. It shows shows how much variation from the mean exists. In + * addition to expressing the variability of a population, the standard + * deviation is commonly used to measure confidence in statistical conclusions. + * + * @return float Estimated standard deviation + */ + public function getStdDev() { + return sqrt( $this->getVariance() ); + } + + /** + * Merge another RunningStat instance into this instance. + * + * This instance then has the state it would have had if all the data had + * been accumulated by it alone. + * + * @param RunningStat RunningStat instance to merge into this one + */ + public function merge( RunningStat $other ) { + // If the other RunningStat is empty, there's nothing to do. + if ( $other->n === 0 ) { + return; + } + + // If this RunningStat is empty, copy values from other RunningStat. + if ( $this->n === 0 ) { + $this->n = $other->n; + $this->m1 = $other->m1; + $this->m2 = $other->m2; + $this->min = $other->min; + $this->max = $other->max; + return; + } + + $n = $this->n + $other->n; + $delta = $other->m1 - $this->m1; + $delta2 = $delta * $delta; + + $this->m1 = ( ( $this->n * $this->m1 ) + ( $other->n * $other->m1 ) ) / $n; + $this->m2 = $this->m2 + $other->m2 + ( $delta2 * $this->n * $other->n / $n ); + $this->min = min( $this->min, $other->min ); + $this->max = max( $this->max, $other->max ); + $this->n = $n; + } +} diff --git a/includes/libs/ScopedCallback.php b/includes/libs/ScopedCallback.php new file mode 100644 index 00000000..631b6519 --- /dev/null +++ b/includes/libs/ScopedCallback.php @@ -0,0 +1,73 @@ +<?php +/** + * This file deals with RAII style scoped callbacks. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Class for asserting that a callback happens when an dummy object leaves scope + * + * @since 1.21 + */ +class ScopedCallback { + /** @var callable */ + protected $callback; + + /** + * @param callable $callback + * @throws Exception + */ + public function __construct( $callback ) { + if ( !is_callable( $callback ) ) { + throw new InvalidArgumentException( "Provided callback is not valid." ); + } + $this->callback = $callback; + } + + /** + * Trigger a scoped callback and destroy it. + * This is the same is just setting it to null. + * + * @param ScopedCallback $sc + */ + public static function consume( ScopedCallback &$sc = null ) { + $sc = null; + } + + /** + * Destroy a scoped callback without triggering it + * + * @param ScopedCallback $sc + */ + public static function cancel( ScopedCallback &$sc = null ) { + if ( $sc ) { + $sc->callback = null; + } + $sc = null; + } + + /** + * Trigger the callback when this leaves scope + */ + function __destruct() { + if ( $this->callback !== null ) { + call_user_func( $this->callback ); + } + } +} diff --git a/includes/libs/ScopedPHPTimeout.php b/includes/libs/ScopedPHPTimeout.php new file mode 100644 index 00000000..d1493c30 --- /dev/null +++ b/includes/libs/ScopedPHPTimeout.php @@ -0,0 +1,84 @@ +<?php +/** + * Expansion of the PHP execution time limit feature for a function call. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Class to expand PHP execution time for a function call. + * Use this when performing changes that should not be interrupted. + * + * On construction, set_time_limit() is called and set to $seconds. + * If the client aborts the connection, PHP will continue to run. + * When the object goes out of scope, the timer is restarted, with + * the original time limit minus the time the object existed. + */ +class ScopedPHPTimeout { + protected $startTime; // float; seconds + protected $oldTimeout; // integer; seconds + protected $oldIgnoreAbort; // boolean + + protected static $stackDepth = 0; // integer + protected static $totalCalls = 0; // integer + protected static $totalElapsed = 0; // float; seconds + + /* Prevent callers in infinite loops from running forever */ + const MAX_TOTAL_CALLS = 1000000; + const MAX_TOTAL_TIME = 300; // seconds + + /** + * @param $seconds integer + */ + public function __construct( $seconds ) { + if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0 + if ( self::$totalCalls >= self::MAX_TOTAL_CALLS ) { + trigger_error( "Maximum invocations of " . __CLASS__ . " exceeded." ); + } elseif ( self::$totalElapsed >= self::MAX_TOTAL_TIME ) { + trigger_error( "Time limit within invocations of " . __CLASS__ . " exceeded." ); + } elseif ( self::$stackDepth > 0 ) { // recursion guard + trigger_error( "Resursive invocation of " . __CLASS__ . " attempted." ); + } else { + $this->oldIgnoreAbort = ignore_user_abort( true ); + $this->oldTimeout = ini_set( 'max_execution_time', $seconds ); + $this->startTime = microtime( true ); + ++self::$stackDepth; + ++self::$totalCalls; // proof against < 1us scopes + } + } + } + + /** + * Restore the original timeout. + * This does not account for the timer value on __construct(). + */ + public function __destruct() { + if ( $this->oldTimeout ) { + $elapsed = microtime( true ) - $this->startTime; + // Note: a limit of 0 is treated as "forever" + set_time_limit( max( 1, $this->oldTimeout - (int)$elapsed ) ); + // If each scoped timeout is for less than one second, we end up + // restoring the original timeout without any decrease in value. + // Thus web scripts in an infinite loop can run forever unless we + // take some measures to prevent this. Track total time and calls. + self::$totalElapsed += $elapsed; + --self::$stackDepth; + ignore_user_abort( $this->oldIgnoreAbort ); + } + } +} diff --git a/includes/libs/XmlTypeCheck.php b/includes/libs/XmlTypeCheck.php new file mode 100644 index 00000000..aca857e9 --- /dev/null +++ b/includes/libs/XmlTypeCheck.php @@ -0,0 +1,264 @@ +<?php +/** + * XML syntax and type checker. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +class XmlTypeCheck { + /** + * Will be set to true or false to indicate whether the file is + * well-formed XML. Note that this doesn't check schema validity. + */ + public $wellFormed = false; + + /** + * Will be set to true if the optional element filter returned + * a match at some point. + */ + public $filterMatch = false; + + /** + * Name of the document's root element, including any namespace + * as an expanded URL. + */ + public $rootElement = ''; + + /** + * A stack of strings containing the data of each xml element as it's processed. Append + * data to the top string of the stack, then pop off the string and process it when the + * element is closed. + */ + protected $elementData = array(); + + /** + * A stack of element names and attributes, as we process them. + */ + protected $elementDataContext = array(); + + /** + * Current depth of the data stack. + */ + protected $stackDepth = 0; + + /** + * Additional parsing options + */ + private $parserOptions = array( + 'processing_instruction_handler' => '', + ); + + /** + * @param string $input a filename or string containing the XML element + * @param callable $filterCallback (optional) + * Function to call to do additional custom validity checks from the + * SAX element handler event. This gives you access to the element + * namespace, name, attributes, and text contents. + * Filter should return 'true' to toggle on $this->filterMatch + * @param boolean $isFile (optional) indicates if the first parameter is a + * filename (default, true) or if it is a string (false) + * @param array $options list of additional parsing options: + * processing_instruction_handler: Callback for xml_set_processing_instruction_handler + */ + function __construct( $input, $filterCallback = null, $isFile = true, $options = array() ) { + $this->filterCallback = $filterCallback; + $this->parserOptions = array_merge( $this->parserOptions, $options ); + + if ( $isFile ) { + $this->validateFromFile( $input ); + } else { + $this->validateFromString( $input ); + } + } + + /** + * Alternative constructor: from filename + * + * @param string $fname the filename of an XML document + * @param callable $filterCallback (optional) + * Function to call to do additional custom validity checks from the + * SAX element handler event. This gives you access to the element + * namespace, name, and attributes, but not to text contents. + * Filter should return 'true' to toggle on $this->filterMatch + * @return XmlTypeCheck + */ + public static function newFromFilename( $fname, $filterCallback = null ) { + return new self( $fname, $filterCallback, true ); + } + + /** + * Alternative constructor: from string + * + * @param string $string a string containing an XML element + * @param callable $filterCallback (optional) + * Function to call to do additional custom validity checks from the + * SAX element handler event. This gives you access to the element + * namespace, name, and attributes, but not to text contents. + * Filter should return 'true' to toggle on $this->filterMatch + * @return XmlTypeCheck + */ + public static function newFromString( $string, $filterCallback = null ) { + return new self( $string, $filterCallback, false ); + } + + /** + * Get the root element. Simple accessor to $rootElement + * + * @return string + */ + public function getRootElement() { + return $this->rootElement; + } + + /** + * Get an XML parser with the root element handler. + * @see XmlTypeCheck::rootElementOpen() + * @return resource a resource handle for the XML parser + */ + private function getParser() { + $parser = xml_parser_create_ns( 'UTF-8' ); + // case folding violates XML standard, turn it off + xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false ); + xml_set_element_handler( $parser, array( $this, 'rootElementOpen' ), false ); + if ( $this->parserOptions['processing_instruction_handler'] ) { + xml_set_processing_instruction_handler( + $parser, + array( $this, 'processingInstructionHandler' ) + ); + } + return $parser; + } + + /** + * @param string $fname the filename + */ + private function validateFromFile( $fname ) { + $parser = $this->getParser(); + + if ( file_exists( $fname ) ) { + $file = fopen( $fname, "rb" ); + if ( $file ) { + do { + $chunk = fread( $file, 32768 ); + $ret = xml_parse( $parser, $chunk, feof( $file ) ); + if ( $ret == 0 ) { + $this->wellFormed = false; + fclose( $file ); + xml_parser_free( $parser ); + return; + } + } while ( !feof( $file ) ); + + fclose( $file ); + } + } + $this->wellFormed = true; + + xml_parser_free( $parser ); + } + + /** + * + * @param string $string the XML-input-string to be checked. + */ + private function validateFromString( $string ) { + $parser = $this->getParser(); + $ret = xml_parse( $parser, $string, true ); + xml_parser_free( $parser ); + if ( $ret == 0 ) { + $this->wellFormed = false; + return; + } + $this->wellFormed = true; + } + + /** + * @param $parser + * @param $name + * @param $attribs + */ + private function rootElementOpen( $parser, $name, $attribs ) { + $this->rootElement = $name; + + if ( is_callable( $this->filterCallback ) ) { + xml_set_element_handler( + $parser, + array( $this, 'elementOpen' ), + array( $this, 'elementClose' ) + ); + xml_set_character_data_handler( $parser, array( $this, 'elementData' ) ); + $this->elementOpen( $parser, $name, $attribs ); + } else { + // We only need the first open element + xml_set_element_handler( $parser, false, false ); + } + } + + /** + * @param $parser + * @param $name + * @param $attribs + */ + private function elementOpen( $parser, $name, $attribs ) { + $this->elementDataContext[] = array( $name, $attribs ); + $this->elementData[] = ''; + $this->stackDepth++; + } + + /** + * @param $parser + * @param $name + */ + private function elementClose( $parser, $name ) { + list( $name, $attribs ) = array_pop( $this->elementDataContext ); + $data = array_pop( $this->elementData ); + $this->stackDepth--; + + if ( call_user_func( + $this->filterCallback, + $name, + $attribs, + $data + ) ) { + // Filter hit! + $this->filterMatch = true; + } + } + + /** + * @param $parser + * @param $data + */ + private function elementData( $parser, $data ) { + // xml_set_character_data_handler breaks the data on & characters, so + // we collect any data here, and we'll run the callback in elementClose + $this->elementData[ $this->stackDepth - 1 ] .= trim( $data ); + } + + /** + * @param $parser + * @param $target + * @param $data + */ + private function processingInstructionHandler( $parser, $target, $data ) { + if ( call_user_func( $this->parserOptions['processing_instruction_handler'], $target, $data ) ) { + // Filter hit! + $this->filterMatch = true; + } + } +} diff --git a/includes/libs/jsminplus.php b/includes/libs/jsminplus.php index f250217f..ed0382cf 100644 --- a/includes/libs/jsminplus.php +++ b/includes/libs/jsminplus.php @@ -1,4 +1,5 @@ <?php +// @codingStandardsIgnoreFile File external to MediaWiki. Ignore coding conventions checks. /** * JSMinPlus version 1.4 * diff --git a/includes/libs/lessc.inc.php b/includes/libs/lessc.inc.php index 3dce961e..61ed771a 100644 --- a/includes/libs/lessc.inc.php +++ b/includes/libs/lessc.inc.php @@ -1,7 +1,7 @@ <?php - +// @codingStandardsIgnoreFile File external to MediaWiki. Ignore coding conventions checks. /** - * lessphp v0.4.0@261f1bd28f + * lessphp v0.4.0@2cc77e3c7b * http://leafo.net/lessphp * * LESS CSS compiler, adapted from http://lesscss.org @@ -847,7 +847,7 @@ class lessc { * The input is expected to be reduced. This function will not work on * things like expressions and variables. */ - protected function compileValue($value) { + public function compileValue($value) { switch ($value[0]) { case 'list': // [1] - delimiter @@ -1011,6 +1011,39 @@ class lessc { return $this->lib_rgbahex($color); } + /** + * Given an url, decide whether to output a regular link or the base64-encoded contents of the file + * + * @param array $value either an argument list (two strings) or a single string + * @return string formatted url(), either as a link or base64-encoded + */ + protected function lib_data_uri($value) { + $mime = ($value[0] === 'list') ? $value[2][0][2] : null; + $url = ($value[0] === 'list') ? $value[2][1][2][0] : $value[2][0]; + + $fullpath = $this->findImport($url); + + if($fullpath && ($fsize = filesize($fullpath)) !== false) { + // IE8 can't handle data uris larger than 32KB + if($fsize/1024 < 32) { + if(is_null($mime)) { + if(class_exists('finfo')) { // php 5.3+ + $finfo = new finfo(FILEINFO_MIME); + $mime = explode('; ', $finfo->file($fullpath)); + $mime = $mime[0]; + } elseif(function_exists('mime_content_type')) { // PHP 5.2 + $mime = mime_content_type($fullpath); + } + } + + if(!is_null($mime)) // fallback if the MIME type is still unknown + $url = sprintf('data:%s;base64,%s', $mime, base64_encode(file_get_contents($fullpath))); + } + } + + return 'url("'.$url.'")'; + } + // utility func to unquote a string protected function lib_e($arg) { switch ($arg[0]) { @@ -1234,24 +1267,44 @@ class lessc { } protected function lib_contrast($args) { - if ($args[0] != 'list' || count($args[2]) < 3) { - return array(array('color', 0, 0, 0), 0); - } + $darkColor = array('color', 0, 0, 0); + $lightColor = array('color', 255, 255, 255); + $threshold = 0.43; - list($inputColor, $darkColor, $lightColor) = $args[2]; + if ( $args[0] == 'list' ) { + $inputColor = ( isset($args[2][0]) ) ? $this->assertColor($args[2][0]) : $lightColor; + $darkColor = ( isset($args[2][1]) ) ? $this->assertColor($args[2][1]) : $darkColor; + $lightColor = ( isset($args[2][2]) ) ? $this->assertColor($args[2][2]) : $lightColor; + $threshold = ( isset($args[2][3]) ) ? $this->assertNumber($args[2][3]) : $threshold; + } + else { + $inputColor = $this->assertColor($args); + } - $inputColor = $this->assertColor($inputColor); - $darkColor = $this->assertColor($darkColor); - $lightColor = $this->assertColor($lightColor); - $hsl = $this->toHSL($inputColor); + $inputColor = $this->coerceColor($inputColor); + $darkColor = $this->coerceColor($darkColor); + $lightColor = $this->coerceColor($lightColor); - if ($hsl[3] > 50) { - return $darkColor; - } + //Figure out which is actually light and dark! + if ( $this->lib_luma($darkColor) > $this->lib_luma($lightColor) ) { + $t = $lightColor; + $lightColor = $darkColor; + $darkColor = $t; + } - return $lightColor; + $inputColor_alpha = $this->lib_alpha($inputColor); + if ( ( $this->lib_luma($inputColor) * $inputColor_alpha) < $threshold) { + return $lightColor; + } + return $darkColor; } + protected function lib_luma($color) { + $color = $this->coerceColor($color); + return (0.2126 * $color[0] / 255) + (0.7152 * $color[1] / 255) + (0.0722 * $color[2] / 255); + } + + public function assertColor($value, $error = "expected color value") { $color = $this->coerceColor($value); if (is_null($color)) $this->throwError($error); @@ -1475,8 +1528,9 @@ class lessc { list(, $name, $args) = $value; if ($name == "%") $name = "_sprintf"; + $f = isset($this->libFunctions[$name]) ? - $this->libFunctions[$name] : array($this, 'lib_'.$name); + $this->libFunctions[$name] : array($this, 'lib_'.str_replace('-', '_', $name)); if (is_callable($f)) { if ($args[0] == 'list') @@ -2338,7 +2392,7 @@ class lessc_parser { $this->throwError(); // TODO report where the block was opened - if (!is_null($this->env->parent)) + if ( !property_exists($this->env, 'parent') || !is_null($this->env->parent) ) throw new exception('parse error: unclosed block'); return $this->env; diff --git a/includes/libs/virtualrest/SwiftVirtualRESTService.php b/includes/libs/virtualrest/SwiftVirtualRESTService.php new file mode 100644 index 00000000..011dabe0 --- /dev/null +++ b/includes/libs/virtualrest/SwiftVirtualRESTService.php @@ -0,0 +1,175 @@ +<?php +/** + * Virtual HTTP service client for Swift + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Example virtual rest service for OpenStack Swift + * @TODO: caching support (APC/memcached) + * @since 1.23 + */ +class SwiftVirtualRESTService extends VirtualRESTService { + /** @var array */ + protected $authCreds; + /** @var int UNIX timestamp */ + protected $authSessionTimestamp = 0; + /** @var int UNIX timestamp */ + protected $authErrorTimestamp = null; + /** @var int */ + protected $authCachedStatus = null; + /** @var string */ + protected $authCachedReason = null; + + /** + * @param array $params Key/value map + * - swiftAuthUrl : Swift authentication server URL + * - swiftUser : Swift user used by MediaWiki (account:username) + * - swiftKey : Swift authentication key for the above user + * - swiftAuthTTL : Swift authentication TTL (seconds) + */ + public function __construct( array $params ) { + parent::__construct( $params ); + } + + /** + * @return int|bool HTTP status on cached failure + */ + protected function needsAuthRequest() { + if ( !$this->authCreds ) { + return true; + } + if ( $this->authErrorTimestamp !== null ) { + if ( ( time() - $this->authErrorTimestamp ) < 60 ) { + return $this->authCachedStatus; // failed last attempt; don't bother + } else { // actually retry this time + $this->authErrorTimestamp = null; + } + } + // Session keys expire after a while, so we renew them periodically + return ( ( time() - $this->authSessionTimestamp ) > $this->params['swiftAuthTTL'] ); + } + + protected function applyAuthResponse( array $req ) { + $this->authSessionTimestamp = 0; + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $req['response']; + if ( $rcode >= 200 && $rcode <= 299 ) { // OK + $this->authCreds = array( + 'auth_token' => $rhdrs['x-auth-token'], + 'storage_url' => $rhdrs['x-storage-url'] + ); + $this->authSessionTimestamp = time(); + return true; + } elseif ( $rcode === 403 ) { + $this->authCachedStatus = 401; + $this->authCachedReason = 'Authorization Required'; + $this->authErrorTimestamp = time(); + return false; + } else { + $this->authCachedStatus = $rcode; + $this->authCachedReason = $rdesc; + $this->authErrorTimestamp = time(); + return null; + } + } + + public function onRequests( array $reqs, Closure $idGeneratorFunc ) { + $result = array(); + $firstReq = reset( $reqs ); + if ( $firstReq && count( $reqs ) == 1 && isset( $firstReq['isAuth'] ) ) { + // This was an authentication request for work requests... + $result = $reqs; // no change + } else { + // These are actual work requests... + $needsAuth = $this->needsAuthRequest(); + if ( $needsAuth === true ) { + // These are work requests and we don't have any token to use. + // Replace the work requests with an authentication request. + $result = array( + $idGeneratorFunc() => array( + 'method' => 'GET', + 'url' => $this->params['swiftAuthUrl'] . "/v1.0", + 'headers' => array( + 'x-auth-user' => $this->params['swiftUser'], + 'x-auth-key' => $this->params['swiftKey'] ), + 'isAuth' => true, + 'chain' => $reqs + ) + ); + } elseif ( $needsAuth !== false ) { + // These are work requests and authentication has previously failed. + // It is most efficient to just give failed pseudo responses back for + // the original work requests. + foreach ( $reqs as $key => $req ) { + $req['response'] = array( + 'code' => $this->authCachedStatus, + 'reason' => $this->authCachedReason, + 'headers' => array(), + 'body' => '', + 'error' => '' + ); + $result[$key] = $req; + } + } else { + // These are work requests and we have a token already. + // Go through and mangle each request to include a token. + foreach ( $reqs as $key => $req ) { + // The default encoding treats the URL as a REST style path that uses + // forward slash as a hierarchical delimiter (and never otherwise). + // Subclasses can override this, and should be documented in any case. + $parts = array_map( 'rawurlencode', explode( '/', $req['url'] ) ); + $req['url'] = $this->authCreds['storage_url'] . '/' . implode( '/', $parts ); + $req['headers']['x-auth-token'] = $this->authCreds['auth_token']; + $result[$key] = $req; + // @TODO: add ETag/Content-Length and such as needed + } + } + } + return $result; + } + + public function onResponses( array $reqs, Closure $idGeneratorFunc ) { + $firstReq = reset( $reqs ); + if ( $firstReq && count( $reqs ) == 1 && isset( $firstReq['isAuth'] ) ) { + $result = array(); + // This was an authentication request for work requests... + if ( $this->applyAuthResponse( $firstReq ) ) { + // If it succeeded, we can subsitute the work requests back. + // Call this recursively in order to munge and add headers. + $result = $this->onRequests( $firstReq['chain'], $idGeneratorFunc ); + } else { + // If it failed, it is most efficient to just give failing + // pseudo-responses back for the actual work requests. + foreach ( $firstReq['chain'] as $key => $req ) { + $req['response'] = array( + 'code' => $this->authCachedStatus, + 'reason' => $this->authCachedReason, + 'headers' => array(), + 'body' => '', + 'error' => '' + ); + $result[$key] = $req; + } + } + } else { + $result = $reqs; // no change + } + return $result; + } +} diff --git a/includes/libs/virtualrest/VirtualRESTService.php b/includes/libs/virtualrest/VirtualRESTService.php new file mode 100644 index 00000000..05c2afc1 --- /dev/null +++ b/includes/libs/virtualrest/VirtualRESTService.php @@ -0,0 +1,107 @@ +<?php +/** + * Virtual HTTP service client + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Virtual HTTP service instance that can be mounted on to a VirtualRESTService + * + * Sub-classes manage the logic of either: + * - a) Munging virtual HTTP request arrays to have qualified URLs and auth headers + * - b) Emulating the execution of virtual HTTP requests (e.g. brokering) + * + * Authentication information can be cached in instances of the class for performance. + * Such information should also be cached locally on the server and auth requests should + * have reasonable timeouts. + * + * @since 1.23 + */ +abstract class VirtualRESTService { + /** @var array Key/value map */ + protected $params = array(); + + /** + * @param array $params Key/value map + */ + public function __construct( array $params ) { + $this->params = $params; + } + + /** + * Prepare virtual HTTP(S) requests (for this service) for execution + * + * This method should mangle any of the $reqs entry fields as needed: + * - url : munge the URL to have an absolute URL with a protocol + * and encode path components as needed by the backend [required] + * - query : include any authentication signatures/parameters [as needed] + * - headers : include any authentication tokens/headers [as needed] + * + * The incoming URL parameter will be relative to the service mount point. + * + * This method can also remove some of the requests as well as add new ones + * (using $idGenerator to set each of the entries' array keys). For any existing + * or added request, the 'response' array can be filled in, which will prevent the + * client from executing it. If an original request is removed, at some point it + * must be added back (with the same key) in onRequests() or onResponses(); + * it's reponse may be filled in as with other requests. + * + * @param array $reqs Map of Virtual HTTP request arrays + * @param Closure $idGeneratorFunc Method to generate unique keys for new requests + * @return array Modified HTTP request array map + */ + public function onRequests( array $reqs, Closure $idGeneratorFunc ) { + $result = array(); + foreach ( $reqs as $key => $req ) { + // The default encoding treats the URL as a REST style path that uses + // forward slash as a hierarchical delimiter (and never otherwise). + // Subclasses can override this, and should be documented in any case. + $parts = array_map( 'rawurlencode', explode( '/', $req['url'] ) ); + $req['url'] = $this->params['baseUrl'] . '/' . implode( '/', $parts ); + $result[$key] = $req; + } + return $result; + } + + /** + * Mangle or replace virtual HTTP(S) requests which have been responded to + * + * This method may mangle any of the $reqs entry 'response' fields as needed: + * - code : perform any code normalization [as needed] + * - reason : perform any reason normalization [as needed] + * - headers : perform any header normalization [as needed] + * + * This method can also remove some of the requests as well as add new ones + * (using $idGenerator to set each of the entries' array keys). For any existing + * or added request, the 'response' array can be filled in, which will prevent the + * client from executing it. If an original request is removed, at some point it + * must be added back (with the same key) in onRequests() or onResponses(); + * it's reponse may be filled in as with other requests. All requests added to $reqs + * will be passed through onRequests() to handle any munging required as normal. + * + * The incoming URL parameter will be relative to the service mount point. + * + * @param array $reqs Map of Virtual HTTP request arrays with 'response' set + * @param Closure $idGeneratorFunc Method to generate unique keys for new requests + * @return array Modified HTTP request array map + */ + public function onResponses( array $reqs, Closure $idGeneratorFunc ) { + return $reqs; + } +} diff --git a/includes/libs/virtualrest/VirtualRESTServiceClient.php b/includes/libs/virtualrest/VirtualRESTServiceClient.php new file mode 100644 index 00000000..2d21d3cf --- /dev/null +++ b/includes/libs/virtualrest/VirtualRESTServiceClient.php @@ -0,0 +1,289 @@ +<?php +/** + * Virtual HTTP service client + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Virtual HTTP service client loosely styled after a Virtual File System + * + * Services can be mounted on path prefixes so that virtual HTTP operations + * against sub-paths will map to those services. Operations can actually be + * done using HTTP messages over the wire or may simple be emulated locally. + * + * Virtual HTTP request maps are arrays that use the following format: + * - method : GET/HEAD/PUT/POST/DELETE + * - url : HTTP/HTTPS URL or virtual service path with a registered prefix + * - query : <query parameter field/value associative array> (uses RFC 3986) + * - headers : <header name/value associative array> + * - body : source to get the HTTP request body from; + * this can simply be a string (always), a resource for + * PUT requests, and a field/value array for POST request; + * array bodies are encoded as multipart/form-data and strings + * use application/x-www-form-urlencoded (headers sent automatically) + * - stream : resource to stream the HTTP response body to + * Request maps can use integer index 0 instead of 'method' and 1 instead of 'url'. + * + * @author Aaron Schulz + * @since 1.23 + */ +class VirtualRESTServiceClient { + /** @var MultiHttpClient */ + protected $http; + /** @var Array Map of (prefix => VirtualRESTService) */ + protected $instances = array(); + + const VALID_MOUNT_REGEX = '#^/[0-9a-z]+/([0-9a-z]+/)*$#'; + + /** + * @param MultiHttpClient $http + */ + public function __construct( MultiHttpClient $http ) { + $this->http = $http; + } + + /** + * Map a prefix to service handler + * + * @param string $prefix Virtual path + * @param VirtualRESTService $instance + */ + public function mount( $prefix, VirtualRESTService $instance ) { + if ( !preg_match( self::VALID_MOUNT_REGEX, $prefix ) ) { + throw new UnexpectedValueException( "Invalid service mount point '$prefix'." ); + } elseif ( isset( $this->instances[$prefix] ) ) { + throw new UnexpectedValueException( "A service is already mounted on '$prefix'." ); + } + $this->instances[$prefix] = $instance; + } + + /** + * Unmap a prefix to service handler + * + * @param string $prefix Virtual path + */ + public function unmount( $prefix ) { + if ( !preg_match( self::VALID_MOUNT_REGEX, $prefix ) ) { + throw new UnexpectedValueException( "Invalid service mount point '$prefix'." ); + } elseif ( !isset( $this->instances[$prefix] ) ) { + throw new UnexpectedValueException( "No service is mounted on '$prefix'." ); + } + unset( $this->instances[$prefix] ); + } + + /** + * Get the prefix and service that a virtual path is serviced by + * + * @param string $path + * @return array (prefix,VirtualRESTService) or (null,null) if none found + */ + public function getMountAndService( $path ) { + $cmpFunc = function( $a, $b ) { + $al = substr_count( $a, '/' ); + $bl = substr_count( $b, '/' ); + if ( $al === $bl ) { + return 0; // should not actually happen + } + return ( $al < $bl ) ? 1 : -1; // largest prefix first + }; + + $matches = array(); // matching prefixes (mount points) + foreach ( $this->instances as $prefix => $service ) { + if ( strpos( $path, $prefix ) === 0 ) { + $matches[] = $prefix; + } + } + usort( $matches, $cmpFunc ); + + // Return the most specific prefix and corresponding service + return isset( $matches[0] ) + ? array( $matches[0], $this->instances[$matches[0]] ) + : array( null, null ); + } + + /** + * Execute a virtual HTTP(S) request + * + * This method returns a response map of: + * - code : HTTP response code or 0 if there was a serious cURL error + * - reason : HTTP response reason (empty if there was a serious cURL error) + * - headers : <header name/value associative array> + * - body : HTTP response body or resource (if "stream" was set) + * - err : Any cURL error string + * The map also stores integer-indexed copies of these values. This lets callers do: + * <code> + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $client->run( $req ); + * </code> + * @param array $req Virtual HTTP request array + * @return array Response array for request + */ + public function run( array $req ) { + $req = $this->runMulti( array( $req ) ); + return $req[0]['response']; + } + + /** + * Execute a set of virtual HTTP(S) requests concurrently + * + * A map of requests keys to response maps is returned. Each response map has: + * - code : HTTP response code or 0 if there was a serious cURL error + * - reason : HTTP response reason (empty if there was a serious cURL error) + * - headers : <header name/value associative array> + * - body : HTTP response body or resource (if "stream" was set) + * - err : Any cURL error string + * The map also stores integer-indexed copies of these values. This lets callers do: + * <code> + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $responses[0]; + * </code> + * + * @param array $req Map of Virtual HTTP request arrays + * @return array $reqs Map of corresponding response values with the same keys/order + */ + public function runMulti( array $reqs ) { + foreach ( $reqs as $index => &$req ) { + if ( isset( $req[0] ) ) { + $req['method'] = $req[0]; // short-form + unset( $req[0] ); + } + if ( isset( $req[1] ) ) { + $req['url'] = $req[1]; // short-form + unset( $req[1] ); + } + $req['chain'] = array(); // chain or list of replaced requests + } + unset( $req ); // don't assign over this by accident + + $curUniqueId = 0; + $armoredIndexMap = array(); // (original index => new index) + + $doneReqs = array(); // (index => request) + $executeReqs = array(); // (index => request) + $replaceReqsByService = array(); // (prefix => index => request) + $origPending = array(); // (index => 1) for original requests + + foreach ( $reqs as $origIndex => $req ) { + // Re-index keys to consecutive integers (they will be swapped back later) + $index = $curUniqueId++; + $armoredIndexMap[$origIndex] = $index; + $origPending[$index] = 1; + if ( preg_match( '#^(http|ftp)s?://#', $req['url'] ) ) { + // Absolute FTP/HTTP(S) URL, run it as normal + $executeReqs[$index] = $req; + } else { + // Must be a virtual HTTP URL; resolve it + list( $prefix, $service ) = $this->getMountAndService( $req['url'] ); + if ( !$service ) { + throw new UnexpectedValueException( "Path '{$req['url']}' has no service." ); + } + // Set the URL to the mount-relative portion + $req['url'] = substr( $req['url'], strlen( $prefix ) ); + $replaceReqsByService[$prefix][$index] = $req; + } + } + + // Function to get IDs that won't collide with keys in $armoredIndexMap + $idFunc = function() use ( &$curUniqueId ) { + return $curUniqueId++; + }; + + $rounds = 0; + do { + if ( ++$rounds > 5 ) { // sanity + throw new Exception( "Too many replacement rounds detected. Aborting." ); + } + // Resolve the virtual URLs valid and qualified HTTP(S) URLs + // and add any required authentication headers for the backend. + // Services can also replace requests with new ones, either to + // defer the original or to set a proxy response to the original. + $newReplaceReqsByService = array(); + foreach ( $replaceReqsByService as $prefix => $servReqs ) { + $service = $this->instances[$prefix]; + foreach ( $service->onRequests( $servReqs, $idFunc ) as $index => $req ) { + // Services use unique IDs for replacement requests + if ( isset( $servReqs[$index] ) || isset( $origPending[$index] ) ) { + // A current or original request which was not modified + } else { + // Replacement requests with pre-set responses should not execute + $newReplaceReqsByService[$prefix][$index] = $req; + } + if ( isset( $req['response'] ) ) { + // Replacement requests with pre-set responses should not execute + unset( $executeReqs[$index] ); + unset( $origPending[$index] ); + $doneReqs[$index] = $req; + } else { + // Original or mangled request included + $executeReqs[$index] = $req; + } + } + } + // Update index of requests to inspect for replacement + $replaceReqsByService = $newReplaceReqsByService; + // Run the actual work HTTP requests + foreach ( $this->http->runMulti( $executeReqs ) as $index => $ranReq ) { + $doneReqs[$index] = $ranReq; + unset( $origPending[$index] ); + } + $executeReqs = array(); + // Services can also replace requests with new ones, either to + // defer the original or to set a proxy response to the original. + // Any replacement requests executed above will need to be replaced + // with new requests (eventually the original). The responses can be + // forced instead of having the request sent over the wire. + $newReplaceReqsByService = array(); + foreach ( $replaceReqsByService as $prefix => $servReqs ) { + $service = $this->instances[$prefix]; + // Only the request copies stored in $doneReqs actually have the response + $servReqs = array_intersect_key( $doneReqs, $servReqs ); + foreach ( $service->onResponses( $servReqs, $idFunc ) as $index => $req ) { + // Services use unique IDs for replacement requests + if ( isset( $servReqs[$index] ) || isset( $origPending[$index] ) ) { + // A current or original request which was not modified + } else { + // Replacement requests with pre-set responses should not execute + $newReplaceReqsByService[$prefix][$index] = $req; + } + if ( isset( $req['response'] ) ) { + // Replacement requests with pre-set responses should not execute + unset( $origPending[$index] ); + $doneReqs[$index] = $req; + } else { + // Update the request in case it was mangled + $executeReqs[$index] = $req; + } + } + } + // Update index of requests to inspect for replacement + $replaceReqsByService = $newReplaceReqsByService; + } while ( count( $origPending ) ); + + $responses = array(); + // Update $reqs to include 'response' and normalized request 'headers'. + // This maintains the original order of $reqs. + foreach ( $reqs as $origIndex => $req ) { + $index = $armoredIndexMap[$origIndex]; + if ( !isset( $doneReqs[$index] ) ) { + throw new UnexpectedValueException( "Response for request '$index' is NULL." ); + } + $responses[$origIndex] = $doneReqs[$index]['response']; + } + + return $responses; + } +} |