diff options
Diffstat (limited to 'extensions/LocalisationUpdate/LocalisationUpdate.class.php')
| -rw-r--r-- | extensions/LocalisationUpdate/LocalisationUpdate.class.php | 588 |
1 files changed, 588 insertions, 0 deletions
diff --git a/extensions/LocalisationUpdate/LocalisationUpdate.class.php b/extensions/LocalisationUpdate/LocalisationUpdate.class.php new file mode 100644 index 00000000..39368b7c --- /dev/null +++ b/extensions/LocalisationUpdate/LocalisationUpdate.class.php @@ -0,0 +1,588 @@ +<?php + +/** + * Class for localization updates. + * + * TODO: refactor code to remove duplication + */ +class LocalisationUpdate { + + private static $newHashes = null; + private static $filecache = array(); + + /** + * LocalisationCacheRecache hook handler. + * + * @param $lc LocalisationCache + * @param $langcode String + * @param $cache Array + * + * @return true + */ + public static function onRecache( LocalisationCache $lc, $langcode, array &$cache ) { + // Handle fallback sequence and load all fallback messages from the cache + $codeSequence = array_merge( array( $langcode ), $cache['fallbackSequence'] ); + // Iterate over the fallback sequence in reverse, otherwise the fallback + // language will override the requested language + foreach ( array_reverse( $codeSequence ) as $code ) { + if ( $code == 'en' ) { + // Skip English, otherwise we end up trying to read + // the nonexistent cache file for en a couple hundred times + continue; + } + + $cache['messages'] = array_merge( + $cache['messages'], + self::readFile( $code ) + ); + + $cache['deps'][] = new FileDependency( + self::filename( $code ) + ); + } + + return true; + } + + /** + * Called from the cronjob to fetch new messages from SVN. + * + * @param $options Array + * + * @return true + */ + public static function updateMessages( array $options ) { + global $wgLocalisationUpdateDirectory, $wgLocalisationUpdateCoreURL, + $wgLocalisationUpdateExtensionURL, $wgLocalisationUpdateSVNURL; + + $verbose = !isset( $options['quiet'] ); + $all = isset( $options['all'] ); + $skipCore = isset( $options['skip-core'] ); + $skipExtensions = isset( $options['skip-extensions'] ); + + if( isset( $options['outdir'] ) ) { + $wgLocalisationUpdateDirectory = $options['outdir']; + } + + $coreUrl = $wgLocalisationUpdateCoreURL; + $extUrl = $wgLocalisationUpdateExtensionURL; + + // Some ugly BC + if ( $wgLocalisationUpdateSVNURL ) { + $coreUrl = $wgLocalisationUpdateSVNURL . '/phase3/$2'; + $extUrl = $wgLocalisationUpdateSVNURL . '/extensions/$1/$2'; + } + + // Some more ugly BC + if ( isset( $options['svnurl'] ) ) { + $coreUrl = $options['svnurl'] . '/phase3/$2'; + $extUrl = $options['svnurl'] . '/extensions/$1/$2'; + } + + $result = 0; + + // Update all MW core messages. + if( !$skipCore ) { + $result = self::updateMediawikiMessages( $verbose, $coreUrl ); + } + + // Update all Extension messages. + if( !$skipExtensions ) { + if( $all ) { + global $IP; + $extFiles = array(); + + // Look in extensions/ for all available items... + // TODO: add support for $wgExtensionAssetsPath + $dirs = new RecursiveDirectoryIterator( "$IP/extensions/" ); + + // I ain't kidding... RecursiveIteratorIterator. + foreach( new RecursiveIteratorIterator( $dirs ) as $pathname => $item ) { + $filename = basename( $pathname ); + $matches = array(); + if( preg_match( '/^(.*)\.i18n\.php$/', $filename, $matches ) ) { + $group = $matches[1]; + $extFiles[$group] = $pathname; + } + } + } else { + global $wgExtensionMessagesFiles; + $extFiles = $wgExtensionMessagesFiles; + } + foreach ( $extFiles as $extension => $locFile ) { + $result += self::updateExtensionMessages( $locFile, $extension, $verbose, $extUrl ); + } + } + + self::writeHashes(); + + // And output the result! + self::myLog( "Updated {$result} messages in total" ); + self::myLog( "Done" ); + + return true; + } + + /** + * Update Extension Messages. + * + * @param $file String + * @param $extension String + * @param $verbose Boolean + * + * @return Integer: the amount of updated messages + */ + public static function updateExtensionMessages( $file, $extension, $verbose, $extUrl ) { + $match = array(); + $ok = preg_match( '~^.*/extensions/([^/]+)/(.*)$~U', $file, $match ); + if ( !$ok ) { + return null; + } + + $ext = $match[1]; + $extFile = $match[2]; + + // Create a full path. + $svnfile = str_replace( + array( '$1', '$2' ), + array( $ext, $extFile ), + $extUrl + ); + + // Compare the 2 files. + $result = self::compareExtensionFiles( $extension, $svnfile, $file, $verbose ); + + return $result; + } + + /** + * Update the MediaWiki Core Messages. + * + * @param $verbose Boolean + * + * @return Integer: the amount of updated messages + */ + public static function updateMediawikiMessages( $verbose, $coreUrl ) { + // Find the changed English strings (as these messages won't be updated in ANY language). + $localUrl = Language::getMessagesFileName( 'en' ); + $repoUrl = str_replace( '$2', 'languages/messages/MessagesEn.php', $coreUrl ); + $changedEnglishStrings = self::compareFiles( $repoUrl, $localUrl, $verbose ); + + // Count the changes. + $changedCount = 0; + + $languages = Language::fetchLanguageNames( null, 'mwfile' ); + foreach ( array_keys( $languages ) as $code ) { + $localUrl = Language::getMessagesFileName( $code ); + // Not prefixed with $IP + $filename = Language::getFilename( 'languages/messages/Messages', $code ); + $repoUrl = str_replace( '$2', $filename, $coreUrl ); + + // Compare the files. + $changedCount += self::compareFiles( $repoUrl, $localUrl, $verbose, $changedEnglishStrings, false, true ); + } + + // Log some nice info. + self::myLog( "{$changedCount} MediaWiki messages are updated" ); + + return $changedCount; + } + + /** + * Removes all unneeded content from a file and returns it. + * + * @param $contents String + * + * @return String + */ + public static function cleanupFile( $contents ) { + // We don't need any PHP tags. + $contents = strtr( $contents, + array( + '<?php' => '', + '?' . '>' => '' + ) + ); + + $results = array(); + + // And we only want message arrays. + preg_match_all( '/\$messages(.*\s)*?\);/', $contents, $results ); + + // But we want them all in one string. + if( !empty( $results[0] ) && is_array( $results[0] ) ) { + $contents = implode( "\n\n", $results[0] ); + } else { + $contents = ''; + } + + // And we hate the windows vs linux linebreaks. + $contents = preg_replace( '/\r\n?/', "\n", $contents ); + + return $contents; + } + + /** + * Returns the contents of a file or false on failiure. + * + * @param $file String + * + * @return string or false + */ + public static function getFileContents( $file ) { + global $wgLocalisationUpdateRetryAttempts; + + $attempts = 0; + $filecontents = ''; + + // Use cURL to get the SVN contents. + if ( preg_match( "/^http/", $file ) ) { + while( !$filecontents && $attempts <= $wgLocalisationUpdateRetryAttempts ) { + if( $attempts > 0 ) { + $delay = 1; + self::myLog( 'Failed to download ' . $file . "; retrying in ${delay}s..." ); + sleep( $delay ); + } + + $filecontents = Http::get( $file ); + $attempts++; + } + if ( !$filecontents ) { + self::myLog( 'Cannot get the contents of ' . $file . ' (curl)' ); + return false; + } + } else {// otherwise try file_get_contents + if ( !( $filecontents = file_get_contents( $file ) ) ) { + self::myLog( 'Cannot get the contents of ' . $file ); + return false; + } + } + + return $filecontents; + } + + /** + * Returns a pair of arrays containing the messages from two files, or + * a pair of nulls if the files don't need to be checked. + * + * @param $tag String + * @param $file1 String + * @param $file2 String + * @param $verbose Boolean + * @param $alwaysGetResult Boolean + * + * @return array + */ + public static function loadFilesToCompare( $tag, $file1, $file2, $verbose, $alwaysGetResult = true ) { + $file1contents = self::getFileContents( $file1 ); + if ( $file1contents === false || $file1contents === '' ) { + self::myLog( "Failed to read $file1" ); + return array( null, null ); + } + + $file2contents = self::getFileContents( $file2 ); + if ( $file2contents === false || $file2contents === '' ) { + self::myLog( "Failed to read $file2" ); + return array( null, null ); + } + + // Only get the part we need. + $file1contents = self::cleanupFile( $file1contents ); + $file1hash = md5( $file1contents ); + + $file2contents = self::cleanupFile( $file2contents ); + $file2hash = md5( $file2contents ); + + // Check if the file has changed since our last update. + if ( !$alwaysGetResult ) { + if ( !self::checkHash( $file1, $file1hash ) && !self::checkHash( $file2, $file2hash ) ) { + self::myLog( "Skipping {$tag} since the files haven't changed since our last update", $verbose ); + return array( null, null ); + } + } + + // Get the array with messages. + $messages1 = self::parsePHP( $file1contents, 'messages' ); + if ( !is_array( $messages1 ) ) { + if ( strpos( $file1contents, '$messages' ) === false ) { + // No $messages array. This happens for some languages that only have a fallback + $messages1 = array(); + } else { + // Broken file? Report and bail + self::myLog( "Failed to parse $file1" ); + return array( null, null ); + } + } + + $messages2 = self::parsePHP( $file2contents, 'messages' ); + if ( !is_array( $messages2 ) ) { + // Broken file? Report and bail + if ( strpos( $file2contents, '$messages' ) === false ) { + // No $messages array. This happens for some languages that only have a fallback + $messages2 = array(); + } else { + self::myLog( "Failed to parse $file2" ); + return array( null, null ); + } + } + + self::saveHash( $file1, $file1hash ); + self::saveHash( $file2, $file2hash ); + + return array( $messages1, $messages2 ); + } + + /** + * Compare new and old messages lists, and optionally save the new + * messages if they've changed. + * + * @param $langcode String + * @param $old_messages Array + * @param $new_messages Array + * @param $verbose Boolean + * @param $forbiddenKeys Array + * @param $saveResults Boolean + * + * @return array|int + */ + private static function compareLanguageArrays( $langcode, $old_messages, $new_messages, $verbose, $forbiddenKeys, $saveResults ) { + // Get the currently-cached messages, if any + $cur_messages = self::readFile( $langcode ); + + // Update the messages lists with the cached messages + $old_messages = array_merge( $old_messages, $cur_messages ); + $new_messages = array_merge( $cur_messages, $new_messages ); + + // Use the old/cached version for any forbidden keys + if ( count( $forbiddenKeys ) ) { + $new_messages = array_merge( + array_diff_key( $new_messages, $forbiddenKeys ), + array_intersect_key( $old_messages, $forbiddenKeys ) + ); + } + + + if ( $saveResults ) { + // If anything has changed from the saved version, save the new version + if ( $new_messages != $cur_messages ) { + // Count added, updated, and deleted messages: + // diff( new, cur ) gives added + updated, and diff( cur, new ) + // gives deleted + updated. + $changed = array_diff_assoc( $new_messages, $cur_messages ) + + array_diff_assoc( $cur_messages, $new_messages ); + $updates = count( $changed ); + self::myLog( "{$updates} messages updated for {$langcode}.", $verbose ); + self::writeFile( $langcode, $new_messages ); + } else { + $updates = 0; + } + return $updates; + } else { + // Find all deleted or changed messages + $changedStrings = array_diff_assoc( $old_messages, $new_messages ); + return $changedStrings; + } + } + + /** + * Returns an array containing the differences between the files. + * + * @param $newfile String + * @param $oldfile String + * @param $verbose Boolean + * @param $forbiddenKeys Array + * @param $alwaysGetResult Boolean + * @param $saveResults Boolean + * + * @return array|int + */ + public static function compareFiles( $newfile, $oldfile, $verbose, array $forbiddenKeys = array(), $alwaysGetResult = true, $saveResults = false ) { + // Get the languagecode. + $langcode = Language::getCodeFromFileName( $newfile, 'Messages' ); + + list( $new_messages, $old_messages ) = self::loadFilesToCompare( + $langcode, $newfile, $oldfile, $verbose, $alwaysGetResult + ); + if ( $new_messages === null || $old_messages === null ) { + return $saveResults ? 0 : array(); + } + + return self::compareLanguageArrays( $langcode, $old_messages, $new_messages, $verbose, $forbiddenKeys, $saveResults ); + } + + /** + * + * @param $extension String + * @param $newfile String + * @param $oldfile String + * @param $verbose Boolean + * @param $alwaysGetResult Boolean + * @param $saveResults Boolean + * + * @return Integer: the amount of updated messages + */ + public static function compareExtensionFiles( $extension, $newfile, $oldfile, $verbose ) { + list( $new_messages, $old_messages ) = self::loadFilesToCompare( + $extension, $newfile, $oldfile, $verbose, false + ); + if ( $new_messages === null || $old_messages === null ) { + return 0; + } + + // Update counter. + $updates = 0; + + if ( empty( $new_messages['en'] ) ) { + $new_messages['en'] = array(); + } + + if ( empty( $old_messages['en'] ) ) { + $old_messages['en'] = array(); + } + + // Find the changed english strings. + $forbiddenKeys = self::compareLanguageArrays( 'en', $old_messages['en'], $new_messages['en'], $verbose, array(), false ); + + // Do an update for each language. + foreach ( $new_messages as $language => $messages ) { + if ( $language == 'en' ) { // Skip english. + continue; + } + + if ( !isset( $old_messages[$language] ) ) { + $old_messages[$language] = array(); + } + + $updates += self::compareLanguageArrays( $language, $old_messages[$language], $messages, $verbose, $forbiddenKeys, true ); + } + + // And log some stuff. + self::myLog( "Updated " . $updates . " messages for the '{$extension}' extension", $verbose ); + + return $updates; + } + + /** + * Checks whether a messages file has a certain hash. + * + * TODO: Swap return values, this is insane + * + * @param $file string Filename + * @param $hash string Hash + * + * @return bool True if $file does NOT have hash $hash, false if it does + */ + public static function checkHash( $file, $hash ) { + $hashes = self::readFile( 'hashes' ); + return @$hashes[$file] !== $hash; + } + + /** + * @param $file + * @param $hash + */ + public static function saveHash( $file, $hash ) { + if ( is_null( self::$newHashes ) ) { + self::$newHashes = self::readFile( 'hashes' ); + } + + self::$newHashes[$file] = $hash; + } + + public static function writeHashes() { + self::writeFile( 'hashes', self::$newHashes ); + } + + /** + * Logs a message. + * + * @param $log String + * @param bool $verbose + */ + public static function myLog( $log, $verbose = true ) { + if ( !$verbose ) { + return; + } + if ( isset( $_SERVER ) && array_key_exists( 'REQUEST_METHOD', $_SERVER ) ) { + wfDebug( $log . "\n" ); + } else { + print( $log . "\n" ); + } + } + + /** + * @param $php + * @param $varname + * @return bool|array + */ + public static function parsePHP( $php, $varname ) { + try { + $reader = new QuickArrayReader("<?php $php"); + return $reader->getVar( $varname ); + } catch( Exception $e ) { + self::myLog( "Failed to read file: " . $e ); + return false; + } + } + + /** + * @param $lang + * @return string + * @throws MWException + */ + public static function filename( $lang ) { + global $wgLocalisationUpdateDirectory, $wgCacheDirectory; + + $dir = $wgLocalisationUpdateDirectory ? + $wgLocalisationUpdateDirectory : + $wgCacheDirectory; + + if ( !$dir ) { + throw new MWException( 'No cache directory configured' ); + } + + return "$dir/l10nupdate-$lang.cache"; + } + + /** + * @param $lang + * @return mixed + */ + public static function readFile( $lang ) { + if ( !isset( self::$filecache[$lang] ) ) { + $file = self::filename( $lang ); + $contents = @file_get_contents( $file ); + + if ( $contents === false ) { + wfDebug( "Failed to read file '$file'\n" ); + $retval = array(); + } else { + $retval = unserialize( $contents ); + + if ( $retval === false ) { + wfDebug( "Corrupted data in file '$file'\n" ); + $retval = array(); + } + } + self::$filecache[$lang] = $retval; + } + + return self::$filecache[$lang]; + } + + /** + * @param $lang + * @param $var + * @throws MWException + */ + public static function writeFile( $lang, $var ) { + $file = self::filename( $lang ); + + if ( !@file_put_contents( $file, serialize( $var ) ) ) { + throw new MWException( "Failed to write to file '$file'" ); + } + + self::$filecache[$lang] = $var; + } + +} |