diff options
Diffstat (limited to 'includes/normal/UtfNormalGenerate.php')
| -rw-r--r-- | includes/normal/UtfNormalGenerate.php | 250 |
1 files changed, 0 insertions, 250 deletions
diff --git a/includes/normal/UtfNormalGenerate.php b/includes/normal/UtfNormalGenerate.php deleted file mode 100644 index f57aa6cc..00000000 --- a/includes/normal/UtfNormalGenerate.php +++ /dev/null @@ -1,250 +0,0 @@ -<?php -/** - * This script generates UniNormalData.inc from the Unicode Character Database - * and supplementary files. - * - * Copyright (C) 2004 Brion Vibber <brion@pobox.com> - * https://www.mediawiki.org/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * http://www.gnu.org/copyleft/gpl.html - * - * @file - * @ingroup UtfNormal - */ - -if ( PHP_SAPI != 'cli' ) { - die( "Run me from the command line please.\n" ); -} - -require_once 'UtfNormalDefines.php'; -require_once 'UtfNormalUtil.php'; - -$in = fopen( "DerivedNormalizationProps.txt", "rt" ); -if ( !$in ) { - print "Can't open DerivedNormalizationProps.txt for reading.\n"; - print "If necessary, fetch this file from the internet:\n"; - print "http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt\n"; - exit( -1 ); -} -print "Initializing normalization quick check tables...\n"; -$checkNFC = array(); -while ( false !== ( $line = fgets( $in ) ) ) { - $matches = array(); - if ( preg_match( - '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', - $line, - $matches ) - ) { - list( $junk, $first, $last, $prop, $value ) = $matches; - #print "$first $last $prop $value\n"; - if ( !$last ) { - $last = $first; - } - - $lastInDecimal = hexdec( $last ); - for ( $i = hexdec( $first ); $i <= $lastInDecimal; $i++ ) { - $char = codepointToUtf8( $i ); - $checkNFC[$char] = $value; - } - } -} -fclose( $in ); - -$in = fopen( "CompositionExclusions.txt", "rt" ); -if ( !$in ) { - print "Can't open CompositionExclusions.txt for reading.\n"; - print "If necessary, fetch this file from the internet:\n"; - print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n"; - exit( -1 ); -} -$exclude = array(); -while ( false !== ( $line = fgets( $in ) ) ) { - if ( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) { - $codepoint = $matches[1]; - $source = codepointToUtf8( hexdec( $codepoint ) ); - $exclude[$source] = true; - } -} -fclose( $in ); - -$in = fopen( "UnicodeData.txt", "rt" ); -if ( !$in ) { - print "Can't open UnicodeData.txt for reading.\n"; - print "If necessary, fetch this file from the internet:\n"; - print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n"; - exit( -1 ); -} - -$compatibilityDecomp = array(); -$canonicalDecomp = array(); -$canonicalComp = array(); -$combiningClass = array(); -$total = 0; -$compat = 0; -$canon = 0; - -print "Reading character definitions...\n"; -while ( false !== ( $line = fgets( $in ) ) ) { - $columns = explode( ';', $line ); - $codepoint = $columns[0]; - $name = $columns[1]; - $canonicalCombiningClass = $columns[3]; - $decompositionMapping = $columns[5]; - - $source = codepointToUtf8( hexdec( $codepoint ) ); - - if ( $canonicalCombiningClass != 0 ) { - $combiningClass[$source] = intval( $canonicalCombiningClass ); - } - - if ( $decompositionMapping === '' ) continue; - if ( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) { - # Compatibility decomposition - $canonical = false; - $decompositionMapping = $matches[2]; - $compat++; - } else { - $canonical = true; - $canon++; - } - $total++; - $dest = hexSequenceToUtf8( $decompositionMapping ); - - $compatibilityDecomp[$source] = $dest; - if ( $canonical ) { - $canonicalDecomp[$source] = $dest; - if ( empty( $exclude[$source] ) ) { - $canonicalComp[$dest] = $source; - } - } - #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n"; -} -fclose( $in ); - -print "Recursively expanding canonical mappings...\n"; -$changed = 42; -$pass = 1; -while ( $changed > 0 ) { - print "pass $pass\n"; - $changed = 0; - foreach ( $canonicalDecomp as $source => $dest ) { - $newDest = preg_replace_callback( - '/([\xc0-\xff][\x80-\xbf]+)/', - 'callbackCanonical', - $dest ); - if ( $newDest === $dest ) continue; - $changed++; - $canonicalDecomp[$source] = $newDest; - } - $pass++; -} - -print "Recursively expanding compatibility mappings...\n"; -$changed = 42; -$pass = 1; -while ( $changed > 0 ) { - print "pass $pass\n"; - $changed = 0; - foreach ( $compatibilityDecomp as $source => $dest ) { - $newDest = preg_replace_callback( - '/([\xc0-\xff][\x80-\xbf]+)/', - 'callbackCompat', - $dest ); - if ( $newDest === $dest ) continue; - $changed++; - $compatibilityDecomp[$source] = $newDest; - } - $pass++; -} - -print "$total decomposition mappings ($canon canonical, $compat compatibility)\n"; - -$out = fopen( "UtfNormalData.inc", "wt" ); -if ( $out ) { - $serCombining = escapeSingleString( serialize( $combiningClass ) ); - $serComp = escapeSingleString( serialize( $canonicalComp ) ); - $serCanon = escapeSingleString( serialize( $canonicalDecomp ) ); - $serCheckNFC = escapeSingleString( serialize( $checkNFC ) ); - $outdata = "<" . "?php -/** - * This file was automatically generated -- do not edit! - * Run UtfNormalGenerate.php to create this file again (make clean && make) - * - * @file - */ -// @codingStandardsIgnoreFile - -UtfNormal::\$utfCombiningClass = unserialize( '$serCombining' ); -UtfNormal::\$utfCanonicalComp = unserialize( '$serComp' ); -UtfNormal::\$utfCanonicalDecomp = unserialize( '$serCanon' ); -UtfNormal::\$utfCheckNFC = unserialize( '$serCheckNFC' ); -\n"; - fputs( $out, $outdata ); - fclose( $out ); - print "Wrote out UtfNormalData.inc\n"; -} else { - print "Can't create file UtfNormalData.inc\n"; - exit( -1 ); -} - -$out = fopen( "UtfNormalDataK.inc", "wt" ); -if ( $out ) { - $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) ); - $outdata = "<" . "?php -/** - * This file was automatically generated -- do not edit! - * Run UtfNormalGenerate.php to create this file again (make clean && make) - * - * @file - */ -// @codingStandardsIgnoreFile - -UtfNormal::\$utfCompatibilityDecomp = unserialize( '$serCompat' ); -\n"; - fputs( $out, $outdata ); - fclose( $out ); - print "Wrote out UtfNormalDataK.inc\n"; - exit( 0 ); -} else { - print "Can't create file UtfNormalDataK.inc\n"; - exit( -1 ); -} - -# --------------- - -function callbackCanonical( $matches ) { - // @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix - global $canonicalDecomp; - // @codingStandardsIgnoreEnd - - if ( isset( $canonicalDecomp[$matches[1]] ) ) { - return $canonicalDecomp[$matches[1]]; - } - - return $matches[1]; -} - -function callbackCompat( $matches ) { - // @codingStandardsIgnoreStart MediaWiki.NamingConventions.ValidGlobalName.wgPrefix - global $compatibilityDecomp; - // @codingStandardsIgnoreEnd - - if ( isset( $compatibilityDecomp[$matches[1]] ) ) { - return $compatibilityDecomp[$matches[1]]; - } - - return $matches[1]; -} |