diff options
Diffstat (limited to 'maintenance')
58 files changed, 2700 insertions, 1283 deletions
diff --git a/maintenance/addwiki.php b/maintenance/addwiki.php index 3b6bb5d6..a19b24ce 100644 --- a/maintenance/addwiki.php +++ b/maintenance/addwiki.php @@ -71,23 +71,32 @@ function addWiki( $lang, $site, $dbName ) } } + global $wgTitle, $wgArticle; $wgTitle = Title::newMainPage(); $wgArticle = new Article( $wgTitle ); $ucsite = ucfirst( $site ); - $wgArticle->insertNewArticle( " -==This subdomain is reserved for the creation of a $ucsite in '''[[:en:{$name}|{$name}]]''' language== + $wgArticle->insertNewArticle( <<<EOT +==This subdomain is reserved for the creation of a [[wikimedia:Our projects|$ucsite]] in '''[[w:en:{$name}|{$name}]]''' language== -If you can write in this language and want to collaborate in the creation of this encyclopedia then '''you''' can make it. +* Please '''do not start editing''' this new site. This site has a test project on the [[incubator:|Wikimedia Incubator]] (or on the [[betawikiversity:|BetaWikiversity]] or on the [[oldwikisource:|Old Wikisource]]) and it will be imported to here. -Go ahead. Translate this page and start working on your encyclopedia. +* If you would like to help translating the interface to this language, please do not translate here, but go to [[betawiki:|Betawiki]], a special wiki for translating the interface. That way everyone can use it on every wiki using the [[mw:|same software]]. -For help, see '''[[m:Help:How to start a new Wikipedia|how to start a new Wikipedia]]'''. +* For information about how to edit and for other general help, see [[m:Help:Contents|Help on Wikimedia's Meta-Wiki]] or [[mw:Help:Contents|Help on MediaWiki.org]]. -==Sister projects== -[http://meta.wikipedia.org Meta-Wikipedia] | [http://www.wiktionary.org Wikitonary] | [http://www.wikibooks.org Wikibooks] | [http://www.wikinews.org Wikinews] | [http://www.wikiquote.org Wikiquote] | [http://www.wikisource.org Wikisource] +== Sister projects == +<span class="plainlinks"> +[http://www.wikipedia.org Wikipedia] | +[http://www.wiktionary.org Wiktonary] | +[http://www.wikibooks.org Wikibooks] | +[http://www.wikinews.org Wikinews] | +[http://www.wikiquote.org Wikiquote] | +[http://www.wikisource.org Wikisource] +[http://www.wikiversity.org Wikiversity] +</span> -See the [http://www.wikipedia.org Wikipedia portal] for other language Wikipedias. +See Wikimedia's [[m:|Meta-Wiki]] for the coordination of these projects. [[aa:]] [[af:]] @@ -99,6 +108,7 @@ See the [http://www.wikipedia.org Wikipedia portal] for other language Wikipedia [[ast:]] [[ay:]] [[az:]] +[[bcl:]] [[be:]] [[bg:]] [[bn:]] @@ -125,6 +135,7 @@ See the [http://www.wikipedia.org Wikipedia portal] for other language Wikipedia [[he:]] [[hi:]] [[hr:]] +[[hsb:]] [[hy:]] [[ia:]] [[id:]] @@ -195,7 +206,9 @@ See the [http://www.wikipedia.org Wikipedia portal] for other language Wikipedia [[za:]] [[zh:]] [[zu:]] -", '', false, false ); + +EOT +, '', false, false ); print "Adding to dblists\n"; diff --git a/maintenance/archives/patch-image_reditects.sql b/maintenance/archives/patch-image_reditects.sql new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/maintenance/archives/patch-image_reditects.sql diff --git a/maintenance/archives/patch-protected_titles.sql b/maintenance/archives/patch-protected_titles.sql new file mode 100644 index 00000000..5307cbdd --- /dev/null +++ b/maintenance/archives/patch-protected_titles.sql @@ -0,0 +1,12 @@ +-- Protected titles - nonexistent pages that have been protected +CREATE TABLE /*$wgDBprefix*/protected_titles ( + pt_namespace int NOT NULL, + pt_title varchar(255) NOT NULL, + pt_user int unsigned NOT NULL, + pt_reason tinyblob, + pt_timestamp binary(14) NOT NULL, + pt_expiry varbinary(14) NOT NULL default '', + pt_create_perm varbinary(60) NOT NULL, + PRIMARY KEY (pt_namespace,pt_title), + KEY pt_timestamp (pt_timestamp) +) /*$wgDBTableOptions*/; diff --git a/maintenance/archives/populateSha1.php b/maintenance/archives/populateSha1.php index a34d36d3..45f29c43 100644 --- a/maintenance/archives/populateSha1.php +++ b/maintenance/archives/populateSha1.php @@ -4,7 +4,7 @@ $optionsWithArgs = array( 'method' ); require_once( dirname(__FILE__).'/../commandLine.inc' ); -$method = isset( $args['method'] ) ? $args['method'] : 'normal'; +$method = isset( $options['method'] ) ? $options['method'] : 'normal'; $t = -microtime( true ); $fname = 'populateSha1.php'; @@ -14,30 +14,42 @@ $imageTable = $dbw->tableName( 'image' ); $oldimageTable = $dbw->tableName( 'oldimage' ); $batch = array(); -$cmd = 'mysql -u ' . wfEscapeShellArg( $wgDBuser ) . ' -p' . wfEscapeShellArg( $wgDBpassword, $wgDBname ); +$cmd = 'mysql -u' . wfEscapeShellArg( $wgDBuser ) . + ' -h' . wfEscapeShellArg( $wgDBserver ) . + ' -p' . wfEscapeShellArg( $wgDBpassword, $wgDBname ); if ( $method == 'pipe' ) { + echo "Using pipe method\n"; $pipe = popen( $cmd, 'w' ); - fwrite( $pipe, "-- hello\n" ); } +$numRows = $res->numRows(); +$i = 0; foreach ( $res as $row ) { + if ( $i % 100 == 0 ) { + printf( "Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100 ); + wfWaitForSlaves( 5 ); + } $file = wfLocalFile( $row->img_name ); + if ( !$file ) { + continue; + } $sha1 = File::sha1Base36( $file->getPath() ); if ( strval( $sha1 ) !== '' ) { $sql = "UPDATE $imageTable SET img_sha1=" . $dbw->addQuotes( $sha1 ) . " WHERE img_name=" . $dbw->addQuotes( $row->img_name ); if ( $method == 'pipe' ) { - fwrite( $pipe, $sql ); + fwrite( $pipe, "$sql;\n" ); } else { $dbw->query( $sql, $fname ); } } + $i++; } if ( $method == 'pipe' ) { fflush( $pipe ); pclose( $pipe ); } $t += microtime( true ); -print "Done in $t seconds\n"; +printf( "\nDone %d files in %.1f seconds\n", $numRows, $t ); ?> diff --git a/maintenance/backup.inc b/maintenance/backup.inc index ee44954c..94fb48c9 100644 --- a/maintenance/backup.inc +++ b/maintenance/backup.inc @@ -170,7 +170,8 @@ class BackupDumper { function dump( $history, $text = MW_EXPORT_TEXT ) { # Notice messages will foul up your XML output even if they're # relatively harmless. - ini_set( 'display_errors', false ); + if( ini_get( 'display_errors' ) ) + ini_set( 'display_errors', 'stderr' ); $this->initProgress( $history ); diff --git a/maintenance/cleanupImages.php b/maintenance/cleanupImages.php index 1c0edeb5..d6ed5a7a 100644 --- a/maintenance/cleanupImages.php +++ b/maintenance/cleanupImages.php @@ -66,8 +66,8 @@ class ImageCleanup extends TableCleanup { return $this->progress( 1 ); } - if( $title->getDbKey() !== $source ) { - $munged = $title->getDbKey(); + if( $title->getDBkey() !== $source ) { + $munged = $title->getDBkey(); $this->log( "page $source ($munged) doesn't match self." ); $this->pokeFile( $source, $munged ); return $this->progress( 1 ); @@ -154,7 +154,7 @@ class ImageCleanup extends TableCleanup { $name ); $test = Title::makeTitleSafe( NS_IMAGE, $x ); - if( is_null( $test ) || $test->getDbKey() !== $x ) { + if( is_null( $test ) || $test->getDBkey() !== $x ) { $this->log( "Unable to generate safe title from '$name', got '$x'" ); return false; } diff --git a/maintenance/cleanupSpam.php b/maintenance/cleanupSpam.php index 2c269b34..36d8b258 100644 --- a/maintenance/cleanupSpam.php +++ b/maintenance/cleanupSpam.php @@ -14,9 +14,8 @@ function cleanupArticle( $id, $domain ) { $rev = Revision::newFromTitle( $title ); $revId = $rev->getId(); $currentRevId = $revId; - $regex = LinkFilter::makeRegex( $domain ); - while ( $rev && preg_match( $regex, $rev->getText() ) ) { + while ( $rev && LinkFilter::matchEntry( $rev->getText() , $domain ) ) { # Revision::getPrevious can't be used in this way before MW 1.6 (Revision.php 1.26) #$rev = $rev->getPrevious(); $revId = $title->getPreviousRevisionID( $revId ); diff --git a/maintenance/cleanupTitles.php b/maintenance/cleanupTitles.php index 0ec57d4e..1f06b165 100644 --- a/maintenance/cleanupTitles.php +++ b/maintenance/cleanupTitles.php @@ -79,7 +79,7 @@ class TitleCleanup extends TableCleanup { $title = Title::newFromText( $clean ); } - $dest = $title->getDbKey(); + $dest = $title->getDBkey(); if( $this->dryrun ) { $this->log( "DRY RUN: would rename $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')" ); } else { @@ -97,7 +97,7 @@ class TitleCleanup extends TableCleanup { if( $title->getInterwiki() ) { $prior = $title->getPrefixedDbKey(); } else { - $prior = $title->getDbKey(); + $prior = $title->getDBkey(); } $clean = 'Broken/' . $prior; $verified = Title::makeTitleSafe( $row->page_namespace, $clean ); @@ -112,7 +112,7 @@ class TitleCleanup extends TableCleanup { wfDie( "Something awry; empty title.\n" ); } $ns = $title->getNamespace(); - $dest = $title->getDbKey(); + $dest = $title->getDBkey(); if( $this->dryrun ) { $this->log( "DRY RUN: would rename $row->page_id ($row->page_namespace,'$row->page_title') to ($row->page_namespace,'$dest')" ); } else { diff --git a/maintenance/commandLine.inc b/maintenance/commandLine.inc index 4466344f..f7bb53ff 100644 --- a/maintenance/commandLine.inc +++ b/maintenance/commandLine.inc @@ -216,6 +216,20 @@ if ( defined( 'MW_CMDLINE_CALLBACK' ) ) { ini_set( 'memory_limit', -1 ); +if( version_compare( phpversion(), '5.2.4' ) >= 0 ) { + // Send PHP warnings and errors to stderr instead of stdout. + // This aids in diagnosing problems, while keeping messages + // out of redirected output. + if( ini_get( 'display_errors' ) ) { + ini_set( 'display_errors', 'stderr' ); + } + + // Don't touch the setting on earlier versions of PHP, + // as setting it would disable output if you'd wanted it. + + // Note that exceptions are also sent to stderr when + // command-line mode is on, regardless of PHP version. +} $wgShowSQLErrors = true; require_once( "$IP/includes/Setup.php" ); diff --git a/maintenance/createAndPromote.php b/maintenance/createAndPromote.php index af4a1dab..0d30fe73 100644 --- a/maintenance/createAndPromote.php +++ b/maintenance/createAndPromote.php @@ -38,7 +38,7 @@ if( !is_object( $user ) ) { # Insert the account into the database $user->addToDatabase(); $user->setPassword( $password ); -$user->setToken(); +$user->saveSettings(); # Promote user $user->addGroup( 'sysop' ); diff --git a/maintenance/deleteBatch.php b/maintenance/deleteBatch.php index 6821ee29..62169641 100644 --- a/maintenance/deleteBatch.php +++ b/maintenance/deleteBatch.php @@ -67,6 +67,10 @@ for ( $linenum = 1; !feof( $file ); $linenum++ ) { $dbw->begin(); if( $page->getNamespace() == NS_IMAGE ) { $art = new ImagePage( $page ); + $img = wfFindFile( $art->mTitle ); + if( !$img || !$img->delete( $reason ) ) { + print "FAILED to delete image file... "; + } } else { $art = new Article( $page ); } @@ -75,7 +79,7 @@ for ( $linenum = 1; !feof( $file ); $linenum++ ) { if ( $success ) { print "\n"; } else { - print " FAILED\n"; + print " FAILED to delete image page\n"; } if ( $interval ) { diff --git a/maintenance/deleteDefaultMessages.php b/maintenance/deleteDefaultMessages.php index 32e03494..9a7f5c6a 100644 --- a/maintenance/deleteDefaultMessages.php +++ b/maintenance/deleteDefaultMessages.php @@ -1,7 +1,7 @@ <?php /** - * Deletes all pages in the MediaWiki namespace which were last edited by + * Deletes all pages in the MediaWiki namespace which were last edited by * "MediaWiki default". */ @@ -17,7 +17,7 @@ function deleteDefaultMessages() { global $wgUser; $wgUser = User::newFromName( $user ); $wgUser->addGroup( 'bot' ); - + $dbr = wfGetDB( DB_SLAVE ); $res = $dbr->select( array( 'page', 'revision' ), array( 'page_namespace', 'page_title' ), diff --git a/maintenance/deleteOldRevisions.inc b/maintenance/deleteOldRevisions.inc index 8d8ca9a1..03fb2306 100644 --- a/maintenance/deleteOldRevisions.inc +++ b/maintenance/deleteOldRevisions.inc @@ -9,7 +9,7 @@ require_once( 'purgeOldText.inc' ); -function DeleteOldRevisions( $delete = false ) { +function DeleteOldRevisions( $delete = false, $args = array() ) { # Data should come off the master, wrapped in a transaction $dbw = wfGetDB( DB_MASTER ); @@ -18,9 +18,20 @@ function DeleteOldRevisions( $delete = false ) { $tbl_pag = $dbw->tableName( 'page' ); $tbl_rev = $dbw->tableName( 'revision' ); + $pageIdClause = ''; + $revPageClause = ''; + + # If a list of page_ids was provided, limit results to that set of page_ids + if ( sizeof( $args ) > 0 ) { + $pageIdList = implode( ',', $args ); + $pageIdClause = " WHERE page_id IN ({$pageIdList})"; + $revPageClause = " AND rev_page IN ({$pageIdList})"; + echo( "Limiting to {$tbl_pag}.page_id IN ({$pageIdList})\n" ); + } + # Get "active" revisions from the page table echo( "Searching for active revisions..." ); - $res = $dbw->query( "SELECT page_latest FROM $tbl_pag" ); + $res = $dbw->query( "SELECT page_latest FROM $tbl_pag{$pageIdClause}" ); while( $row = $dbw->fetchObject( $res ) ) { $cur[] = $row->page_latest; } @@ -29,7 +40,7 @@ function DeleteOldRevisions( $delete = false ) { # Get all revisions that aren't in this set echo( "Searching for inactive revisions..." ); $set = implode( ', ', $cur ); - $res = $dbw->query( "SELECT rev_id FROM $tbl_rev WHERE rev_id NOT IN ( $set )" ); + $res = $dbw->query( "SELECT rev_id FROM $tbl_rev WHERE rev_id NOT IN ( $set ){$revPageClause}" ); while( $row = $dbw->fetchObject( $res ) ) { $old[] = $row->rev_id; } diff --git a/maintenance/deleteOldRevisions.php b/maintenance/deleteOldRevisions.php index 8d676ab1..8454479b 100644 --- a/maintenance/deleteOldRevisions.php +++ b/maintenance/deleteOldRevisions.php @@ -16,12 +16,12 @@ echo( "Delete Old Revisions\n\n" ); if( @$options['help'] ) { ShowUsage(); } else { - DeleteOldRevisions( @$options['delete'] ); + DeleteOldRevisions( @$options['delete'], $args ); } function ShowUsage() { echo( "Deletes non-current revisions from the database.\n\n" ); - echo( "Usage: php deleteOldRevisions.php [--delete|--help]\n\n" ); + echo( "Usage: php deleteOldRevisions.php [--delete|--help] [<page_id> ...]\n\n" ); echo( "delete : Performs the deletion\n" ); echo( " help : Show this usage information\n" ); } diff --git a/maintenance/dumpHTML.php b/maintenance/dumpHTML.php index 26e914ff..bd94958e 100644 --- a/maintenance/dumpHTML.php +++ b/maintenance/dumpHTML.php @@ -1,160 +1,7 @@ -<?php -/** - * @todo document - * @addtogroup Maintenance - */ - -function ShowUsage() { -echo <<<END -Usage: -php dumpHTML.php --help -php dumpHTML.php [options...] - - --help show this message - - -d <dest> destination directory - -s <start> start ID - -e <end> end ID - -k <skin> skin to use (defaults to htmldump) - --no-overwrite skip existing HTML files - --checkpoint <file> use a checkpoint file to allow restarting of interrupted dumps - --slice <n/m> split the job into m segments and do the n'th one - --images only do image description pages - --shared-desc only do shared (commons) image description pages - --no-shared-desc don't do shared image description pages - --categories only do category pages - --redirects only do redirects - --special only do miscellaneous stuff - --force-copy copy commons instead of symlink, needed for Wikimedia - --interlang allow interlanguage links - --image-snapshot copy all images used to the destination directory - --compress generate compressed version of the html pages - --udp-profile <N> profile 1/N rendering operations using ProfilerSimpleUDP - -END; -} - -$optionsWithArgs = array( 's', 'd', 'e', 'k', 'checkpoint', 'slice', 'udp-profile' ); -$options = array( 'help' ); -$profiling = false; - -if ( $profiling ) { - define( 'MW_CMDLINE_CALLBACK', 'wfSetupDump' ); - function wfSetupDump() { - global $wgProfiling, $wgProfileToDatabase, $wgProfileSampleRate; - $wgProfiling = true; - $wgProfileToDatabase = false; - $wgProfileSampleRate = 1; - } -} - -if ( in_array( '--udp-profile', $argv ) ) { - define( 'MW_FORCE_PROFILE', 1 ); -} - -require_once( "commandLine.inc" ); -require_once( "dumpHTML.inc" ); - -error_reporting( E_ALL & (~E_NOTICE) ); - -if( isset( $options['help'] ) ) { - ShowUsage(); - exit(); -} - -if ( !empty( $options['s'] ) ) { - $start = $options['s']; -} else { - $start = 1; -} - -if ( !empty( $options['e'] ) ) { - $end = $options['e']; -} else { - $dbr = wfGetDB( DB_SLAVE ); - $end = $dbr->selectField( 'page', 'max(page_id)', false ); -} - -if ( !empty( $options['d'] ) ) { - $dest = $options['d']; -} else { - $dest = "$IP/static"; -} - -$skin = isset( $options['k'] ) ? $options['k'] : 'htmldump'; - -if ( $options['slice'] ) { - $bits = explode( '/', $options['slice'] ); - if ( count( $bits ) != 2 || $bits[0] < 1 || $bits[0] > $bits[1] ) { - print "Invalid slice specification"; - exit; - } - $sliceNumerator = $bits[0]; - $sliceDenominator = $bits[1]; -} else { - $sliceNumerator = $sliceDenominator = 1; -} - -$wgHTMLDump = new DumpHTML( array( - 'dest' => $dest, - 'forceCopy' => $options['force-copy'], - 'alternateScriptPath' => $options['interlang'], - 'interwiki' => $options['interlang'], - 'skin' => $skin, - 'makeSnapshot' => $options['image-snapshot'], - 'checkpointFile' => $options['checkpoint'], - 'startID' => $start, - 'endID' => $end, - 'sliceNumerator' => $sliceNumerator, - 'sliceDenominator' => $sliceDenominator, - 'noOverwrite' => $options['no-overwrite'], - 'compress' => $options['compress'], - 'noSharedDesc' => $options['no-shared-desc'], - 'udpProfile' => $options['udp-profile'], -)); - - -if ( $options['special'] ) { - $wgHTMLDump->doSpecials(); -} elseif ( $options['images'] ) { - $wgHTMLDump->doImageDescriptions(); -} elseif ( $options['categories'] ) { - $wgHTMLDump->doCategories(); -} elseif ( $options['redirects'] ) { - $wgHTMLDump->doRedirects(); -} elseif ( $options['shared-desc'] ) { - $wgHTMLDump->doSharedImageDescriptions(); -} else { - print "Creating static HTML dump in directory $dest. \n"; - $dbr = wfGetDB( DB_SLAVE ); - $server = $dbr->getProperty( 'mServer' ); - print "Using database {$server}\n"; - - if ( !isset( $options['e'] ) ) { - $wgHTMLDump->doEverything(); - } else { - $wgHTMLDump->doArticles(); - } -} - -if ( isset( $options['debug'] ) ) { - #print_r($GLOBALS); - # Workaround for bug #36957 - $globals = array_keys( $GLOBALS ); - #sort( $globals ); - $sizes = array(); - foreach ( $globals as $name ) { - $sizes[$name] = strlen( serialize( $GLOBALS[$name] ) ); - } - arsort($sizes); - $sizes = array_slice( $sizes, 0, 20 ); - foreach ( $sizes as $name => $size ) { - printf( "%9d %s\n", $size, $name ); - } -} - -if ( $profiling ) { - echo $wgProfiler->getOutput(); -} +dumpHTML has moved to the DumpHTML extension. +WebDAV/SVN: +http://svn.wikimedia.org/svnroot/mediawiki/trunk/extensions/DumpHTML/ +Web: +http://svn.wikimedia.org/viewvc/mediawiki/trunk/extensions/DumpHTML/ diff --git a/maintenance/dumpTextPass.php b/maintenance/dumpTextPass.php index 92ab4b4e..440f6d27 100644 --- a/maintenance/dumpTextPass.php +++ b/maintenance/dumpTextPass.php @@ -104,6 +104,13 @@ class TextPassDumper extends BackupDumper { var $failures = 0; var $maxFailures = 200; var $failureTimeout = 5; // Seconds to sleep after db failure + + var $php = "php"; + var $spawn = false; + var $spawnProc = false; + var $spawnWrite = false; + var $spawnRead = false; + var $spawnErr = false; function dump() { # This shouldn't happen if on console... ;) @@ -111,7 +118,8 @@ class TextPassDumper extends BackupDumper { # Notice messages will foul up your XML output even if they're # relatively harmless. -// ini_set( 'display_errors', false ); + if( ini_get( 'display_errors' ) ) + ini_set( 'display_errors', 'stderr' ); $this->initProgress( $this->history ); @@ -125,6 +133,10 @@ class TextPassDumper extends BackupDumper { if( WikiError::isError( $result ) ) { wfDie( $result->getMessage() ); } + + if( $this->spawnProc ) { + $this->closeSpawn(); + } $this->report( true ); } @@ -134,7 +146,8 @@ class TextPassDumper extends BackupDumper { switch( $opt ) { case 'prefetch': - require_once 'maintenance/backupPrefetch.inc'; + global $IP; + require_once "$IP/maintenance/backupPrefetch.inc"; $this->prefetch = new BaseDump( $url ); break; case 'stub': @@ -146,6 +159,12 @@ class TextPassDumper extends BackupDumper { case 'full': $this->history = WikiExporter::FULL; break; + case 'spawn': + $this->spawn = true; + if( $val ) { + $this->php = $val; + } + break; } } @@ -237,9 +256,26 @@ class TextPassDumper extends BackupDumper { return $text; } } + return $this->doGetText( $id ); + } + + private function doGetText( $id ) { + if( $this->spawn ) { + return $this->getTextSpawned( $id ); + } else { + return $this->getTextDbSafe( $id ); + } + } + + /** + * Fetch a text revision from the database, retrying in case of failure. + * This may survive some transitory errors by reconnecting, but + * may not survive a long-term server outage. + */ + private function getTextDbSafe( $id ) { while( true ) { try { - $text = $this->doGetText( $id ); + $text = $this->getTextDb( $id ); $ex = new MWException("Graceful storage failure"); } catch (DBQueryError $ex) { $text = false; @@ -263,7 +299,7 @@ class TextPassDumper extends BackupDumper { /** * May throw a database error if, say, the server dies during query. */ - private function doGetText( $id ) { + private function getTextDb( $id ) { $id = intval( $id ); $row = $this->db->selectRow( 'text', array( 'old_text', 'old_flags' ), @@ -277,6 +313,111 @@ class TextPassDumper extends BackupDumper { $normalized = UtfNormal::cleanUp( $stripped ); return $normalized; } + + private function getTextSpawned( $id ) { + wfSuppressWarnings(); + if( !$this->spawnProc ) { + // First time? + $this->openSpawn(); + } + while( true ) { + + $text = $this->getTextSpawnedOnce( $id ); + if( !is_string( $text ) ) { + $this->progress("Database subprocess failed. Respawning..."); + + $this->closeSpawn(); + sleep( $this->failureTimeout ); + $this->openSpawn(); + + continue; + } + wfRestoreWarnings(); + return $text; + } + } + + function openSpawn() { + global $IP, $wgDBname; + + $cmd = implode( " ", + array_map( 'wfEscapeShellArg', + array( + $this->php, + "$IP/maintenance/fetchText.php", + $wgDBname ) ) ); + $spec = array( + 0 => array( "pipe", "r" ), + 1 => array( "pipe", "w" ), + 2 => array( "file", "/dev/null", "a" ) ); + $pipes = array(); + + $this->progress( "Spawning database subprocess: $cmd" ); + $this->spawnProc = proc_open( $cmd, $spec, $pipes ); + if( !$this->spawnProc ) { + // shit + $this->progress( "Subprocess spawn failed." ); + return false; + } + list( + $this->spawnWrite, // -> stdin + $this->spawnRead, // <- stdout + ) = $pipes; + + return true; + } + + private function closeSpawn() { + wfSuppressWarnings(); + if( $this->spawnRead ) + fclose( $this->spawnRead ); + $this->spawnRead = false; + if( $this->spawnWrite ) + fclose( $this->spawnWrite ); + $this->spawnWrite = false; + if( $this->spawnErr ) + fclose( $this->spawnErr ); + $this->spawnErr = false; + if( $this->spawnProc ) + pclose( $this->spawnProc ); + $this->spawnProc = false; + wfRestoreWarnings(); + } + + private function getTextSpawnedOnce( $id ) { + $ok = fwrite( $this->spawnWrite, "$id\n" ); + //$this->progress( ">> $id" ); + if( !$ok ) return false; + + $ok = fflush( $this->spawnWrite ); + //$this->progress( ">> [flush]" ); + if( !$ok ) return false; + + $len = fgets( $this->spawnRead ); + //$this->progress( "<< " . trim( $len ) ); + if( $len === false ) return false; + + $nbytes = intval( $len ); + $text = ""; + + // Subprocess may not send everything at once, we have to loop. + while( $nbytes > strlen( $text ) ) { + $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) ); + if( $text === false ) break; + $text .= $buffer; + } + + $gotbytes = strlen( $text ); + if( $gotbytes != $nbytes ) { + $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes "); + return false; + } + + // Do normalization in the dump thread... + $stripped = str_replace( "\r", "", $text ); + $normalized = UtfNormal::cleanUp( $stripped ); + return $normalized; + } function startElement( $parser, $name, $attribs ) { $this->clearOpenElement( null ); @@ -371,6 +512,7 @@ Options: (Default: 100) --server=h Force reading from MySQL server h --current Base ETA on number of pages in database instead of all revisions + --spawn Spawn a subprocess for loading text records END ); } diff --git a/maintenance/dumpUploads.php b/maintenance/dumpUploads.php index 74a28380..50d03ae1 100644 --- a/maintenance/dumpUploads.php +++ b/maintenance/dumpUploads.php @@ -3,12 +3,12 @@ require_once 'commandLine.inc'; class UploadDumper { - function __construct( $args ) { global $IP, $wgUseSharedUploads; - $this->mAction = 'fetchUsed'; + $this->mAction = 'fetchLocal'; $this->mBasePath = $IP; - $this->mShared = $wgUseSharedUploads; + $this->mShared = false; + $this->mSharedSupplement = false; if( isset( $args['help'] ) ) { $this->mAction = 'help'; @@ -17,10 +17,31 @@ class UploadDumper { if( isset( $args['base'] ) ) { $this->mBasePath = $args['base']; } + + if( isset( $args['local'] ) ) { + $this->mAction = 'fetchLocal'; + } + + if( isset( $args['used'] ) ) { + $this->mAction = 'fetchUsed'; + } + + if( isset( $args['shared'] ) ) { + if( isset( $args['used'] ) ) { + // Include shared-repo files in the used check + $this->mShared = true; + } else { + // Grab all local *plus* used shared + $this->mSharedSupplement = true; + } + } } function run() { - $this->{$this->mAction}(); + $this->{$this->mAction}( $this->mShared ); + if( $this->mSharedSupplement ) { + $this->fetchUsed( true ); + } } function help() { @@ -35,8 +56,6 @@ php dumpUploads.php [options] > list-o-files.txt Options: --base=<path> Set base relative path instead of wiki include root -FIXME: other options not implemented yet ;) - --local List all local files, used or not. No shared files included. --used Skip local images that are not used --shared Include images used from shared repository @@ -50,7 +69,7 @@ END; * @param string $directory Base directory where files are located * @param bool $shared true to pass shared-dir settings to hash func */ - function fetchUsed() { + function fetchUsed( $shared ) { $dbr = wfGetDB( DB_SLAVE ); $image = $dbr->tableName( 'image' ); $imagelinks = $dbr->tableName( 'imagelinks' ); @@ -61,52 +80,38 @@ END; ON il_to=img_name"; $result = $dbr->query( $sql ); - while( $row = $dbr->fetchObject( $result ) ) { - if( is_null( $row->img_name ) ) { - if( $this->mShared ) { - $this->outputShared( $row->il_to ); - } - } else { - $this->outputLocal( $row->il_to ); - } + foreach( $result as $row ) { + $this->outputItem( $row->il_to, $shared ); } $dbr->freeResult( $result ); } - function outputLocal( $name ) { - global $wgUploadDirectory; - return $this->outputItem( $name, $wgUploadDirectory, false ); + function fetchLocal( $shared ) { + $dbr = wfGetDB( DB_SLAVE ); + $result = $dbr->select( 'image', + array( 'img_name' ), + '', + __METHOD__ ); + + foreach( $result as $row ) { + $this->outputItem( $row->img_name, $shared ); + } + $dbr->freeResult( $result ); } - function outputShared( $name ) { - global $wgSharedUploadDirectory; - return $this->outputItem( $name, $wgSharedUploadDirectory, true ); + function outputItem( $name, $shared ) { + $file = wfFindFile( $name ); + if( $file && $this->filterItem( $file, $shared ) ) { + $filename = $file->getFullPath(); + $rel = wfRelativePath( $filename, $this->mBasePath ); + echo "$rel\n"; + } else { + wfDebug( __METHOD__ . ": base file? $name\n" ); + } } - function outputItem( $name, $directory, $shared ) { - $filename = $directory . - wfGetHashPath( $name, $shared ) . - $name; - $rel = $this->relativePath( $filename, $this->mBasePath ); - echo "$rel\n"; - } - - /** - * Return a relative path to $path from the base directory $base - * For instance relativePath( '/foo/bar/baz', '/foo' ) should return - * 'bar/baz'. - */ - function relativePath( $path, $base) { - $path = explode( DIRECTORY_SEPARATOR, $path ); - $base = explode( DIRECTORY_SEPARATOR, $base ); - while( count( $base ) && $path[0] == $base[0] ) { - array_shift( $path ); - array_shift( $base ); - } - foreach( $base as $prefix ) { - array_unshift( $path, '..' ); - } - return implode( DIRECTORY_SEPARATOR, $path ); + function filterItem( $file, $shared ) { + return $shared || $file->isLocal(); } } diff --git a/maintenance/fetchText.php b/maintenance/fetchText.php new file mode 100644 index 00000000..3b745c0a --- /dev/null +++ b/maintenance/fetchText.php @@ -0,0 +1,36 @@ +<?php + +/** + * Communications protocol... + */ + +require "commandLine.inc"; + +$db = wfGetDB( DB_SLAVE ); +$stdin = fopen( "php://stdin", "rt" ); +while( !feof( $stdin ) ) { + $line = fgets( $stdin ); + $textId = intval( $line ); + $text = doGetText( $db, $textId ); + echo strlen( $text ) . "\n"; + echo $text; +} + +/** + * May throw a database error if, say, the server dies during query. + */ +function doGetText( $db, $id ) { + $id = intval( $id ); + $row = $db->selectRow( 'text', + array( 'old_text', 'old_flags' ), + array( 'old_id' => $id ), + 'TextPassDumper::getText' ); + $text = Revision::getRevisionText( $row ); + if( $text === false ) { + return false; + } + return $text; +} + + +?>
\ No newline at end of file diff --git a/maintenance/findhooks.php b/maintenance/findhooks.php index 284e7906..8433571d 100644 --- a/maintenance/findhooks.php +++ b/maintenance/findhooks.php @@ -7,25 +7,26 @@ * - hooks names in hooks.txt are at the beginning of a line and single quoted. * - hooks names in code are the first parameter of wfRunHooks. * + * Any instance of wfRunHooks that doesn't meet these parameters will be noted. + * * @addtogroup Maintenance * * @author Ashar Voultoiz <hashar@altern.org> * @copyright Copyright © Ashar voultoiz * @license http://www.gnu.org/copyleft/gpl.html GNU General Public Licence 2.0 or later */ - + /** This is a command line script*/ include('commandLine.inc'); - - + + # GLOBALS - + $doc = $IP . '/docs/hooks.txt'; -$pathinc = $IP . '/includes/'; - - +$pathinc = array( $IP.'/includes/', $IP.'/includes/api/', $IP.'/includes/filerepo/', $IP.'/languages/', $IP.'/maintenance/', $IP.'/skins/' ); + # FUNCTIONS - + /** * @return array of documented hooks */ @@ -36,19 +37,19 @@ function getHooksFromDoc() { preg_match_all( "/\n'(.*?)'/", $content, $m); return $m[1]; } - + /** - * Get hooks from a php file + * Get hooks from a PHP file * @param $file Full filename to the PHP file. * @return array of hooks found. */ function getHooksFromFile( $file ) { $content = file_get_contents( $file ); $m = array(); - preg_match_all( "/wfRunHooks\(\s*\'(.*?)\'/", $content, $m); - return $m[1]; + preg_match_all( '/wfRunHooks\(\s*([\'"])(.*?)\1/', $content, $m); + return $m[2]; } - + /** * Get hooks from the source code. * @param $path Directory where the include files can be found @@ -66,7 +67,43 @@ function getHooksFromPath( $path ) { } return $hooks; } - + +/** + * Get bad hooks (where the hook name could not be determined) from a PHP file + * @param $file Full filename to the PHP file. + * @return array of bad wfRunHooks() lines + */ +function getBadHooksFromFile( $file ) { + $content = file_get_contents( $file ); + $m = array(); + # We want to skip the "function wfRunHooks()" one. :) + preg_match_all( '/(?<!function )wfRunHooks\(\s*[^\s\'"].*/', $content, $m); + $list = array(); + foreach( $m[0] as $match ){ + $list[] = $match . "(" . $file . ")"; + } + return $list; +} + +/** + * Get bad hooks from the source code. + * @param $path Directory where the include files can be found + * @return array of bad wfRunHooks() lines + */ +function getBadHooksFromPath( $path ) { + $hooks = array(); + if( $dh = opendir($path) ) { + while(($file = readdir($dh)) !== false) { + # We don't want to read this file as it contains bad calls to wfRunHooks() + if( filetype( $path.$file ) == 'file' && !$path.$file == __FILE__ ) { + $hooks = array_merge( $hooks, getBadHooksFromFile($path.$file) ); + } + } + closedir($dh); + } + return $hooks; +} + /** * Nicely output the array * @param $msg A message to show before the value @@ -75,20 +112,29 @@ function getHooksFromPath( $path ) { */ function printArray( $msg, $arr, $sort = true ) { if($sort) asort($arr); - foreach($arr as $v) print "$msg: $v\n"; + foreach($arr as $v) echo "$msg: $v\n"; } - - -# MAIN - + + +# MAIN + $documented = getHooksFromDoc($doc); -$potential = getHooksFromPath($pathinc); - -$todo = array_diff($potential, $documented); -$deprecated = array_diff($documented, $potential); - +$potential = array(); +$bad = array(); +foreach( $pathinc as $dir ) { + $potential = array_merge( $potential, getHooksFromPath( $dir ) ); + $bad = array_merge( $bad, getBadHooksFromPath( $dir ) ); +} + +$potential = array_unique( $potential ); +$bad = array_unique( $bad ); +$todo = array_diff( $potential, $documented ); +$deprecated = array_diff( $documented, $potential ); + // let's show the results: printArray('undocumented', $todo ); printArray('not found', $deprecated ); - - +printArray('unclear hook calls', $bad ); + +if ( count( $todo ) == 0 && count( $deprecated ) == 0 && count( $bad ) == 0 ) + echo "Looks good!\n";
\ No newline at end of file diff --git a/maintenance/interwiki.sql b/maintenance/interwiki.sql index cfa25d9e..ed71c067 100644 --- a/maintenance/interwiki.sql +++ b/maintenance/interwiki.sql @@ -68,6 +68,7 @@ REPLACE INTO /*$wgDBprefix*/interwiki (iw_prefix,iw_url,iw_local) VALUES ('jefo','http://www.esperanto-jeunes.org/vikio/index.php?$1',0), ('jiniwiki','http://www.cdegroot.com/cgi-bin/jini?$1',0), ('jspwiki','http://www.ecyrd.com/JSPWiki/Wiki.jsp?page=$1',0), +('keiki','http://kei.ki/en/$1',0), ('kerimwiki','http://wiki.oxus.net/$1',0), ('kmwiki','http://www.voght.com/cgi-bin/pywiki?$1',0), ('knowhow','http://www2.iro.umontreal.ca/~paquetse/cgi-bin/wiki.cgi?$1',0), @@ -142,6 +143,7 @@ REPLACE INTO /*$wgDBprefix*/interwiki (iw_prefix,iw_url,iw_local) VALUES ('ursine','http://wiki.ursine.ca/$1',0), ('usej','http://www.tejo.org/usej/$1',0), ('usemod','http://www.usemod.com/cgi-bin/wiki.pl?$1',0), +('vinismo','http://vinismo.com/en/$1',0), ('visualworks','http://wiki.cs.uiuc.edu/VisualWorks/$1',0), ('warpedview','http://www.warpedview.com/index.php/$1',0), ('webdevwikinl','http://www.promo-it.nl/WebDevWiki/index.php?page=$1',0), @@ -154,7 +156,7 @@ REPLACE INTO /*$wgDBprefix*/interwiki (iw_prefix,iw_url,iw_local) VALUES ('wikicities','http://www.wikicities.com/index.php/$1',0), ('wikif1','http://www.wikif1.org/$1',0), ('wikihow','http://www.wikihow.com/$1',0), -('wikinfo','http://www.wikinfo.org/wiki.php?title=$1',0), +('wikinfo','http://www.wikinfo.org/index.php/$1',0), ('wikimedia','http://wikimediafoundation.org/wiki/$1',0), ('wikiquote','http://en.wikiquote.org/wiki/$1',1), ('wikinews','http://en.wikinews.org/wiki/$1',1), diff --git a/maintenance/language/StatOutputs.php b/maintenance/language/StatOutputs.php new file mode 100644 index 00000000..50829cbe --- /dev/null +++ b/maintenance/language/StatOutputs.php @@ -0,0 +1,103 @@ +<?php +if (!defined('MEDIAWIKI')) die(); +/** + * Statistic output classes. + * + * @author Ævar Arnfjörð Bjarmason <avarab@gmail.com> + * @author Ashar Voultoiz <thoane@altern.org> + */ + +/** A general output object. Need to be overriden */ +class statsOutput { + function formatPercent( $subset, $total, $revert = false, $accuracy = 2 ) { + return @sprintf( '%.' . $accuracy . 'f%%', 100 * $subset / $total ); + } + + # Override the following methods + function heading() { + } + function footer() { + } + function blockstart() { + } + function blockend() { + } + function element( $in, $heading = false ) { + } +} + +/** Outputs WikiText */ +class wikiStatsOutput extends statsOutput { + function heading() { + global $IP; + $version = SpecialVersion::getVersion( $IP ); + echo "'''Statistics are based on:''' <code>" . $version . "</code>\n\n"; + echo "'''Note:''' These statistics can be generated by running <code>php maintenance/language/transstat.php</code>.\n\n"; + echo "For additional information on specific languages (the message names, the actual problems, etc.), run <code>php maintenance/language/checkLanguage.php --lang=foo</code>.\n\n"; + echo '{| class="sortable wikitable" border="2" cellpadding="4" cellspacing="0" style="background-color: #F9F9F9; border: 1px #AAAAAA solid; border-collapse: collapse; clear:both;" width="100%"'."\n"; + } + function footer() { + echo "|}\n"; + } + function blockstart() { + echo "|-\n"; + } + function blockend() { + echo ''; + } + function element( $in, $heading = false ) { + echo ($heading ? '!' : '|') . " $in\n"; + } + function formatPercent( $subset, $total, $revert = false, $accuracy = 2 ) { + $v = @round(255 * $subset / $total); + if ( $revert ) { + $v = 255 - $v; + } + if ( $v < 128 ) { + # Red to Yellow + $red = 'FF'; + $green = sprintf( '%02X', 2 * $v ); + } else { + # Yellow to Green + $red = sprintf('%02X', 2 * ( 255 - $v ) ); + $green = 'FF'; + } + $blue = '00'; + $color = $red . $green . $blue; + + $percent = statsOutput::formatPercent( $subset, $total, $revert, $accuracy ); + return 'bgcolor="#'. $color .'" | '. $percent; + } +} + +/** Outputs WikiText and appends category and text only used for Meta-Wiki */ +class metawikiStatsOutput extends wikiStatsOutput { + function heading() { + echo "See [[MediaWiki localisation]] to learn how you can help translating MediaWiki.\n\n"; + parent::heading(); + } + function footer() { + parent::footer(); + echo "\n[[Category:Localisation|Statistics]]\n"; + } +} + +/** Output text. To be used on a terminal for example. */ +class textStatsOutput extends statsOutput { + function element( $in, $heading = false ) { + echo $in."\t"; + } + function blockend() { + echo "\n"; + } +} + +/** csv output. Some people love excel */ +class csvStatsOutput extends statsOutput { + function element( $in, $heading = false ) { + echo $in . ";"; + } + function blockend() { + echo "\n"; + } +} diff --git a/maintenance/language/checkLanguage.inc b/maintenance/language/checkLanguage.inc index 468db550..51de8014 100644 --- a/maintenance/language/checkLanguage.inc +++ b/maintenance/language/checkLanguage.inc @@ -1,109 +1,23 @@ <?php -/** - * Check a language. - * - * @param $languages The languages object. - * @param $code The language code (default content language of the wiki running the script on). - * @param: $links Show wiki links to messages (default false)? - * @param: $wikiLanguage Language of the wiki to show the output in, if showing links (default en). - * @param: $checks Checks to do (default all except for duplicates and plural). - * @return Number of errors found. - */ -function checkLanguage( $languages, $code = null, $displayLevel = 2, $links = false, $wikiLanguage = 'en', $checks = null ) { - # Get messages - $messages = $languages->getMessages( $code ); - $messagesNumber = count( $messages['translated'] ); - # Skip the checks if told so - if ( $displayLevel == 0 ) { - return; - } - - # Initialize counts - $problems = 0; - - # Set default language code and checks - if ( !$code ) { - global $wgContLang; - $code = $wgContLang->getCode(); - } - if ( !$checks ) { - $checks = array( 'untranslated', 'obsolete', 'variables', 'empty', 'whitespace', 'xhtml', 'chars' ); - } - - # Untranslated messages - if ( in_array( 'untranslated', $checks ) ) { - $generalMessages = $languages->getGeneralMessages(); - $requiredMessagesNumber = count( $generalMessages['required'] ); - $untranslatedMessages = $languages->getUntranslatedMessages( $code ); - $untranslatedMessagesNumber = count( $untranslatedMessages ); - $languages->outputMessagesList( $untranslatedMessages, $code, "\n$untranslatedMessagesNumber messages of $requiredMessagesNumber are not translated to $code, but exist in en:", $displayLevel, $links, $wikiLanguage ); - $problems += $untranslatedMessagesNumber; - } - - # Duplicate messages - if ( in_array( 'duplicate', $checks ) ) { - $duplicateMessages = $languages->getDuplicateMessages( $code ); - $duplicateMessagesNumber = count( $duplicateMessages ); - $languages->outputMessagesList( $duplicateMessages, $code, "\n$duplicateMessagesNumber messages of $messagesNumber are translated the same in en and $code:", $displayLevel, $links, $wikiLanguage ); - $problems += $duplicateMessagesNumber; - } - - # Obsolete messages - if ( in_array( 'obsolete', $checks ) ) { - $obsoleteMessages = $messages['obsolete']; - $obsoleteMessagesNumber = count( $obsoleteMessages ); - $languages->outputMessagesList( $obsoleteMessages, $code, "\n$obsoleteMessagesNumber messages of $messagesNumber do not exist in en (or are in the ignored list), but still exist in $code:", $displayLevel, $links, $wikiLanguage ); - $problems += $obsoleteMessagesNumber; - } - - # Messages without variables - if ( in_array( 'variables', $checks ) ) { - $messagesWithoutVariables = $languages->getMessagesWithoutVariables( $code ); - $messagesWithoutVariablesNumber = count( $messagesWithoutVariables ); - $languages->outputMessagesList( $messagesWithoutVariables, $code, "\n$messagesWithoutVariablesNumber messages of $messagesNumber in $code don't use some variables while en uses them:", $displayLevel, $links, $wikiLanguage ); - $problems += $messagesWithoutVariablesNumber; - } - - # Messages without plural - if ( in_array( 'plural', $checks ) ) { - $messagesWithoutPlural = $languages->getMessagesWithoutPlural( $code ); - $messagesWithoutPluralNumber = count( $messagesWithoutPlural ); - $languages->outputMessagesList( $messagesWithoutPlural, $code, "\n$messagesWithoutPluralNumber messages of $messagesNumber in $code don't use {{plural}} while en uses it:", $displayLevel, $links, $wikiLanguage ); - $problems += $messagesWithoutPluralNumber; - } - - # Empty messages - if ( in_array( 'empty', $checks ) ) { - $emptyMessages = $languages->getEmptyMessages( $code ); - $emptyMessagesNumber = count( $emptyMessages ); - $languages->outputMessagesList( $emptyMessages, $code, "\n$emptyMessagesNumber messages of $messagesNumber in $code are empty or -:", $displayLevel, $links, $wikiLanguage ); - $problems += $emptyMessagesNumber; - } - - # Messages with whitespace - if ( in_array( 'whitespace', $checks ) ) { - $messagesWithWhitespace = $languages->getMessagesWithWhitespace( $code ); - $messagesWithWhitespaceNumber = count( $messagesWithWhitespace ); - $languages->outputMessagesList( $messagesWithWhitespace, $code, "\n$messagesWithWhitespaceNumber messages of $messagesNumber in $code have a trailing whitespace:", $displayLevel, $links, $wikiLanguage ); - $problems += $messagesWithWhitespaceNumber; - } - - # Non-XHTML messages - if ( in_array( 'xhtml', $checks ) ) { - $nonXHTMLMessages = $languages->getNonXHTMLMessages( $code ); - $nonXHTMLMessagesNumber = count( $nonXHTMLMessages ); - $languages->outputMessagesList( $nonXHTMLMessages, $code, "\n$nonXHTMLMessagesNumber messages of $messagesNumber in $code are not well-formed XHTML:", $displayLevel, $links, $wikiLanguage ); - $problems += $nonXHTMLMessagesNumber; - } - - # Messages with wrong characters - if ( in_array( 'chars', $checks ) ) { - $messagesWithWrongChars = $languages->getMessagesWithWrongChars( $code ); - $messagesWithWrongCharsNumber = count( $messagesWithWrongChars ); - $languages->outputMessagesList( $messagesWithWrongChars, $code, "\n$messagesWithWrongCharsNumber messages of $messagesNumber in $code include hidden chars which should not be used in the messages:", $displayLevel, $links, $wikiLanguage ); - $problems += $messagesWithWrongCharsNumber; - } - - return $problems; -} +# Blacklist some checks for some languages +$checkBlacklist = array( +#'code' => array( 'check1', 'check2' ... ) +'gan' => array( 'plural' ), +'hak' => array( 'plural' ), +'ja' => array( 'plural' ), // Does not use plural +'my' => array( 'chars' ), // Uses a lot zwnj +'tet' => array( 'plural' ), +'th' => array( 'plural' ), +'wuu' => array( 'plural' ), +'yue' => array( 'plural' ), +'zh' => array( 'plural' ), +'zh-classical' => array( 'plural' ), +'zh-cn' => array( 'plural' ), +'zh-hans' => array( 'plural' ), +'zh-hant' => array( 'plural' ), +'zh-hk' => array( 'plural' ), +'zh-sg' => array( 'plural' ), +'zh-tw' => array( 'plural' ), +'zh-yue' => array( 'plural' ), +); diff --git a/maintenance/language/checkLanguage.php b/maintenance/language/checkLanguage.php index 42a43c02..36d32a48 100644 --- a/maintenance/language/checkLanguage.php +++ b/maintenance/language/checkLanguage.php @@ -7,31 +7,137 @@ require_once( dirname(__FILE__).'/../commandLine.inc' ); require_once( 'languages.inc' ); -require_once( 'checkLanguage.inc' ); -# Show help -if ( isset( $options['help'] ) ) { - echo <<<ENDS +$cli = new CheckLanguageCLI( $options ); +$cli->execute(); + +class CheckLanguageCLI { + private $code = null; + private $level = 2; + private $doLinks = false; + private $wikiCode = 'en'; + private $includeExif = false; + private $checkAll = false; + private $output = 'plain'; + private $checks = array(); + + private $defaultChecks = array( + 'untranslated', 'obsolete', 'variables', 'empty', 'plural', + 'whitespace', 'xhtml', 'chars', 'links', 'unbalanced' + ); + + private $L = null; + + /** + * GLOBALS: $wgLanguageCode; + */ + public function __construct( Array $options ) { + + if ( isset( $options['help'] ) ) { + echo $this->help(); + exit(); + } + + if ( isset($options['lang']) ) { + $this->code = $options['lang']; + } else { + global $wgLanguageCode; + $this->code = $wgLanguageCode; + } + + if ( isset($options['level']) ) { + $this->level = $options['level']; + } + + $this->doLinks = isset($options['links']); + $this->includeExif = !isset($options['noexif']); + $this->checkAll = isset($options['all']); + + if ( isset($options['wikilang']) ) { + $this->wikiCode = $options['wikilang']; + } + + if ( isset( $options['whitelist'] ) ) { + $this->checks = explode( ',', $options['whitelist'] ); + } elseif ( isset( $options['blacklist'] ) ) { + $this->checks = array_diff( + $this->defaultChecks, + explode( ',', $options['blacklist'] ) + ); + } else { + $this->checks = $this->defaultChecks; + } + + if ( isset($options['output']) ) { + $this->output = $options['output']; + } + + # Some additional checks not enabled by default + if ( isset( $options['duplicate'] ) ) { + $this->checks[] = 'duplicate'; + } + + $this->L = new languages( $this->includeExif ); + } + + protected function getChecks() { + $checks = array(); + $checks['untranslated'] = 'getUntranslatedMessages'; + $checks['duplicate'] = 'getDuplicateMessages'; + $checks['obsolete'] = 'getObsoleteMessages'; + $checks['variables'] = 'getMessagesWithoutVariables'; + $checks['plural'] = 'getMessagesWithoutPlural'; + $checks['empty'] = 'getEmptyMessages'; + $checks['whitespace'] = 'getMessagesWithWhitespace'; + $checks['xhtml'] = 'getNonXHTMLMessages'; + $checks['chars'] = 'getMessagesWithWrongChars'; + $checks['links'] = 'getMessagesWithDubiousLinks'; + $checks['unbalanced'] = 'getMessagesWithUnbalanced'; + return $checks; + } + + protected function getDescriptions() { + $descriptions = array(); + $descriptions['untranslated'] = '$1 message(s) of $2 are not translated to $3, but exist in en:'; + $descriptions['duplicate'] = '$1 message(s) of $2 are translated the same in en and $3:'; + $descriptions['obsolete'] = '$1 message(s) of $2 do not exist in en or are in the ignore list, but are in $3'; + $descriptions['variables'] = '$1 message(s) of $2 in $3 don\'t use some variables that en uses:'; + $descriptions['plural'] = '$1 message(s) of $2 in $3 don\'t use {{plural}} while en uses:'; + $descriptions['empty'] = '$1 message(s) of $2 in $3 are empty or -:'; + $descriptions['whitespace'] = '$1 message(s) of $2 in $3 have trailing whitespace:'; + $descriptions['xhtml'] = '$1 message(s) of $2 in $3 contain illegal XHTML:'; + $descriptions['chars'] = '$1 message(s) of $2 in $3 include hidden chars which should not be used in the messages:'; + $descriptions['links'] = '$1 message(s) of $2 in $3 have problematic link(s):'; + $descriptions['unbalanced'] = '$1 message(s) of $2 in $3 have unbalanced {[]}:'; + return $descriptions; + } + + protected function help() { + return <<<ENDS Run this script to check a specific language file, or all of them. +Command line settings are in form --parameter[=value]. Parameters: - * lang: Language code (default: the installation default language). You can also specify "all" to check all the languages. + * lang: Language code (default: the installation default language). + * all: Check all customized languages * help: Show this help. * level: Show the following level (default: 2). * links: Link the message values (default off). * wikilang: For the links, what is the content language of the wiki to display the output in (default en). - * whitelist: Make only the following checks (form: code,code). - * blacklist: Don't make the following checks (form: code,code). + * whitelist: Do only the following checks (form: code,code). + * blacklist: Don't do the following checks (form: code,code). * duplicate: Additionally check for messages which are translated the same to English (default off). - * plural: Additionally check for messages that don't use plural while English does (default off). * noexif: Don't check for EXIF messages (a bit hard and boring to translate), if you know that they are currently not translated and want to focus on other problems (default off). -Check codes (ideally, all of them should result 0; all the checks are executed by default): +Check codes (ideally, all of them should result 0; all the checks are executed by default (except duplicate and language specific check blacklists in checkLanguage.inc): * untranslated: Messages which are required to translate, but are not translated. + * duplicate: Messages which translation equal to fallback * obsolete: Messages which are untranslatable, but translated. * variables: Messages without variables which should be used. * empty: Empty messages. * whitespace: Messages which have trailing whitespace. - * xhtml: Messages which are not well-formed XHTML. + * xhtml: Messages which are not well-formed XHTML (checks only few common errors). * chars: Messages with hidden characters. + * links: Messages which contains broken links to pages (does not find all). + * unbalanced: Messages which contains unequal numbers of opening {[ and closing ]}. Display levels (default: 2): * 0: Skip the checks (useful for checking syntax). * 1: Show only the stub headers and number of wrong messages, without list of messages. @@ -39,47 +145,170 @@ Display levels (default: 2): * 3: Show both the headers and the complete messages, with both keys and values. ENDS; - exit(); -} + } -# Get the parameters -$wgCode = isset( $options['lang'] ) ? $options['lang'] : null; -$wgDisplayLevel = isset( $options['level'] ) ? $options['level'] : 2; -$wgLinks = isset( $options['links'] ); -$wgWikiLanguage = isset( $options['wikilang'] ) ? $options['wikilang'] : 'en'; -$wgCheckEXIF = !isset( $options['noexif'] ); - -# Get the checks -$wgChecks = array( 'untranslated', 'obsolete', 'variables', 'empty', 'whitespace', 'xhtml', 'chars' ); -if ( isset( $options['whitelist'] ) ) { - $wgChecks = explode( ',', $options['whitelist'] ); -} elseif ( isset( $options['blacklist'] ) ) { - $wgChecks = array_diff( $wgChecks, explode( ',', $options['blacklist'] ) ); -} -if ( isset( $options['duplicate'] ) ) { - $wgChecks[] = 'duplicate'; -} -if ( isset( $options['plural'] ) ) { - $wgChecks[] = 'plural'; -} + private $results = array(); + + public function execute() { + $this->doChecks(); + if ( $this->level > 0 ) { + switch ($this->output) { + case 'plain': + $this->outputText(); + break; + case 'wiki': + $this->outputWiki(); + break; + default: + throw new MWException( "Invalid output type $this->output"); + } + } + } -# Get language object -$wgLanguages = new languages( $wgCheckEXIF ); + protected function doChecks() { + $ignoredCodes = array( 'en', 'enRTL' ); -# Check the language -if ( $wgCode == 'all' ) { - foreach ( $wgLanguages->getLanguages() as $language ) { - if ( $language != 'en' && $language != 'enRTL' ) { - checkLanguage( $wgLanguages, $language ); + $this->results = array(); + # Check the language + if ( $this->checkAll ) { + foreach ( $this->L->getLanguages() as $language ) { + if ( !in_array($language, $ignoredCodes) ) { + $this->results[$language] = $this->checkLanguage( $language ); + } + } + } else { + if ( in_array($this->code, $ignoredCodes) ) { + throw new MWException("Cannot check code $this->code."); + } else { + $this->results[$this->code] = $this->checkLanguage( $this->code ); + } } } -} else { - # Can't check English or English RTL - if ( $wgCode == 'en' ) { - echo "Current selected language is English, which cannot be checked.\n"; - } else if ( $wgCode == 'enRTL' ) { - echo "Current selected language is RTL English, which cannot be checked.\n"; - } else { - checkLanguage( $wgLanguages, $wgCode, $wgDisplayLevel, $wgLinks, $wgWikiLanguage, $wgChecks ); + + protected function getCheckBlacklist() { + static $checkBlacklist = null; + if ( $checkBlacklist === null ) { + $checkBlacklist = array(); + require( dirname(__FILE__) . '/checkLanguage.inc' ); + } + return $checkBlacklist; } + + protected function checkLanguage( $code ) { + # Syntax check only + if ( $this->level === 0 ) { + $this->L->getMessages( $code ); + return; + } + + $results = array(); + $checkFunctions = $this->getChecks(); + $checkBlacklist = $this->getCheckBlacklist(); + foreach ( $this->checks as $check ) { + if ( isset($checkBlacklist[$code]) && + in_array($check, $checkBlacklist[$code]) ) { + $result[$check] = array(); + continue; + } + + $callback = array( $this->L, $checkFunctions[$check] ); + if ( !is_callable($callback ) ) { + throw new MWException( "Unkown check $check." ); + } + $results[$check] = call_user_func( $callback , $code ); + } + + return $results; + } + + protected function outputText( ) { + foreach ( $this->results as $code => $results ) { + $translated = $this->L->getMessages( $code ); + $translated = count( $translated['translated'] ); + foreach ( $results as $check => $messages ) { + $count = count( $messages ); + if ( $count ) { + $search = array( '$1', '$2', '$3' ); + $replace = array( $count, $translated, $code ); + $descriptions = $this->getDescriptions(); + echo "\n" . str_replace( $search, $replace, $descriptions[$check] ) . "\n"; + if ( $this->level == 1 ) { + echo "[messages are hidden]\n"; + } else { + foreach ( $messages as $key => $value ) { + if ( $this->doLinks ) { + $displayKey = ucfirst( $key ); + if ( $code == $this->wikiCode ) { + $displayKey = "[[MediaWiki:$displayKey|$key]]"; + } else { + $displayKey = "[[MediaWiki:$displayKey/$code|$key]]"; + } + } else { + $displayKey = $key; + } + if ( $this->level == 2 ) { + echo "* $displayKey\n"; + } else { + echo "* $displayKey: '$value'\n"; + } + } + } + } + } + } + } + + /** + * Globals: $wgContLang, $IP + */ + function outputWiki() { + global $wgContLang, $IP; + $detailText = ''; + $rows[] = '! Language !! Code !! Total !! ' . implode( ' !! ', $this->checks ); + foreach ( $this->results as $code => $results ) { + $detailTextForLang = "==$code==\n"; + $numbers = array(); + $problems = 0; + $detailTextForLangChecks = array(); + foreach ( $results as $check => $messages ) { + $count = count( $messages ); + if ( $count ) { + $problems += $count; + $messageDetails = array(); + foreach ( $messages as $key => $details ) { + $messageDetails[] = $key; + } + $detailTextForLangChecks[] = "===$code-$check===\n* " . implode( ', ', $messageDetails ); + $numbers[] = "'''[[#$code-$check|$count]]'''"; + } else { + $numbers[] = $count; + } + + } + + if ( count( $detailTextForLangChecks ) ) { + $detailText .= $detailTextForLang . implode( "\n", $detailTextForLangChecks ) . "\n"; + } + + if ( !$problems ) { continue; } // Don't list languages without problems + $language = $wgContLang->getLanguageName( $code ); + $rows[] = "| $language || $code || $problems || " . implode( ' || ', $numbers ); + } + + $tableRows = implode( "\n|-\n", $rows ); + + $version = SpecialVersion::getVersion( $IP ); + echo <<<EOL +'''Check results are for:''' <code>$version</code> + + +{| class="sortable wikitable" border="2" cellpadding="4" cellspacing="0" style="background-color: #F9F9F9; border: 1px #AAAAAA solid; border-collapse: collapse; clear:both;" +$tableRows +|} + +$detailText + +EOL; + } + } diff --git a/maintenance/language/lang2po.php b/maintenance/language/lang2po.php index 0ea3faaa..a5aa81aa 100644 --- a/maintenance/language/lang2po.php +++ b/maintenance/language/lang2po.php @@ -7,6 +7,8 @@ * - fix escaping of \ */ +$optionsWithArgs[] = 'lang'; + /** This is a command line script */ require_once(dirname(__FILE__).'/../commandLine.inc'); require_once(dirname(__FILE__).'/languages.inc'); @@ -73,7 +75,7 @@ function generatePo($langcode, $messages) { $data = poHeader(); // Generate .po entries - foreach($messages as $identifier => $content) { + foreach($messages['all'] as $identifier => $content) { $data .= "msgid \"$identifier\"\n"; // Escape backslashes @@ -134,11 +136,17 @@ echo "done.\n"; $langTool = new languages(); +if( $options['lang'] === ALL_LANGUAGES ) { + $codes = $langTool->getLanguages(); +} else { + $codes = array( $options['lang'] ); +} + // Do all languages -foreach ( $langTool->getLanguages() as $langcode) { +foreach ( $codes as $langcode) { echo "Loading messages for $langcode:\n"; if( ! generatePo($langcode, $langTool->getMessages($langcode) ) ) { - echo "ERROR: Failed to wrote file.\n"; + echo "ERROR: Failed to write file.\n"; } else { echo "Applying template:"; applyPot($langcode); diff --git a/maintenance/language/languages.inc b/maintenance/language/languages.inc index a10cae9e..9472e254 100644 --- a/maintenance/language/languages.inc +++ b/maintenance/language/languages.inc @@ -5,8 +5,6 @@ * @addtogroup Maintenance */ -require_once( 'messageTypes.inc' ); - class languages { protected $mLanguages; # List of languages protected $mRawMessages; # Raw list of the messages in each language @@ -22,7 +20,7 @@ class languages { * @param $exif Treat the EXIF messages? */ function __construct( $exif = true ) { - global $wgIgnoredMessages, $wgOptionalMessages, $wgEXIFMessages; + require( dirname(__FILE__) . '/messageTypes.inc' ); $this->mIgnoredMessages = $wgIgnoredMessages; if ( $exif ) { $this->mOptionalMessages = array_merge( $wgOptionalMessages ); @@ -62,9 +60,9 @@ class languages { } /** - * Load the raw messages for a specific langauge from the messages file. + * Load the raw messages for a specific language from the messages file. * - * @param $code The langauge code. + * @param $code The language code. */ protected function loadRawMessages( $code ) { if ( isset( $this->mRawMessages[$code] ) ) { @@ -149,7 +147,7 @@ class languages { } /** - * Get all the messages for a specific langauge (not English), without the + * Get all the messages for a specific language (not English), without the * fallback language messages, divided to groups: * all - all the messages. * required - messages which should be translated in order to get a complete translation. @@ -157,7 +155,7 @@ class languages { * obsolete - messages which should not be translated, either because they are not exist, or they are ignored messages. * translated - messages which are either required or optional, but translated from English and needed. * - * @param $code The langauge code. + * @param $code The language code. * * @return The messages in this language. */ @@ -184,7 +182,7 @@ class languages { /** * Get the untranslated messages for a specific language. * - * @param $code The langauge code. + * @param $code The language code. * * @return The untranslated messages for this language. */ @@ -203,7 +201,7 @@ class languages { /** * Get the duplicate messages for a specific language. * - * @param $code The langauge code. + * @param $code The language code. * * @return The duplicate messages for this language. */ @@ -219,10 +217,16 @@ class languages { return $duplicateMessages; } + public function getObsoleteMessages( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + return $this->mMessages[$code]['obsolete']; + } + /** * Get the messages which do not use some variables. * - * @param $code The langauge code. + * @param $code The language code. * * @return The messages which do not use some variables in this language. */ @@ -249,7 +253,7 @@ class languages { /** * Get the messages which do not use plural. * - * @param $code The langauge code. + * @param $code The language code. * * @return The messages which do not use plural in this language. */ @@ -268,7 +272,7 @@ class languages { /** * Get the empty messages. * - * @param $code The langauge code. + * @param $code The language code. * * @return The empty messages for this language. */ @@ -287,7 +291,7 @@ class languages { /** * Get the messages with trailing whitespace. * - * @param $code The langauge code. + * @param $code The language code. * * @return The messages with trailing whitespace in this language. */ @@ -306,7 +310,7 @@ class languages { /** * Get the non-XHTML messages. * - * @param $code The langauge code. + * @param $code The language code. * * @return The non-XHTML messages for this language. */ @@ -332,7 +336,7 @@ class languages { /** * Get the messages which include wrong characters. * - * @param $code The langauge code. + * @param $code The language code. * * @return The messages which include wrong characters in this language. */ @@ -366,49 +370,52 @@ class languages { return $wrongCharsMessages; } - /** - * Output a messages list - * - * @param $messages The messages list - * @param $code The language code - * @param $text The text to show before the list (optional) - * @param $level The display level (optional) - * @param $links Show links (optional) - * @param $wikilang The langauge of the wiki to display the list in, for the links (optional) - */ - public function outputMessagesList( $messages, $code, $text = '', $level = 2, $links = false, $wikilang = null ) { - if ( count( $messages ) == 0 ) { - return; - } - if ( $text ) { - echo "$text\n"; - } - if ( $level == 1 ) { - echo "[messages are hidden]\n"; - } else { - foreach ( $messages as $key => $value ) { - if ( $links ) { - $displayKey = ucfirst( $key ); - if ( !isset( $wikilang ) ) { - global $wgContLang; - $wikilang = $wgContLang->getCode(); - } - if ( $code == $wikilang ) { - $displayKey = "[[MediaWiki:$displayKey|$key]]"; - } else { - $displayKey = "[[MediaWiki:$displayKey/$code|$key]]"; - } - } else { - $displayKey = $key; + public function getMessagesWithDubiousLinks( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $tc = Title::legalChars() . '#%{}'; + $messages = array(); + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + $matches = array(); + preg_match_all( "/\[\[([{$tc}]+)(?:\\|(.+?))?]]/sDu", $value, $matches); + for ($i = 0; $i < count($matches[0]); $i++ ) { + if ( preg_match( "/.*project.*/isDu", $matches[1][$i]) ) { + $messages[$key][] = $matches[0][$i]; } - if ( $level == 2 ) { - echo "* $displayKey\n"; - } else { - echo "* $displayKey: '$value'\n"; + } + + + if ( isset( $messages[$key] ) ) { + $messages[$key] = implode( $messages[$key],", " ); + } + } + return $messages; + } + + public function getMessagesWithUnbalanced( $code ) { + $this->loadGeneralMessages(); + $this->loadMessages( $code ); + $messages = array(); + foreach ( $this->mMessages[$code]['translated'] as $key => $value ) { + + $a = $b = $c = $d = 0; + foreach ( preg_split('//', $value) as $char ) { + switch ($char) { + case '[': $a++; break; + case ']': $b++; break; + case '{': $c++; break; + case '}': $d++; break; } } + + if ( $a !== $b || $c !== $d ) { + $messages[$key] = "$a, $b, $c, $d"; + } + } + return $messages; } + } ?> diff --git a/maintenance/language/messageTypes.inc b/maintenance/language/messageTypes.inc index 21734eb3..43ca41c2 100644 --- a/maintenance/language/messageTypes.inc +++ b/maintenance/language/messageTypes.inc @@ -30,6 +30,8 @@ $wgIgnoredMessages = array( 'accesskey-ca-watch', 'accesskey-ca-unwatch', 'accesskey-search', + 'accesskey-search-go', + 'accesskey-search-fulltext', 'accesskey-p-logo', 'accesskey-n-mainpage', 'accesskey-n-portal', @@ -75,6 +77,7 @@ $wgIgnoredMessages = array( 'exif-software-value', 'history_copyright', 'licenses', + 'loginstart', 'loginend', 'loginlanguagelinks', 'markaspatrolledlink', @@ -92,6 +95,8 @@ $wgIgnoredMessages = array( 'revision-nav', 'rfcurl', 'shareddescriptionfollows', + 'signature', + 'signature-anon', 'signupend', 'sitenotice', 'sitesubtitle', @@ -116,6 +121,7 @@ $wgOptionalMessages = array( 'allpages-summary', 'booksources-summary', 'ipblocklist-summary', + 'protectedtitles-summary', 'listusers-summary', 'longpages-summary', 'preferences-summary', @@ -153,6 +159,11 @@ $wgOptionalMessages = array( 'unusedtemplates-summary', 'fewestrevisions-summary', 'withoutinterwiki-summary', + 'upload-summary', + 'tog-nolangconversion', + 'yourvariant', + 'variantname-zh-hans', + 'variantname-zh-hant', 'variantname-zh-cn', 'variantname-zh-tw', 'variantname-zh-hk', @@ -163,6 +174,9 @@ $wgOptionalMessages = array( 'variantname-sr-jc', 'variantname-sr-jl', 'variantname-sr', + 'variantname-kk-arab', + 'variantname-kk-cyrl', + 'variantname-kk-latn', 'variantname-kk-tr', 'variantname-kk-kz', 'variantname-kk-cn', @@ -172,6 +186,12 @@ $wgOptionalMessages = array( 'variantname-ku', 'rc-change-size', 'resetpass_text', + 'image_sample', + 'media_sample', + 'common.css', + 'monobook.css', + 'common.js', + 'monobook.js', 'widthheight', 'exif-fnumber-format', 'exif-focallength-format', @@ -203,7 +223,57 @@ $wgOptionalMessages = array( 'hours-abbrev', 'filerevert-backlink', 'filedelete-backlink', + 'delete-backlink', 'pagetitle', + 'filename-prefix-blacklist', + 'edittools', + 'patrol-log-diff', + 'size-bytes', + 'size-kilobytes', + 'size-megabytes', + 'size-gigabytes', + 'iranian-calendar-m1', + 'iranian-calendar-m2', + 'iranian-calendar-m3', + 'iranian-calendar-m4', + 'iranian-calendar-m5', + 'iranian-calendar-m6', + 'iranian-calendar-m7', + 'iranian-calendar-m8', + 'iranian-calendar-m9', + 'iranian-calendar-m10', + 'iranian-calendar-m11', + 'iranian-calendar-m12', + 'hebrew-calendar-m1', + 'hebrew-calendar-m2', + 'hebrew-calendar-m3', + 'hebrew-calendar-m4', + 'hebrew-calendar-m5', + 'hebrew-calendar-m6', + 'hebrew-calendar-m6a', + 'hebrew-calendar-m6b', + 'hebrew-calendar-m7', + 'hebrew-calendar-m8', + 'hebrew-calendar-m9', + 'hebrew-calendar-m10', + 'hebrew-calendar-m11', + 'hebrew-calendar-m12', + 'hebrew-calendar-m1-gen', + 'hebrew-calendar-m2-gen', + 'hebrew-calendar-m3-gen', + 'hebrew-calendar-m4-gen', + 'hebrew-calendar-m5-gen', + 'hebrew-calendar-m6-gen', + 'hebrew-calendar-m6a-gen', + 'hebrew-calendar-m6b-gen', + 'hebrew-calendar-m7-gen', + 'hebrew-calendar-m8-gen', + 'hebrew-calendar-m9-gen', + 'hebrew-calendar-m10-gen', + 'hebrew-calendar-m11-gen', + 'hebrew-calendar-m12-gen', + 'semicolon-separator', + 'comma-separator', ); /** EXIF messages, which may be set as optional in several checks, but are generally mandatory */ diff --git a/maintenance/language/messages.inc b/maintenance/language/messages.inc index 00763033..8b818d16 100644 --- a/maintenance/language/messages.inc +++ b/maintenance/language/messages.inc @@ -142,6 +142,7 @@ $wgMessageStructure = array( 'mytalk', 'anontalk', 'navigation', + 'and', ), 'metadata_help' => array( 'metadata_help', @@ -262,11 +263,17 @@ $wgMessageStructure = array( 'restorelink', 'feedlinks', 'feed-invalid', + 'feed-unavailable', + 'site-rss-feed', + 'site-atom-feed', + 'page-rss-feed', + 'page-atom-feed', 'feed-atom', 'feed-rss', 'sitenotice', 'anonnotice', 'newsectionheaderdefaultlevel', + 'red-link-title', ), 'nstab' => array( 'nstab-main', @@ -321,6 +328,8 @@ $wgMessageStructure = array( 'wrong_wfQuery_params', 'viewsource', 'viewsourcefor', + 'actionthrottled', + 'actionthrottledtext', 'protectedpagetext', 'viewsourcetext', 'protectedinterface', @@ -330,6 +339,7 @@ $wgMessageStructure = array( 'namespaceprotected', 'customcssjsprotected', 'ns-specialprotected', + 'titleprotected', ), 'login' => array( 'logouttitle', @@ -370,6 +380,7 @@ $wgMessageStructure = array( 'prefs-help-realname', 'loginerror', 'prefs-help-email', + 'prefs-help-email-required', 'nocookiesnew', 'nocookieslogin', 'noname', @@ -389,6 +400,7 @@ $wgMessageStructure = array( 'blocked-mailpassword', 'eauthentsent', 'throttled-mailpassword', + 'loginstart', 'loginend', 'signupend', 'mailerror', @@ -400,6 +412,8 @@ $wgMessageStructure = array( 'invalidemailaddress', 'accountcreated', 'accountcreatedtext', + 'createaccount-title', + 'createaccount-text', 'loginlanguagelabel', 'loginlanguagelinks', ), @@ -455,6 +469,7 @@ $wgMessageStructure = array( 'blockedtitle', 'blockedtext', 'autoblockedtext', + 'blockednoreason', 'blockedoriginalsource', 'blockededitsource', 'whitelistedittitle', @@ -479,6 +494,7 @@ $wgMessageStructure = array( 'anontalkpagetext', 'noarticletext', 'noarticletextanon', + 'userpage-userdoesnotexist', 'clearyourcache', 'usercssjsyoucanpreview', 'usercsspreview', @@ -510,6 +526,7 @@ $wgMessageStructure = array( 'protectedpagewarning', 'semiprotectedpagewarning', 'cascadeprotectedwarning', + 'titleprotectedwarning', 'templatesused', 'templatesusedpreview', 'templatesusedsection', @@ -530,10 +547,9 @@ $wgMessageStructure = array( ), 'cantcreateaccount' => array( 'cantcreateaccounttitle', - 'cantcreateaccounttext', + 'cantcreateaccount-text', ), 'history' => array( - 'revhistory', 'viewpagelogs', 'nohistory', 'revnotfound', @@ -602,13 +618,34 @@ $wgMessageStructure = array( 'oversightlog', 'overlogpagetext', ), + 'mergehistory' => array( + 'mergehistory', + 'mergehistory-header', + 'mergehistory-box', + 'mergehistory-from', + 'mergehistory-into', + 'mergehistory-list', + 'mergehistory-merge', + 'mergehistory-go', + 'mergehistory-submit', + 'mergehistory-empty', + 'mergehistory-success', + 'mergehistory-fail', + 'mergehistory-no-source', + 'mergehistory-no-destination', + 'mergehistory-invalid-source', + 'mergehistory-invalid-destination', + ), + 'mergelog' => array( + 'mergelog', + 'pagemerge-logentry', + 'revertmerge', + 'mergelogpagetext', + ), 'diffs' => array( + 'history-title', 'difference', - 'loadingrev', 'lineno', - 'editcurrent', - 'selectnewerversionfordiff', - 'selectolderversionfordiff', 'compareselectedversions', 'editundo', 'diff-multi', @@ -619,6 +656,8 @@ $wgMessageStructure = array( 'searchsubtitle', 'searchsubtitleinvalid', 'noexactmatch', + 'noexactmatch-nocreate', + 'toomanymatches', 'titlematches', 'notitlematches', 'textmatches', @@ -703,12 +742,19 @@ $wgMessageStructure = array( 'userrights-editusergroup', 'saveusergroups', 'userrights-groupsmember', + 'userrights-groupsremovable', 'userrights-groupsavailable', 'userrights-groupshelp', 'userrights-reason', 'userrights-available-none', 'userrights-available-add', 'userrights-available-remove', + 'userrights-available-add-self', + 'userrights-available-remove-self', + 'userrights-no-interwiki', + 'userrights-nodatabase', + 'userrights-nologin', + 'userrights-notallowed', ), 'group' => array( 'group', @@ -782,7 +828,11 @@ $wgMessageStructure = array( 'uploadnologintext', 'upload_directory_read_only', 'uploaderror', + 'upload-summary', 'uploadtext', + 'upload-permitted', + 'upload-preferred', + 'upload-prohibited', 'uploadlog', 'uploadlogpage', 'uploadlogpagetext', @@ -798,12 +848,14 @@ $wgMessageStructure = array( 'illegalfilename', 'badfilename', 'filetype-badmime', - 'filetype-badtype', + 'filetype-unwanted-type', + 'filetype-banned-type', 'filetype-missing', 'large-file', 'largefileserver', 'emptyfile', 'fileexists', + 'filepageexists', 'fileexists-extension', 'fileexists-thumb', 'fileexists-thumbnail-yes', @@ -824,6 +876,9 @@ $wgMessageStructure = array( 'destfilename', 'watchthisupload', 'filewasdeleted', + 'upload-wasdeleted', + 'filename-bad-prefix', + 'filename-prefix-blacklist', ), 'upload-errors' => array( 'upload-proto-error', @@ -876,6 +931,7 @@ $wgMessageStructure = array( 'nolinkstoimage', 'sharedupload', 'shareduploadwiki', + 'shareduploadwiki-desc', 'shareduploadwiki-linktext', 'shareddescriptionfollows', 'noimage', @@ -912,6 +968,9 @@ $wgMessageStructure = array( 'filedelete-nofile', 'filedelete-nofile-old', 'filedelete-iscurrent', + 'filedelete-otherreason', + 'filedelete-reason-otherlist', + 'filedelete-reason-dropdown', ), 'mimesearch' => array( 'mimesearch', @@ -933,6 +992,11 @@ $wgMessageStructure = array( 'unusedtemplatestext', 'unusedtemplateswlh', ), + 'randompage' => array( + 'randompage', + 'randompage-nopages', + 'randompage-url', + ), 'randomredirect' => array( 'randomredirect', 'randomredirect-nopages', @@ -968,6 +1032,7 @@ $wgMessageStructure = array( 'withoutinterwiki', 'withoutinterwiki-header', 'withoutinterwiki-summary', + 'withoutinterwiki-submit', ), 'fewestrevisions' => array( 'fewestrevisions', @@ -1017,9 +1082,6 @@ $wgMessageStructure = array( 'allpages-summary', 'prefixindex', 'prefixindex-summary', - 'randompage', - 'randompage-nopages', - 'randompage-url', 'shortpages', 'shortpages-summary', 'longpages', @@ -1031,13 +1093,16 @@ $wgMessageStructure = array( 'protectedpages-summary', 'protectedpagestext', 'protectedpagesempty', + 'protectedtitles', + 'protectedtitles-summary', + 'protectedtitlestext', + 'protectedtitlesempty', 'listusers', 'listusers-summary', 'specialpages', 'specialpages-summary', 'spheading', 'restrictedpheading', - 'rclsub', 'newpages', 'newpages-summary', 'newpages-username', @@ -1048,6 +1113,10 @@ $wgMessageStructure = array( 'movethispage', 'unusedimagestext', 'unusedcategoriestext', + 'notargettitle', + 'notargettext', + 'pager-newer-n', + 'pager-older-n', ), 'booksources' => array( 'booksources', @@ -1177,8 +1246,9 @@ $wgMessageStructure = array( 'excontentauthor', 'exbeforeblank', 'exblank', - 'confirmdelete', - 'deletesub', + 'delete-confirm', + 'delete-backlink', + 'delete-legend', 'historywarning', 'confirmdeletetext', 'actioncomplete', @@ -1189,6 +1259,11 @@ $wgMessageStructure = array( 'deletionlog', 'reverted', 'deletecomment', + 'deleteotherreason', + 'deletereasonotherlist', + 'deletereason-dropdown', + 'delete-toobig', + 'delete-warning-toobig', 'rollback', 'rollback_short', 'rollbacklink', @@ -1224,6 +1299,7 @@ $wgMessageStructure = array( 'protect-summary-cascade', 'protect-expiring', 'protect-cascade', + 'protect-cantedit', 'restriction-type', 'restriction-level', 'minimum-size', @@ -1233,6 +1309,7 @@ $wgMessageStructure = array( 'restrictions' => array( 'restriction-edit', 'restriction-move', + 'restriction-create', ), 'restriction-levels' => array( 'restriction-level-sysop', @@ -1251,7 +1328,9 @@ $wgMessageStructure = array( 'undeletehistorynoadmin', 'undelete-revision', 'undeleterevision-missing', + 'undelete-nodiff', 'undeletebtn', + 'undeletelink', 'undeletereset', 'undeletecomment', 'undeletedarticle', @@ -1289,10 +1368,6 @@ $wgMessageStructure = array( 'year', ), 'sp-contributions' => array( - 'sp-contributions-newest', - 'sp-contributions-oldest', - 'sp-contributions-newer', - 'sp-contributions-older', 'sp-contributions-newbies', 'sp-contributions-newbies-sub', 'sp-contributions-blocklog', @@ -1310,9 +1385,8 @@ $wgMessageStructure = array( 'whatlinkshere', 'whatlinkshere-title', 'whatlinkshere-summary', + 'whatlinkshere-page', 'whatlinkshere-barrow', - 'notargettitle', - 'notargettext', 'linklistsub', 'linkshere', 'nolinkshere', @@ -1384,9 +1458,12 @@ $wgMessageStructure = array( 'range_block_disabled', 'ipb_expiry_invalid', 'ipb_already_blocked', + 'ipb_cant_unblock', + 'ipb_blocked_as_range', 'ip_range_invalid', + 'blockme', 'proxyblocker', - 'ipb_cant_unblock', + 'proxyblocker-disabled', 'proxyblockreason', 'proxyblocksuccess', 'sorbs', @@ -1424,6 +1501,7 @@ $wgMessageStructure = array( 'pagemovedsub', 'movepage-moved', 'articleexists', + 'cantmove-titleprotected', 'talkexists', 'movedto', 'movetalk', @@ -1451,6 +1529,7 @@ $wgMessageStructure = array( 'export-addcattext', 'export-addcat', 'export-download', + 'export-templates', ), 'allmessages' => array( 'allmessages', @@ -1464,7 +1543,6 @@ $wgMessageStructure = array( ), 'thumbnails' => array( 'thumbnail-more', - 'missingimage', 'filemissing', 'thumbnail_error', 'djvu_page_error', @@ -1492,7 +1570,13 @@ $wgMessageStructure = array( 'importhistoryconflict', 'importnosources', 'importnofile', - 'importuploaderror', + 'importuploaderrorsize', + 'importuploaderrorpartial', + 'importuploaderrortemp', + 'import-parse-failure', + 'import-noarticle', + 'import-nonewrevisions', + 'xml-error-string', ), 'importlog' => array( 'importlogpage', @@ -1525,6 +1609,8 @@ $wgMessageStructure = array( 'accesskey-ca-watch', 'accesskey-ca-unwatch', 'accesskey-search', + 'accesskey-search-go', + 'accesskey-search-fulltext', 'accesskey-p-logo', 'accesskey-n-mainpage', 'accesskey-n-portal', @@ -1535,6 +1621,7 @@ $wgMessageStructure = array( 'accesskey-n-sitesupport', 'accesskey-t-whatlinkshere', 'accesskey-t-recentchangeslinked', + 'accesskey-t-random', 'accesskey-feed-rss', 'accesskey-feed-atom', 'accesskey-t-contributions', @@ -1584,6 +1671,8 @@ $wgMessageStructure = array( 'tooltip-ca-watch', 'tooltip-ca-unwatch', 'tooltip-search', + 'tooltip-search-go', + 'tooltip-search-fulltext', 'tooltip-p-logo', 'tooltip-n-mainpage', 'tooltip-n-portal', @@ -1594,6 +1683,7 @@ $wgMessageStructure = array( 'tooltip-n-sitesupport', 'tooltip-t-whatlinkshere', 'tooltip-t-recentchangeslinked', + 'tooltip-t-random', 'tooltip-feed-rss', 'tooltip-feed-atom', 'tooltip-t-contributions', @@ -1638,7 +1728,6 @@ $wgMessageStructure = array( 'anonymous', 'siteuser', 'lastmodifiedatby', - 'and', 'othercontribs', 'others', 'siteusers', @@ -1734,6 +1823,8 @@ $wgMessageStructure = array( 'bad_image_list', ), 'variantname-zh' => array( + 'variantname-zh-hans', + 'variantname-zh-hant', 'variantname-zh-cn', 'variantname-zh-tw', 'variantname-zh-hk', @@ -1748,9 +1839,12 @@ $wgMessageStructure = array( 'variantname-sr', ), 'variantname-kk' => array( - 'variantname-kk-tr', 'variantname-kk-kz', + 'variantname-kk-tr', 'variantname-kk-cn', + 'variantname-kk-cyrl', + 'variantname-kk-latn', + 'variantname-kk-arab', 'variantname-kk', ), 'variantname-ku' => array( @@ -2132,9 +2226,12 @@ $wgMessageStructure = array( 'searchnamed', 'articletitles', 'hideresults', + 'useajaxsearch', ), - 'catseparator' => array( + 'separators' => array( 'catseparator', + 'semicolon-separator', + 'comma-separator', ), 'imgmulti' => array( 'imgmultipageprev', @@ -2142,7 +2239,6 @@ $wgMessageStructure = array( 'imgmultigo', 'imgmultigotopre', 'imgmultigotopost', - 'imgmultiparseerror', ), 'tablepager' => array( 'ascending_abbrev', @@ -2183,11 +2279,6 @@ $wgMessageStructure = array( 'watchlisteditor' => array( 'watchlistedit-numitems', 'watchlistedit-noitems', - 'watchlistedit-clear-title', - 'watchlistedit-clear-legend', - 'watchlistedit-clear-confirm', - 'watchlistedit-clear-submit', - 'watchlistedit-clear-done', 'watchlistedit-normal-title', 'watchlistedit-normal-legend', 'watchlistedit-normal-explain', @@ -2206,7 +2297,83 @@ $wgMessageStructure = array( 'watchlisttools-view', 'watchlisttools-edit', 'watchlisttools-raw', - 'watchlisttools-clear', + ), + 'iranian-dates' => array( + 'iranian-calendar-m1', + 'iranian-calendar-m2', + 'iranian-calendar-m3', + 'iranian-calendar-m4', + 'iranian-calendar-m5', + 'iranian-calendar-m6', + 'iranian-calendar-m7', + 'iranian-calendar-m8', + 'iranian-calendar-m9', + 'iranian-calendar-m10', + 'iranian-calendar-m11', + 'iranian-calendar-m12', + ), + 'hebrew-dates' => array( + 'hebrew-calendar-m1', + 'hebrew-calendar-m2', + 'hebrew-calendar-m3', + 'hebrew-calendar-m4', + 'hebrew-calendar-m5', + 'hebrew-calendar-m6', + 'hebrew-calendar-m6a', + 'hebrew-calendar-m6b', + 'hebrew-calendar-m7', + 'hebrew-calendar-m8', + 'hebrew-calendar-m9', + 'hebrew-calendar-m10', + 'hebrew-calendar-m11', + 'hebrew-calendar-m12', + 'hebrew-calendar-m1-gen', + 'hebrew-calendar-m2-gen', + 'hebrew-calendar-m3-gen', + 'hebrew-calendar-m4-gen', + 'hebrew-calendar-m5-gen', + 'hebrew-calendar-m6-gen', + 'hebrew-calendar-m6a-gen', + 'hebrew-calendar-m6b-gen', + 'hebrew-calendar-m7-gen', + 'hebrew-calendar-m8-gen', + 'hebrew-calendar-m9-gen', + 'hebrew-calendar-m10-gen', + 'hebrew-calendar-m11-gen', + 'hebrew-calendar-m12-gen', + ), + 'signatures' => array( + 'signature', + 'signature-anon', + ), + 'CoreParserFunctions' => array( + 'unknown_extension_tag', + ), + 'version' => array( + 'version-extensions', + 'version-specialpages', + 'version-parserhooks', + 'version-variables', + 'version-other', + 'version-mediahandlers', + 'version-hooks', + 'version-extension-functions', + 'version-parser-extensiontags', + 'version-parser-function-hooks', + 'version-skin-extension-functions', + 'version-hook-name', + 'version-hook-subscribedby', + 'version-version', + 'version-license', + 'version-software', + 'version-software-product', + 'version-software-version', + ), + 'filepath' => array( + 'filepath', + 'filepath-page', + 'filepath-submit', + 'filepath-summary', ), ); /** Comments for each block */ @@ -2230,7 +2397,7 @@ XHTML id names.", 'badaccess' => '', 'versionrequired' => '', 'miscellaneous3' => '', - 'nstab' => "Short words for each namespace, by default used in the 'article' tab in monobook", + 'nstab' => "Short words for each namespace, by default used in the namespace tab in monobook", 'main' => 'Main script and global functions', 'errors' => 'General errors', 'login' => 'Login and logout pages', @@ -2242,7 +2409,9 @@ XHTML id names.", 'history' => 'History pages', 'history-feed' => 'Revision feed', 'revdelete' => 'Revision deletion', - 'oversightlog' => 'Oversight log', + 'oversightlog' => 'Oversight log', + 'mergehistory' => 'History merging', + 'mergelog' => 'Merge log', 'diffs' => 'Diffs', 'search' => 'Search results', 'preferences' => 'Preferences page', @@ -2264,6 +2433,7 @@ XHTML id names.", 'unwatchedpages' => 'Unwatched pages', 'listredirects' => 'List redirects', 'unusedtemplates' => 'Unused templates', + 'randompage' => 'Random page', 'randomredirect' => 'Random redirect', 'statistics' => 'Statistics', 'disambiguations' => '', @@ -2356,26 +2526,32 @@ Variants for Chinese language", 'exif-gpsmeasuremode' => '', 'exif-gpsspeed' => 'Pseudotags used for GPSSpeedRef and GPSDestDistanceRef', 'exif-gpsdirection' => 'Pseudotags used for GPSTrackRef, GPSImgDirectionRef and GPSDestBearingRef', - 'edit-externally' => 'External editor support', - 'all' => "'all' in various places, this might be different for inflected languages", - 'confirmemail' => 'E-mail address confirmation', - 'scarytransclusion' => 'Scary transclusion', - 'trackbacks' => 'Trackbacks', - 'deleteconflict' => 'Delete conflict', - 'unit-pixel' => '', - 'htmldump' => 'HTML dump', - 'purge' => 'action=purge', - 'search2' => 'AJAX search', - 'catseparator' => 'Separator for categories in page lists', - 'imgmulti' => 'Multipage image navigation', - 'tablepager' => 'Table pager', - 'autosumm' => 'Auto-summaries', - 'autoblock_whitelist' => 'Autoblock whitelist', - 'sizeunits' => 'Size units', - 'livepreview' => 'Live preview', - 'lagwarning' => 'Friendlier slave lag warnings', - 'watchlisteditor' => 'Watchlist editor', - 'watchlisttools' => 'Watchlist editing tools', + 'edit-externally' => 'External editor support', + 'all' => "'all' in various places, this might be different for inflected languages", + 'confirmemail' => 'E-mail address confirmation', + 'scarytransclusion' => 'Scary transclusion', + 'trackbacks' => 'Trackbacks', + 'deleteconflict' => 'Delete conflict', + 'unit-pixel' => '', + 'htmldump' => 'HTML dump', + 'purge' => 'action=purge', + 'search2' => 'AJAX search', + 'separators' => 'Separators for various lists', + 'imgmulti' => 'Multipage image navigation', + 'tablepager' => 'Table pager', + 'autosumm' => 'Auto-summaries', + 'autoblock_whitelist' => 'Autoblock whitelist', + 'sizeunits' => 'Size units', + 'livepreview' => 'Live preview', + 'lagwarning' => 'Friendlier slave lag warnings', + 'watchlisteditor' => 'Watchlist editor', + 'watchlisttools' => 'Watchlist editing tools', + 'iranian-dates' => 'Iranian month names', + 'hebrew-dates' => 'Hebrew month names', + 'signatures' => 'Signatures', + 'CoreParserFunctions' => 'Core parser functions', + 'version' => 'Special:Version', + 'filepath' => 'Special:Filepath', ); /** Short comments for standalone messages */ @@ -2384,6 +2560,7 @@ $wgMessageComments = array( 'sitenotice' => 'the equivalent to wgSiteNotice', 'history-feed-item-nocomment' => 'user at time', 'editcomment' => 'only shown if there is an edit comment', + 'revertpage' => 'Additional available: $3: revid of the revision reverted to, $4: timestamp of the revision reverted to, $5: revid of the revision reverted from, $6: timestamp of the revision reverted from', 'lastmodifiedatby' => '$1 date, $2 time, $3 user', 'exif-orientation-1' => '0th row: top; 0th column: left', 'exif-orientation-2' => '0th row: top; 0th column: right', @@ -2393,13 +2570,7 @@ $wgMessageComments = array( 'exif-orientation-6' => '0th row: right; 0th column: top', 'exif-orientation-7' => '0th row: right; 0th column: bottom', 'exif-orientation-8' => '0th row: left; 0th column: bottom', - 'movepage-moved' => 'The two titles are passed in plain text as $3 and $4 to allow additional goodies in the message.' -); - -/** Messages which contain dollar signs (which are not followed by numbers), and therefore should use a single apostrophe */ -$wgMessagseWithDollarSigns = array( - 'linkprefix', - 'enotif_subject', - 'enotif_body', - 'allmessagesnotsupportedDB', + 'movepage-moved' => 'The two titles are passed in plain text as $3 and $4 to allow additional goodies in the message.', + 'ipboptions' => 'display1:time1,display2:time2,...', + 'metadata-fields' => 'Do not translate list items', ); diff --git a/maintenance/language/rebuildLanguage.php b/maintenance/language/rebuildLanguage.php index 304f8b5c..6c2076eb 100644 --- a/maintenance/language/rebuildLanguage.php +++ b/maintenance/language/rebuildLanguage.php @@ -20,7 +20,7 @@ function rebuildLanguage( $code, $write, $listUnknown ) { global $wgLanguages; $messages = $wgLanguages->getMessages( $code ); $messages = $messages['all']; - writeMessagesToFile( $messages, $code, $write, $listUnknown ); + MessageWriter::writeMessagesToFile( $messages, $code, $write, $listUnknown ); } # Show help diff --git a/maintenance/language/splitLanguageFiles.inc b/maintenance/language/splitLanguageFiles.inc index 500d2cdc..a57744bd 100644 --- a/maintenance/language/splitLanguageFiles.inc +++ b/maintenance/language/splitLanguageFiles.inc @@ -1122,7 +1122,6 @@ $commonMsg = array ( 'movelogpagetext', 'thumbnail-more', -'missingimage', 'filemissing', 'monobook.css', 'nodublincore', diff --git a/maintenance/language/transstat.php b/maintenance/language/transstat.php index 6a1423a8..410bd695 100644 --- a/maintenance/language/transstat.php +++ b/maintenance/language/transstat.php @@ -10,9 +10,12 @@ * Output is posted from time to time on: * http://meta.wikimedia.org/wiki/Localization_statistics */ +$optionsWithArgs = array( 'output' ); require_once( dirname(__FILE__).'/../commandLine.inc' ); require_once( 'languages.inc' ); +require_once( dirname(__FILE__).'/StatOutputs.php' ); + if ( isset( $options['help'] ) ) { showUsage(); @@ -39,100 +42,7 @@ END; exit(); } -/** A general output object. Need to be overriden */ -class statsOutput { - function formatPercent( $subset, $total, $revert = false, $accuracy = 2 ) { - return @sprintf( '%.' . $accuracy . 'f%%', 100 * $subset / $total ); - } - - # Override the following methods - function heading() { - } - function footer() { - } - function blockstart() { - } - function blockend() { - } - function element( $in, $heading = false ) { - } -} - -/** Outputs WikiText */ -class wikiStatsOutput extends statsOutput { - function heading() { - global $IP; - $version = SpecialVersion::getVersion( $IP ); - echo "'''Statistics are based on:''' <code>" . $version . "</code>\n\n"; - echo "'''Note:''' These statistics can be generated by running <code>php maintenance/language/transstat.php</code>.\n\n"; - echo "For additional information on specific languages (the message names, the actual problems, etc.), run <code>php maintenance/language/checkLanguage.php --lang=foo</code>.\n\n"; - echo '{| class="sortable wikitable" border="2" cellpadding="4" cellspacing="0" style="background-color: #F9F9F9; border: 1px #AAAAAA solid; border-collapse: collapse; clear:both;" width="100%"'."\n"; - } - function footer() { - echo "|}\n"; - } - function blockstart() { - echo "|-\n"; - } - function blockend() { - echo ''; - } - function element( $in, $heading = false ) { - echo ($heading ? '!' : '|') . " $in\n"; - } - function formatPercent( $subset, $total, $revert = false, $accuracy = 2 ) { - $v = @round(255 * $subset / $total); - if ( $revert ) { - $v = 255 - $v; - } - if ( $v < 128 ) { - # Red to Yellow - $red = 'FF'; - $green = sprintf( '%02X', 2 * $v ); - } else { - # Yellow to Green - $red = sprintf('%02X', 2 * ( 255 - $v ) ); - $green = 'FF'; - } - $blue = '00'; - $color = $red . $green . $blue; - - $percent = statsOutput::formatPercent( $subset, $total, $revert, $accuracy ); - return 'bgcolor="#'. $color .'" | '. $percent; - } -} - -/** Outputs WikiText and appends category and text only used for Meta-Wiki */ -class metawikiStatsOutput extends wikiStatsOutput { - function heading() { - echo "See [[MediaWiki localisation]] to learn how you can help translating MediaWiki.\n\n"; - parent::heading(); - } - function footer() { - parent::footer(); - echo "\n[[Category:Localisation|Statistics]]\n"; - } -} - -/** Output text. To be used on a terminal for example. */ -class textStatsOutput extends statsOutput { - function element( $in, $heading = false ) { - echo $in."\t"; - } - function blockend() { - echo "\n"; - } -} -/** csv output. Some people love excel */ -class csvStatsOutput extends statsOutput { - function element( $in, $heading = false ) { - echo $in . ";"; - } - function blockend() { - echo "\n"; - } -} # Select an output engine switch ( $options['output'] ) { diff --git a/maintenance/language/writeMessagesArray.inc b/maintenance/language/writeMessagesArray.inc index bcbf05ee..2324785e 100644 --- a/maintenance/language/writeMessagesArray.inc +++ b/maintenance/language/writeMessagesArray.inc @@ -5,187 +5,251 @@ * @addtogroup Maintenance */ -require_once( 'messages.inc' ); -require_once( 'messageTypes.inc' ); - -/** - * Write a messages array as a PHP text and write it to the messages file. - * - * @param $messages The messages array. - * @param $code The language code. - * @param $write Write to the messages file? - * @param $listUnknown List the unknown messages? - */ -function writeMessagesToFile( $messages, $code, $write, $listUnknown ) { - # Rewrite the messages array - $messages = writeMessagesArray( $messages, $code == 'en' ); - $messagesText = $messages[0]; - $sortedMessages = $messages[1]; - - # Write to the file - $filename = Language::getMessagesFileName( $code ); - $contents = file_get_contents( $filename ); - if ( strpos( $contents, '$messages' ) !== false ) { - $contents = explode( '$messages', $contents ); - if ( $messagesText == '$messages' . $contents[1] ) { - echo "Generated messages for language $code. Same as the current file.\n"; - } else { - if ( $write ) { - $new = $contents[0]; - $new .= $messagesText; - file_put_contents( $filename, $new ); - echo "Generated and wrote messages for language $code.\n"; +class MessageWriter { + static $optionalComment = 'only translate this message to other languages if you have to change it'; + static $ignoredComment = "don't translate or duplicate this message to other languages"; + + static $loaded = false; + static $messageStructure; + static $blockComments; + static $messageComments; + static $ignoredMessages; + static $optionalMessages; + + /** + * Write a messages array as a PHP text and write it to the messages file. + * + * @param $messages The messages array. + * @param $code The language code. + * @param $write Write to the messages file? + * @param $listUnknown List the unknown messages? + */ + public static function writeMessagesToFile( $messages, $code, $write, $listUnknown ) { + # Rewrite the messages array + $messages = self::writeMessagesArray( $messages, $code == 'en' ); + $messagesText = $messages[0]; + $sortedMessages = $messages[1]; + + # Write to the file + $filename = Language::getMessagesFileName( $code ); + $contents = file_get_contents( $filename ); + if( strpos( $contents, '$messages' ) !== false ) { + $contents = explode( '$messages', $contents ); + if( $messagesText == '$messages' . $contents[1] ) { + echo "Generated messages for language $code. Same as the current file.\n"; } else { - echo "Generated messages for language $code. Please run the script again (without the parameter \"dry-run\") to write the array to the file.\n"; + if( $write ) { + $new = $contents[0]; + $new .= $messagesText; + file_put_contents( $filename, $new ); + echo "Generated and wrote messages for language $code.\n"; + } else { + echo "Generated messages for language $code. Please run the script again (without the parameter \"dry-run\") to write the array to the file.\n"; + } } - } - if ( $listUnknown && isset( $sortedMessages['unknown'] ) && !empty( $sortedMessages['unknown'] ) ) { - echo "\nThere are " . count( $sortedMessages['unknown'] ) . " unknown messages, please check them:\n"; - foreach ( $sortedMessages['unknown'] as $key => $value ) { - echo "* " . $key . "\n"; + if( $listUnknown && isset( $sortedMessages['unknown'] ) && !empty( $sortedMessages['unknown'] ) ) { + echo "\nThere are " . count( $sortedMessages['unknown'] ) . " unknown messages, please check them:\n"; + foreach( $sortedMessages['unknown'] as $key => $value ) { + echo "* " . $key . "\n"; + } } + } else { + echo "Generated messages for language $code. There seems to be no messages array in the file.\n"; } - } else { - echo "Generated messages for language $code. There seems to be no messages array in the file.\n"; } -} -/** - * Write a messages array as a PHP text. - * - * @param $messages The messages array. - * @param $ignoredComments Show comments about ignored and optional messages? (For English.) - * - * @return Array of the PHP text and the sorted messages array. - */ -function writeMessagesArray( $messages, $ignoredComments = false ) { - global $wgMessageStructure, $wgBlockComments; - - # Sort messages to blocks - $sortedMessages['unknown'] = $messages; - foreach ( $wgMessageStructure as $blockName => $block ) { - foreach ( $block as $key ) { - if ( array_key_exists( $key, $sortedMessages['unknown'] ) ) { - $sortedMessages[$blockName][$key] = $sortedMessages['unknown'][$key]; - unset( $sortedMessages['unknown'][$key] ); + /** + * Write a messages array as a PHP text. + * + * @param $messages The messages array. + * @param $ignoredComments Show comments about ignored and optional messages? (For English.) + * + * @return Array of the PHP text and the sorted messages array. + */ + public static function writeMessagesArray( $messages, $ignoredComments = false ) { + # Load messages + if( !self::$loaded ) { + require( dirname( __FILE__ ) . '/messages.inc' ); + self::$messageStructure = $wgMessageStructure; + self::$blockComments = $wgBlockComments; + self::$messageComments = $wgMessageComments; + + require( dirname( __FILE__ ) . '/messageTypes.inc' ); + self::$ignoredMessages = $wgIgnoredMessages; + self::$optionalMessages = $wgOptionalMessages; + + self::$loaded = true; + } + + # Sort messages to blocks + $sortedMessages['unknown'] = $messages; + foreach( self::$messageStructure as $blockName => $block ) { + foreach( $block as $key ) { + if( array_key_exists( $key, $sortedMessages['unknown'] ) ) { + $sortedMessages[$blockName][$key] = $sortedMessages['unknown'][$key]; + unset( $sortedMessages['unknown'][$key] ); + } } } - } - # Write all the messages - $messagesText = "\$messages = array( + # Write all the messages + $messagesText = "\$messages = array( "; - foreach( $sortedMessages as $block => $messages ) { - # Skip if it's the block of unknown messages - handle that in the end of file - if ( $block == 'unknown' ) { - continue; + foreach( $sortedMessages as $block => $messages ) { + # Skip if it's the block of unknown messages - handle that in the end of file + if( $block == 'unknown' ) { + continue; + } + + if( $ignoredComments ) { + $ignored = self::$ignoredMessages; + $optional = self::$optionalMessages; + } else { + $ignored = array(); + $optional = array(); + } + $comments = self::makeComments( array_keys($messages), self::$messageComments, $ignored, $optional ); + + # Write the block + $messagesText .= self::writeMessagesBlock( self::$blockComments[$block], $messages, $comments ); } - # Write the block - $messagesText .= writeMessagesBlock( $block, $wgBlockComments[$block], $messages, $ignoredComments ); - } - ksort( $sortedMessages['unknown'] ); - $messagesText .= writeMessagesBlock( 'unknown', 'Unknown messages', $sortedMessages['unknown'], $ignoredComments ); # Write the unknown messages, alphabetically sorted - $messagesText .= "); + # Write the unknown messages, alphabetically sorted. + # Of course, we don't have any comments for them, because the are unknown. + ksort( $sortedMessages['unknown'] ); + $messagesText .= self::writeMessagesBlock( 'Unknown messages', $sortedMessages['unknown'] ); + $messagesText .= "); "; - return array( $messagesText, $sortedMessages ); -} + return array( $messagesText, $sortedMessages ); + } -/** - * Write a block of messages to PHP. - * - * @param $name The block name. - * @param $comment The block comment. - * @param $messages The block messages. - * @param $ignoredComments Show comments about ignored and optional messages? (For English.) - * - * @return The block, formatted in PHP. - */ -function writeMessagesBlock( $name, $comment, $messages, $ignoredComments ) { - global $wgMessageComments, $wgMessagseWithDollarSigns; - global $wgIgnoredMessages, $wgOptionalMessages; - $blockText = ''; - - # Skip the block if it includes no messages - if ( empty( $messages ) ) { - return ''; + /** + * Generates an array of comments for messages. + * + * @param $messages Key of messages. + * @param $comments Comments for messages, indexed by key. + * @param $ignored List of ingored message keys. + * @param $optional List of optional message keys. + */ + public static function makeComments( $messages, $comments, $ignored, $optional ) { + # Comment collector + $commentArray = array(); + + # List of keys only + foreach( $messages as $key ) { + $commentsForKey = array(); + + # Add descriptive comment for this message if there is one + if( array_key_exists( $key, $comments ) ) { + $commentsForKey[] = $comments[$key]; + } + + # For translator information only + if( in_array( $key, $ignored ) ) { + $commentsForKey[] = self::$ignoredComment; + } elseif( in_array( $key, $optional ) ) { + $commentsForKey[] = self::$optionalComment; + } + + # Format one or more comments nicely and store in array + if( count( $commentsForKey ) ) { + $commentArray[$key] = ' # ' . implode( '; ', $commentsForKey ); + } + } + + return $commentArray; } - # Format the block comment (if exists); check for multiple lines comments - if ( !empty( $comment ) ) { - if ( strpos( $comment, "\n" ) === false ) { - $blockText .= "# $comment + /** + * Write a block of messages to PHP. + * + * @param $blockComment The comment of whole block. + * @param $messages The block messages. + * @param $messageComments Optional comments for messages in this block. + * @param $prefix Prefix for every line, for indenting purposes. + * + * @return The block, formatted in PHP. + */ + public static function writeMessagesBlock( $blockComment, $messages, + $messageComments = array(), $prefix = '' ) { + + $blockText = ''; + + # Skip the block if it includes no messages + if( empty( $messages ) ) { + return ''; + } + + # Format the block comment (if exists); check for multiple lines comments + if( !empty( $blockComment ) ) { + if( strpos( $blockComment, "\n" ) === false ) { + $blockText .= "$prefix# $blockComment "; - } else { - $blockText .= "/* -$comment + } else { + $blockText .= "$prefix/* +$blockComment */ "; + } } - } - # Get max key length - $maxKeyLength = max( array_map( 'strlen', array_keys( $messages ) ) ); + # Get max key length + $maxKeyLength = max( array_map( 'strlen', array_keys( $messages ) ) ); - # Format the messages - foreach( $messages as $key => $value ) { - # Add the key name - $blockText .= "'$key'"; + # Format the messages + foreach( $messages as $key => $value ) { + # Add the key name + $blockText .= "$prefix'$key'"; - # Add the appropriate block whitespace - $blockText .= str_repeat( ' ', $maxKeyLength - strlen( $key ) ); + # Add the appropriate block whitespace + $blockText .= str_repeat( ' ', $maxKeyLength - strlen( $key ) ); - # Refer to the value - $blockText .= ' => '; + # Refer to the value + $blockText .= ' => '; - # Check for the appropriate apostrophe and add the value - if ( strpos( $value, "'" ) === false ) { - $blockText .= "'$value'"; - } elseif ( strpos( $value, '"' ) === false && !in_array( $key, $wgMessagseWithDollarSigns ) ) { - $blockText .= "\"$value\""; - } else { - # Pick the less numerous one to escape - $quote = substr_count( $value, '"' ) + substr_count( $value, '$' ) >= substr_count( $value, "'" ) ? "'" : '"'; - if ('"' == $quote) { $extra = '$'; } - else { $extra = ''; } - $blockText .= $quote . addcslashes( $value, $quote.'\\'.$extra ) . $quote; - } + # Check for the appropriate apostrophe and add the value + # Quote \ here, because it needs always escaping + $value = addcslashes( $value, '\\' ); + + # For readability + $single = "'"; + $double = '"'; - # Comma - $blockText .= ','; - - $ignoredComment = "don't translate or duplicate this message to other languages"; - $optionalComment = "only translate this message to other languages if you have to change it"; - $showIgnoredOrOptionalComment = in_array( $key, $wgIgnoredMessages ) || in_array( $key, $wgOptionalMessages ); - if ( $ignoredComments ) { - if ( array_key_exists( $key, $wgMessageComments ) ) { - $blockText .= ' # ' . $wgMessageComments[$key]; - if ( $showIgnoredOrOptionalComment ) { - $blockText .= '; '; + if( strpos( $value, $single ) === false ) { + # Nothing ugly, just use ' + $blockText .= $single.$value.$single; + } elseif( strpos( $value, $double ) === false && !preg_match('/\$[a-zA-Z_\x7f-\xff]/', $value) ) { + # No "-quotes, no variables that need quoting, use " + $blockText .= $double.$value.$double; + } else { + # Something needs quoting, pick the quote which causes less quoting + $quote = substr_count( $value, $double ) + substr_count( $value, '$' ) >= substr_count( $value, $single ) ? $single : $double; + if( $quote === $double ) { + $extra = '$'; + } else { + $extra = ''; } - } elseif ( $showIgnoredOrOptionalComment ) { - $blockText .= ' # '; + $blockText .= $quote . addcslashes( $value, $quote . $extra ) . $quote; } - if ( in_array( $key, $wgIgnoredMessages ) ) { - $blockText .= $ignoredComment; - } elseif ( in_array( $key, $wgOptionalMessages ) ) { - $blockText .= $optionalComment; + + # Comma + $blockText .= ','; + + # Add comments, if there is any + if( array_key_exists( $key, $messageComments ) ) { + $blockText .= $messageComments[$key]; } - } elseif ( array_key_exists( $key, $wgMessageComments ) ) { - $blockText .= ' # ' . $wgMessageComments[$key]; - } - # Newline - $blockText .= " + # Newline + $blockText .= " "; - } + } - # Newline to end the block - $blockText .= " + # Newline to end the block + $blockText .= " "; - return $blockText; + return $blockText; + } } - -?> diff --git a/maintenance/namespaceDupes.php b/maintenance/namespaceDupes.php index 16796ba3..dcde7c61 100644 --- a/maintenance/namespaceDupes.php +++ b/maintenance/namespaceDupes.php @@ -23,7 +23,7 @@ $options = array( 'fix', 'suffix', 'help' ); require_once( 'commandLine.inc' ); if(isset( $options['help'] ) ) { -print <<<END +print <<<ENDS usage: namespaceDupes.php [--fix] [--suffix=<text>] [--help] --help : this help message --fix : attempt to automatically fix errors @@ -33,7 +33,7 @@ usage: namespaceDupes.php [--fix] [--suffix=<text>] [--help] in place of the standard namespace list. --verbose : Display output for checked namespaces without conflicts -END; +ENDS; die; } @@ -75,14 +75,29 @@ class NamespaceConflictChecker { $spaces[$name] = $ns; } - if( !$wgCapitalLinks ) { - // We'll need to check for lowercase keys as well, - // since we're doing case-sensitive searches in the db. - foreach( array_values( $spaces ) as $name => $ns ) { - $lcname = $wgContLang->lcfirst( $name ); - $spaces[$lcname] = $ns; + // We'll need to check for lowercase keys as well, + // since we're doing case-sensitive searches in the db. + foreach( $spaces as $name => $ns ) { + $moreNames = array(); + $moreNames[] = $wgContLang->uc( $name ); + $moreNames[] = $wgContLang->ucfirst( $wgContLang->lc( $name ) ); + $moreNames[] = $wgContLang->ucwords( $name ); + $moreNames[] = $wgContLang->ucwords( $wgContLang->lc( $name ) ); + $moreNames[] = $wgContLang->ucwordbreaks( $name ); + $moreNames[] = $wgContLang->ucwordbreaks( $wgContLang->lc( $name ) ); + if( !$wgCapitalLinks ) { + foreach( $moreNames as $altName ) { + $moreNames[] = $wgContLang->lcfirst( $altName ); + } + $moreNames[] = $wgContLang->lcfirst( $name ); + } + foreach( array_unique( $moreNames ) as $altName ) { + if( $altName !== $name ) { + $spaces[$altName] = $ns; + } } } + ksort( $spaces ); asort( $spaces ); @@ -175,11 +190,21 @@ class NamespaceConflictChecker { function reportConflict( $row, $suffix ) { $newTitle = Title::makeTitleSafe( $row->namespace, $row->title ); + if( !$newTitle ) { + // Title is also an illegal title... + // For the moment we'll let these slide to cleanupTitles or whoever. + printf( "... %d (0,\"%s\")\n", + $row->id, + $row->oldtitle ); + echo "... *** cannot resolve automatically; illegal title ***\n"; + return false; + } + printf( "... %d (0,\"%s\") -> (%d,\"%s\") [[%s]]\n", $row->id, $row->oldtitle, $newTitle->getNamespace(), - $newTitle->getDbKey(), + $newTitle->getDBkey(), $newTitle->getPrefixedText() ); $id = $newTitle->getArticleId(); @@ -193,8 +218,14 @@ class NamespaceConflictChecker { function resolveConflict( $row, $resolvable, $suffix ) { if( !$resolvable ) { + echo "... *** old title {$row->title}\n"; $row->title .= $suffix; + echo "... *** new title {$row->title}\n"; $title = Title::makeTitleSafe( $row->namespace, $row->title ); + if ( ! $title ) { + echo "... !!! invalid title\n"; + return false; + } echo "... *** using suffixed form [[" . $title->getPrefixedText() . "]] ***\n"; } $tables = array( 'page' ); @@ -205,19 +236,18 @@ class NamespaceConflictChecker { } function resolveConflictOn( $row, $table ) { - $fname = 'NamespaceConflictChecker::resolveConflictOn'; echo "... resolving on $table... "; $newTitle = Title::makeTitleSafe( $row->namespace, $row->title ); $this->db->update( $table, array( "{$table}_namespace" => $newTitle->getNamespace(), - "{$table}_title" => $newTitle->getDbKey(), + "{$table}_title" => $newTitle->getDBkey(), ), array( "{$table}_namespace" => 0, "{$table}_title" => $row->oldtitle, ), - $fname ); + __METHOD__ ); echo "ok.\n"; return true; } diff --git a/maintenance/nextJobDB.php b/maintenance/nextJobDB.php index dfa8d028..b2500caf 100644 --- a/maintenance/nextJobDB.php +++ b/maintenance/nextJobDB.php @@ -21,7 +21,9 @@ if ( !$pendingDBs ) { $pendingDBs = array(); # Cross-reference DBs by master DB server $dbsByMaster = array(); - $defaultMaster = $wgAlternateMaster['DEFAULT']; + $defaultMaster = isset( $wgAlternateMaster['DEFAULT'] ) + ? $wgAlternateMaster['DEFAULT'] + : $wgDBserver; foreach ( $wgLocalDatabases as $db ) { if ( isset( $wgAlternateMaster[$db] ) ) { $dbsByMaster[$wgAlternateMaster[$db]][] = $db; @@ -31,7 +33,7 @@ if ( !$pendingDBs ) { } foreach ( $dbsByMaster as $master => $dbs ) { - $dbConn = new Database( $master, $wgDBuser, $wgDBpassword ); + $dbConn = new Database( $master, $wgDBuser, $wgDBpassword, $dbs[0] ); $stype = $dbConn->addQuotes($type); # Padding row for MySQL bug diff --git a/maintenance/parserTests.inc b/maintenance/parserTests.inc index 510a2db2..d333d873 100644 --- a/maintenance/parserTests.inc +++ b/maintenance/parserTests.inc @@ -95,6 +95,7 @@ class ParserTest { } else { $this->recorder = new TestRecorder( $this->term ); } + $this->keepUploads = isset( $options['keep-uploads'] ); $this->hooks = array(); $this->functionHooks = array(); @@ -256,6 +257,7 @@ class ParserTest { * @return bool */ private function runTest( $desc, $input, $result, $opts ) { + global $wgParserConf; if( $this->showProgress ) { $this->showTesting( $desc ); } @@ -280,12 +282,14 @@ class ParserTest { $noxml = (bool)preg_match( '~\\b noxml \\b~x', $opts ); - $parser = new Parser(); + $class = $wgParserConf['class']; + $parser = new $class( $wgParserConf ); foreach( $this->hooks as $tag => $callback ) { $parser->setHook( $tag, $callback ); } - foreach( $this->functionHooks as $tag => $callback ) { - $parser->setFunctionHook( $tag, $callback ); + foreach( $this->functionHooks as $tag => $bits ) { + list( $callback, $flags ) = $bits; + $parser->setFunctionHook( $tag, $callback, $flags ); } wfRunHooks( 'ParserTestParser', array( &$parser ) ); @@ -296,11 +300,11 @@ class ParserTest { $out = $parser->preSaveTransform( $input, $title, $user, $options ); } elseif (preg_match('/\\bmsg\\b/i', $opts)) { $out = $parser->transformMsg( $input, $options ); - } elseif( preg_match( '/\\bsection=(\d+)\b/i', $opts, $matches ) ) { - $section = intval( $matches[1] ); + } elseif( preg_match( '/\\bsection=([\w-]+)\b/i', $opts, $matches ) ) { + $section = $matches[1]; $out = $parser->getSection( $input, $section ); - } elseif( preg_match( '/\\breplace=(\d+),"(.*?)"/i', $opts, $matches ) ) { - $section = intval( $matches[1] ); + } elseif( preg_match( '/\\breplace=([\w-]+),"(.*?)"/i', $opts, $matches ) ) { + $section = $matches[1]; $replace = $matches[2]; $out = $parser->replaceSection( $input, $section, $replace ); } else { @@ -424,17 +428,18 @@ class ParserTest { * Some of these probably aren't necessary. */ private function listTables() { - $tables = array('user', 'page', 'page_restrictions', 'revision', 'text', - 'pagelinks', 'imagelinks', 'categorylinks', - 'templatelinks', 'externallinks', 'langlinks', - 'site_stats', 'hitcounter', - 'ipblocks', 'image', 'oldimage', - 'recentchanges', - 'watchlist', 'math', 'searchindex', - 'interwiki', 'querycache', - 'objectcache', 'job', 'redirect', - 'querycachetwo', 'archive', 'user_groups' + global $wgDBtype; + $tables = array('user', 'page', 'page_restrictions', + 'protected_titles', 'revision', 'text', 'pagelinks', 'imagelinks', + 'categorylinks', 'templatelinks', 'externallinks', 'langlinks', + 'site_stats', 'hitcounter', 'ipblocks', 'image', 'oldimage', + 'recentchanges', 'watchlist', 'math', 'interwiki', + 'querycache', 'objectcache', 'job', 'redirect', 'querycachetwo', + 'archive', 'user_groups' ); + + if ($wgDBtype === 'mysql') + array_push( $tables, 'searchindex' ); // Allow extensions to add to the list of tables to duplicate; // may be necessary if they hook into page save or other code @@ -548,7 +553,15 @@ class ParserTest { */ private function setupUploadDir() { global $IP; - $dir = wfTempDir() . "/mwParser-" . mt_rand() . "-images"; + if ( $this->keepUploads ) { + $dir = wfTempDir() . '/mwParser-images'; + if ( is_dir( $dir ) ) { + return $dir; + } + } else { + $dir = wfTempDir() . "/mwParser-" . mt_rand() . "-images"; + } + wfDebug( "Creating upload directory $dir\n" ); mkdir( $dir ); mkdir( $dir . '/3' ); @@ -576,6 +589,10 @@ class ParserTest { * Remove the dummy uploads directory */ private function teardownUploadDir( $dir ) { + if ( $this->keepUploads ) { + return; + } + // delete the files first, then the dirs. self::deleteFiles( array ( @@ -940,13 +957,17 @@ class DbTestRecorder extends TestRecorder { * and all that fun stuff */ function start() { + global $wgDBtype; parent::start(); $this->db->begin(); if( ! $this->db->tableExists( 'testrun' ) or ! $this->db->tableExists( 'testitem') ) { print "WARNING> `testrun` table not found in database. Trying to create table.\n"; - dbsource( 'testRunner.sql', $this->db ); + if ($wgDBtype === 'postgres') + dbsource( dirname(__FILE__) . '/testRunner.postgres.sql', $this->db ); + else + dbsource( dirname(__FILE__) . '/testRunner.sql', $this->db ); echo "OK, resuming.\n"; } @@ -962,7 +983,10 @@ class DbTestRecorder extends TestRecorder { 'tr_uname' => php_uname() ), __METHOD__ ); - $this->curRun = $this->db->insertId(); + if ($wgDBtype === 'postgres') + $this->curRun = $this->db->currentSequenceValue('testrun_id_seq'); + else + $this->curRun = $this->db->insertId(); } /** diff --git a/maintenance/parserTests.php b/maintenance/parserTests.php index 4f8edc12..0ee7e8f2 100644 --- a/maintenance/parserTests.php +++ b/maintenance/parserTests.php @@ -42,6 +42,7 @@ Options: --file Run test cases from a custom file instead of parserTests.txt --record Record tests in database --compare Compare with recorded results, without updating the database. + --keep-uploads Re-use the same upload directory for each test, don't delete it --help Show this help message diff --git a/maintenance/parserTests.txt b/maintenance/parserTests.txt index 27cc792c..b1ddb9b0 100644 --- a/maintenance/parserTests.txt +++ b/maintenance/parserTests.txt @@ -33,6 +33,23 @@ Main Page blah blah !! endarticle +!!article +Template:Foo +!!text +FOO +!!endarticle + +!! article +Template:Blank +!! text +!! endarticle + +!! article +Template:! +!! text +| +!! endarticle + ### ### Basic tests ### @@ -81,6 +98,7 @@ Italics and bold * plain''italic'''bold-italic'''''plain * plain'''bold''bold-italic'''''plain * plain l'''italic''plain +* plain l''''bold''' plain !! result <ul><li> plain </li><li> plain<i>italic</i>plain @@ -96,6 +114,7 @@ Italics and bold </li><li> plain<i>italic<b>bold-italic</b></i>plain </li><li> plain<b>bold<i>bold-italic</i></b>plain </li><li> plain l'<i>italic</i>plain +</li><li> plain l'<b>bold</b> plain </li></ul> !! end @@ -276,6 +295,26 @@ Comment semantics: unclosed comment at end !! end +!! test +Comment in template title +!! input +{{f<!---->oo}} +!! result +<p>FOO +</p> +!! end + +!! test +Comment on its own line post-expand +!! input +a +{{blank}}<!----> +b +!! result +<p>a +</p><p>b +</p> +!! end ### ### Preformatted text @@ -2376,7 +2415,7 @@ Template infinite loop !! input {{loop1}} !! result -<p><a href="/index.php?title=Loop1&action=edit" class="new" title="Loop1">loop1</a><!-- WARNING: template loop detected --> +<p><span class="error">Template loop detected: <a href="/wiki/Template:Loop1" title="Template:Loop1">Template:Loop1</a></span> </p> !! end @@ -2541,6 +2580,64 @@ Foo<noinclude>zar</noinclude><includeonly>bar</includeonly> </p> !! end +!! article +Template:Includeonly section +!! text +<includeonly> +==Includeonly section== +</includeonly> +==Section T-1== +!!endarticle + +!! test +Bug 6563: Edit link generation for section shown by <includeonly> +!! input +{{includeonly section}} +!! result +<a name="Includeonly_section"></a><h2><span class="editsection">[<a href="/index.php?title=Template:Includeonly_section&action=edit&section=T-1" title="Template:Includeonly section">edit</a>]</span> <span class="mw-headline">Includeonly section</span></h2> +<a name="Section_T-1"></a><h2><span class="editsection">[<a href="/index.php?title=Template:Includeonly_section&action=edit&section=T-2" title="Template:Includeonly section">edit</a>]</span> <span class="mw-headline">Section T-1</span></h2> + +!! end + +# Uses same input as the contents of [[Template:Includeonly section]] +!! test +Bug 6563: Section extraction for section shown by <includeonly> +!! options +section=T-2 +!! input +<includeonly> +==Includeonly section== +</includeonly> +==Section T-2== +!! result +==Section T-2== +!! end + +!! test +Bug 6563: Edit link generation for section suppressed by <includeonly> +!! input +<includeonly> +==Includeonly section== +</includeonly> +==Section 1== +!! result +<a name="Section_1"></a><h2><span class="editsection">[<a href="/index.php?title=Parser_test&action=edit&section=1" title="Edit section: Section 1">edit</a>]</span> <span class="mw-headline">Section 1</span></h2> + +!! end + +!! test +Bug 6563: Section extraction for section suppressed by <includeonly> +!! options +section=1 +!! input +<includeonly> +==Includeonly section== +</includeonly> +==Section 1== +!! result +==Section 1== +!! end + ### ### Pre-save transform tests ### @@ -3002,7 +3099,7 @@ Thumbnail image caption with a free URL !! input [[Image:foobar.jpg|thumb|http://example.com]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="http://example.com"><img alt="http://example.com" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><a href="http://example.com" class="external free" title="http://example.com" rel="nofollow">http://example.com</a></div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="http://example.com"><img alt="http://example.com" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><a href="http://example.com" class="external free" title="http://example.com" rel="nofollow">http://example.com</a></div></div></div> !! end @@ -3011,7 +3108,7 @@ BUG 1887: A ISBN with a thumbnail !! input [[Image:foobar.jpg|thumb|ISBN 1235467890]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="ISBN 1235467890"><img alt="ISBN 1235467890" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><a href="/index.php?title=Special:Booksources&isbn=1235467890" class="internal">ISBN 1235467890</a></div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="ISBN 1235467890"><img alt="ISBN 1235467890" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><a href="/index.php?title=Special:Booksources&isbn=1235467890" class="internal">ISBN 1235467890</a></div></div></div> !! end @@ -3020,7 +3117,7 @@ BUG 1887: A RFC with a thumbnail !! input [[Image:foobar.jpg|thumb|This is RFC 12354]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="This is RFC 12354"><img alt="This is RFC 12354" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>This is <a href="http://tools.ietf.org/html/rfc12354" class="external" title="http://tools.ietf.org/html/rfc12354">RFC 12354</a></div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="This is RFC 12354"><img alt="This is RFC 12354" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>This is <a href="http://tools.ietf.org/html/rfc12354" class="external" title="http://tools.ietf.org/html/rfc12354">RFC 12354</a></div></div></div> !! end @@ -3029,7 +3126,7 @@ BUG 1887: A mailto link with a thumbnail !! input [[Image:foobar.jpg|thumb|Please mailto:nobody@example.com]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="Please mailto:nobody@example.com"><img alt="Please mailto:nobody@example.com" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>Please <a href="mailto:nobody@example.com" class="external free" title="mailto:nobody@example.com" rel="nofollow">mailto:nobody@example.com</a></div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="Please mailto:nobody@example.com"><img alt="Please mailto:nobody@example.com" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>Please <a href="mailto:nobody@example.com" class="external free" title="mailto:nobody@example.com" rel="nofollow">mailto:nobody@example.com</a></div></div></div> !! end @@ -3039,7 +3136,7 @@ so math is not stripped and turns up as escaped <math> tags. !! input [[Image:foobar.jpg|thumb|<math>2+2</math>]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="<math>2+2</math>"><img alt="<math>2+2</math>" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><math>2+2</math></div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="<math>2+2</math>"><img alt="<math>2+2</math>" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><math>2+2</math></div></div></div> !! end @@ -3050,7 +3147,7 @@ math !! input [[Image:foobar.jpg|thumb|<math>2+2</math>]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="2 + 2"><img alt="2 + 2" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><span class="texhtml">2 + 2</span></div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="2 + 2"><img alt="2 + 2" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div><span class="texhtml">2 + 2</span></div></div></div> !! end @@ -3123,7 +3220,7 @@ Image caption containing another image !! input [[Image:Foobar.jpg|thumb|This is a caption with another [[Image:icon.png|image]] inside it!]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="This is a caption with another Image:Icon.png inside it!"><img alt="This is a caption with another Image:Icon.png inside it!" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>This is a caption with another <a href="/index.php?title=Special:Upload&wpDestFile=Icon.png" class="new" title="Image:Icon.png">Image:Icon.png</a> inside it!</div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="This is a caption with another Image:Icon.png inside it!"><img alt="This is a caption with another Image:Icon.png inside it!" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>This is a caption with another <a href="/index.php?title=Special:Upload&wpDestFile=Icon.png" class="new" title="Image:Icon.png">Image:Icon.png</a> inside it!</div></div></div> !! end @@ -3143,7 +3240,7 @@ Bug 3090: External links other than http: in image captions !! input [[Image:Foobar.jpg|thumb|200px|This caption has [irc://example.net irc] and [https://example.com Secure] ext links in it.]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:202px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="This caption has irc and Secure ext links in it."><img alt="This caption has irc and Secure ext links in it." src="http://example.com/images/thumb/3/3a/Foobar.jpg/200px-Foobar.jpg" width="200" height="23" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>This caption has <a href="irc://example.net" class="external text" title="irc://example.net" rel="nofollow">irc</a> and <a href="https://example.com" class="external text" title="https://example.com" rel="nofollow">Secure</a> ext links in it.</div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:202px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="This caption has irc and Secure ext links in it."><img alt="This caption has irc and Secure ext links in it." src="http://example.com/images/thumb/3/3a/Foobar.jpg/200px-Foobar.jpg" width="200" height="23" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>This caption has <a href="irc://example.net" class="external text" title="irc://example.net" rel="nofollow">irc</a> and <a href="https://example.com" class="external text" title="https://example.com" rel="nofollow">Secure</a> ext links in it.</div></div></div> !! end @@ -3470,8 +3567,8 @@ __NOTOC__ ==Section 4== !! result <a name="Section_0"></a><h2><span class="editsection">[<a href="/index.php?title=Parser_test&action=edit&section=1" title="Edit section: Section 0">edit</a>]</span> <span class="mw-headline">Section 0</span></h2> -<a name="Section_1"></a><h3><span class="editsection">[<a href="/index.php?title=Template:Sections&action=edit&section=1" title="Template:Sections">edit</a>]</span> <span class="mw-headline">Section 1</span></h3> -<a name="Section_2"></a><h2><span class="editsection">[<a href="/index.php?title=Template:Sections&action=edit&section=2" title="Template:Sections">edit</a>]</span> <span class="mw-headline">Section 2</span></h2> +<a name="Section_1"></a><h3><span class="editsection">[<a href="/index.php?title=Template:Sections&action=edit&section=T-1" title="Template:Sections">edit</a>]</span> <span class="mw-headline">Section 1</span></h3> +<a name="Section_2"></a><h2><span class="editsection">[<a href="/index.php?title=Template:Sections&action=edit&section=T-2" title="Template:Sections">edit</a>]</span> <span class="mw-headline">Section 2</span></h2> <a name="Section_4"></a><h2><span class="editsection">[<a href="/index.php?title=Parser_test&action=edit&section=2" title="Edit section: Section 4">edit</a>]</span> <span class="mw-headline">Section 4</span></h2> !! end @@ -3876,7 +3973,7 @@ Bug 2304: HTML attribute safety (unsafe breakout parameter; 2309) !! input {{div style|"><script>alert(document.cookie)</script>}} !! result -<div>Magic div</div> +<div style="float: right;"><script>alert(document.cookie)</script>">Magic div</div> !! end @@ -3885,7 +3982,7 @@ Bug 2304: HTML attribute safety (unsafe breakout parameter 2; 2309) !! input {{div style|" ><script>alert(document.cookie)</script>}} !! result -<div style="float: right;">Magic div</div> +<div style="float: right;"><script>alert(document.cookie)</script>">Magic div</div> !! end @@ -3912,7 +4009,7 @@ Bug 2304: HTML attribute safety (bold) !! input <div title="'''foobar'''"></div> !! result -<div title="'''foobar'''"></div> +<div title="'''foobar'''"></div> !! end @@ -4014,7 +4111,7 @@ MSIE CSS safety test: comment in url !! input <div style="background-image:u/**/rl(javascript:alert('boo'))">evil</div> !! result -<div style="background-image:u rl(javascript:alert('boo'))">evil</div> +<div style="background-image:u rl(javascript:alert('boo'))">evil</div> !! end @@ -4023,7 +4120,7 @@ MSIE CSS safety test: comment in expression !! input <div style="background-image:expres/**/sion(alert('boo4'))">evil4</div> !! result -<div style="background-image:expres sion(alert('boo4'))">evil4</div> +<div style="background-image:expres sion(alert('boo4'))">evil4</div> !! end @@ -4149,7 +4246,7 @@ array(0) { !! test -Parser hook: case insensetive +Parser hook: case insensitive !! input <TAG>input</TAG> !! result @@ -4163,7 +4260,7 @@ array(0) { !! test -Parser hook: case insensetive, redux +Parser hook: case insensitive, redux !! input <TaG>input</TAg> !! result @@ -4722,9 +4819,9 @@ MOVE YOUR MOUSE CURSOR OVER THIS TEXT | !! result <table> - -<u class="|">} > -<br style="onmouseover='alert(document.cookie);'" /> +{{{| +<u class="|">}}}} > +<br style="onmouseover='alert(document.cookie);'" /> MOVE YOUR MOUSE CURSOR OVER THIS TEXT <tr> @@ -4747,8 +4844,10 @@ noxml > }}}blah" onmouseover="alert('hello world');" align="left"'''MOVE MOUSE CURSOR OVER HERE !! result +<p>{{{| +</p> <li class="||"> -blah" onmouseover="alert('hello world');" align="left"<b>MOVE MOUSE CURSOR OVER HERE</b> +}}}blah" onmouseover="alert('hello world');" align="left"<b>MOVE MOUSE CURSOR OVER HERE</b> !! end @@ -5249,8 +5348,8 @@ Section extraction test with comment after heading (section 1) section=1 !! input ==a== -==legal== <!-- a legal section --> -==b== +==b== <!-- --> +==c== !! result ==a== !! end @@ -5261,10 +5360,10 @@ Section extraction test with comment after heading (section 2) section=2 !! input ==a== -==legal== <!-- a legal section --> -==b== +==b== <!-- --> +==c== !! result -==legal== <!-- a legal section --> +==b== <!-- --> !! end !! test @@ -5293,103 +5392,80 @@ section=2 !! end +# Formerly testing for bug 2587, now resolved by the use of unmarked sections +# instead of respecting commented sections !! test -Section extraction prefixed by comment (section 1) (bug 2587) +Section extraction prefixed by comment (section 1) !! options section=1 !! input <!-- -->==sec1== ==sec2== !!result -<!-- -->==sec1== +==sec2== !!end !! test -Section extraction prefixed by comment (section 2) (bug 2587) +Section extraction prefixed by comment (section 2) !! options section=2 !! input <!-- -->==sec1== ==sec2== !!result -==sec2== + !!end +# Formerly testing for bug 2607, now resolved by the use of unmarked sections +# instead of respecting HTML-style headings !! test -Section extraction, mixed wiki and html (section 1) (bug 2607) +Section extraction, mixed wiki and html (section 1) !! options section=1 !! input -<h2>1</h2> +<h2>unmarked</h2> +unmarked +==1== one ==2== two -==3== -three !! result -<h2>1</h2> +==1== one !! end !! test -Section extraction, mixed wiki and html (section 2) (bug 2607) +Section extraction, mixed wiki and html (section 2) !! options section=2 !! input -<h2>1</h2> +<h2>unmarked</h2> +unmarked +==1== one ==2== two -==3== -three !! result ==2== two !! end +# Formerly testing for bug 3342 !! test -Section extraction, heading surrounded by <noinclude> (bug 3342) +Section extraction, heading surrounded by <noinclude> !! options section=1 !! input -<noinclude>==a==</noinclude> -text +<noinclude>==unmarked==</noinclude> +==marked== !! result -<noinclude>==a==</noinclude> -text +==marked== !!end !! test -Section extraction, HTML heading subsections (bug 5272) -!! options -section=1 -!! input -<h2>a</h2> -<h3>aa</h3> -<h2>b</h2> -!! result -<h2>a</h2> -<h3>aa</h3> -!! end - -!! test -Section extraction, HTML headings should be ignored in extensions (bug 3476) -!! options -section=2 -!! input -<h2>a</h2> -<tag> -<h2>not b</h2> -</tag> -<h2>b</h2> -!! result -<h2>b</h2> -!! end - -!! test Section replacement test (section 0) !! options replace=0,"xxx" @@ -5721,94 +5797,6 @@ xxx !! test -Section extraction, HTML headings not at line boundaries (section 0) -!! options -section=0 -!! input -<h2>Evil</h2><i>blah blah blah</i> - -evil blah - -<h2>Nice</h2> - -nice blah - -<i>extra evil</i><h2>Extra nasty</h2> - -extra nasty -!! result -!! end - -!! test -Section extraction, HTML headings not at line boundaries (section 1) -!! options -section=1 -!! input -<h2>Evil</h2><i>blah blah blah</i> - -evil blah - -<h2>Nice</h2> - -nice blah - -<i>extra evil</i><h2>Extra nasty</h2> - -extra nasty -!! result -<h2>Evil</h2><i>blah blah blah</i> - -evil blah -!! end - -!! test -Section extraction, HTML headings not at line boundaries (section 2) -!! options -section=2 -!! input -<h2>Evil</h2><i>blah blah blah</i> - -evil blah - -<h2>Nice</h2> - -nice blah - -<i>extra evil</i><h2>Extra nasty</h2> - -extra nasty -!! result -<h2>Nice</h2> - -nice blah - -<i>extra evil</i> -!! end - -!! test -Section extraction, HTML headings not at line boundaries (section 3) -!! options -section=3 -!! input -<h2>Evil</h2><i>blah blah blah</i> - -evil blah - -<h2>Nice</h2> - -nice blah - -<i>extra evil</i><h2>Extra nasty</h2> - -extra nasty -!! result -<h2>Extra nasty</h2> - -extra nasty -!! end - - -!! test Section extraction, heading followed by pre with 20 spaces (bug 6398) !! options section=1 @@ -5959,24 +5947,24 @@ image4 |300px| centre !! result <table class="gallery" cellspacing="0" cellpadding="0"> <tr> - <td><div class="gallerybox" style="width: 150px;"> + <td><div class="gallerybox" style="width: 155px;"> <div style="height: 152px;">Image1.png</div> <div class="gallerytext"> </div> </div></td> - <td><div class="gallerybox" style="width: 150px;"> + <td><div class="gallerybox" style="width: 155px;"> <div style="height: 152px;">Image2.gif</div> <div class="gallerytext"> <p>|||| </p> </div> </div></td> - <td><div class="gallerybox" style="width: 150px;"> + <td><div class="gallerybox" style="width: 155px;"> <div style="height: 152px;">Image3</div> <div class="gallerytext"> </div> </div></td> - <td><div class="gallerybox" style="width: 150px;"> + <td><div class="gallerybox" style="width: 155px;"> <div style="height: 152px;">Image4</div> <div class="gallerytext"> <p>300px| centre @@ -5985,14 +5973,14 @@ image4 |300px| centre </div></td> </tr> <tr> - <td><div class="gallerybox" style="width: 150px;"> + <td><div class="gallerybox" style="width: 155px;"> <div style="height: 152px;">Image5.svg</div> <div class="gallerytext"> -<pre><a href="http://///////" class="external free" title="http://///////" rel="nofollow">http://///////</a> -</pre> +<p><a href="http://///////" class="external free" title="http://///////" rel="nofollow">http://///////</a> +</p> </div> </div></td> - <td><div class="gallerybox" style="width: 150px;"> + <td><div class="gallerybox" style="width: 155px;"> <div style="height: 152px;">* image6</div> <div class="gallerytext"> </div> @@ -6136,7 +6124,7 @@ Images with the "|" character in the comment !! input [[image:Foobar.jpg|thumb|An [http://test/?param1=|left|¶m2=|x external] URL]] !! result -<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="An external URL"><img alt="An external URL" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify" style="float:right"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>An <a href="http://test/?param1=|left|&param2=|x" class="external text" title="http://test/?param1=|left|&param2=|x" rel="nofollow">external</a> URL</div></div></div> +<div class="thumb tright"><div class="thumbinner" style="width:182px;"><a href="/wiki/Image:Foobar.jpg" class="image" title="An external URL"><img alt="An external URL" src="http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" width="180" height="20" border="0" class="thumbimage" /></a> <div class="thumbcaption"><div class="magnify"><a href="/wiki/Image:Foobar.jpg" class="internal" title="Enlarge"><img src="/skins/common/images/magnify-clip.png" width="15" height="11" alt="" /></a></div>An <a href="http://test/?param1=|left|&param2=|x" class="external text" title="http://test/?param1=|left|&param2=|x" rel="nofollow">external</a> URL</div></div></div> !!end @@ -6251,7 +6239,7 @@ MSGNW magic word !! input {{MSGNW:msg}} !! result -<p>[[:Template:Msg]] +<p>[[:Template:Msg]] </p> !! end @@ -6297,7 +6285,7 @@ Inclusion of !userCanEdit() content !! input {{MediaWiki:Fake}} !! result -<a name="header"></a><h2><span class="editsection">[<a href="/index.php?title=MediaWiki:Fake&action=edit&section=1" title="MediaWiki:Fake">edit</a>]</span> <span class="mw-headline">header</span></h2> +<a name="header"></a><h2><span class="editsection">[<a href="/index.php?title=MediaWiki:Fake&action=edit&section=T-1" title="MediaWiki:Fake">edit</a>]</span> <span class="mw-headline">header</span></h2> !! end @@ -6643,6 +6631,118 @@ Fridrih IV je car. </p> !! end +!!article +Template:Bullet +!!text +* Bar +!!endarticle + +!! test +Bug 529: Uncovered bullet +!! input +* Foo {{bullet}} +!! result +<ul><li> Foo +</li><li> Bar +</li></ul> + +!! end + +!! test +Bug 529: Uncovered table already at line-start +!! input +x + +{{table}} +y +!! result +<p>x +</p> +<table> +<tr> +<td> 1 </td><td> 2 +</td></tr> +<tr> +<td> 3 </td><td> 4 +</td></tr></table> +<p>y +</p> +!! end + +!! test +Bug 529: Uncovered bullet in parser function result +!! input +* Foo {{lc:{{bullet}} }} +!! result +<ul><li> Foo +</li><li> bar +</li></ul> + +!! end + +!! test +Bug 5678: Double-parsed template argument +!! input +{{lc:{{{1}}}|hello}} +!! result +<p>{{{1}}} +</p> +!! end + +!! test +Bug 5678: Double-parsed template invocation +!! input +{{lc:{{paramtest {{!}} param = hello }} }} +!! result +<p>{{paramtest | param = hello }} +</p> +!! end + +!! test +Morwen/13: Unclosed link followed by heading +!! input +[[link +==heading== +!! result +<p>[[link +</p> +<a name="heading"></a><h2><span class="editsection">[<a href="/index.php?title=Parser_test&action=edit&section=1" title="Edit section: heading">edit</a>]</span> <span class="mw-headline">heading</span></h2> + +!! end + +!! test +HHP2.1: Heuristics for headings in preprocessor parenthetical structures +!! input +{{foo| +=heading= +!! result +<p>{{foo| +</p> +<a name="heading"></a><h1> <span class="mw-headline">heading</span></h1> + +!! end + +!! test +HHP2.2: Heuristics for headings in preprocessor parenthetical structures +!! input +{{foo| +==heading== +!! result +<p>{{foo| +</p> +<a name="heading"></a><h2><span class="editsection">[<a href="/index.php?title=Parser_test&action=edit&section=1" title="Edit section: heading">edit</a>]</span> <span class="mw-headline">heading</span></h2> + +!! end + +!! test +Tildes in comments +!! options +pst +!! input +<!-- ~~~~ --> +!! result +<!-- ~~~~ --> +!! end # # diff --git a/maintenance/postgres/archives/patch-protected_titles.sql b/maintenance/postgres/archives/patch-protected_titles.sql new file mode 100644 index 00000000..93f10e44 --- /dev/null +++ b/maintenance/postgres/archives/patch-protected_titles.sql @@ -0,0 +1,10 @@ +CREATE TABLE protected_titles ( + pt_namespace SMALLINT NOT NULL, + pt_title TEXT NOT NULL, + pt_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL, + pt_reason TEXT NULL, + pt_timestamp TIMESTAMPTZ NOT NULL, + pt_expiry TIMESTAMPTZ NULL, + pt_create_perm TEXT NOT NULL DEFAULT '' +); +CREATE UNIQUE INDEX protected_titles_unique ON protected_titles(pt_namespace, pt_title); diff --git a/maintenance/postgres/archives/patch-ts2pagetitle.sql b/maintenance/postgres/archives/patch-ts2pagetitle.sql new file mode 100644 index 00000000..4ac985e3 --- /dev/null +++ b/maintenance/postgres/archives/patch-ts2pagetitle.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION ts2_page_title() +RETURNS TRIGGER +LANGUAGE plpgsql AS +$mw$ +BEGIN +IF TG_OP = 'INSERT' THEN + NEW.titlevector = to_tsvector('default',REPLACE(NEW.page_title,'/',' ')); +ELSIF NEW.page_title != OLD.page_title THEN + NEW.titlevector := to_tsvector('default',REPLACE(NEW.page_title,'/',' ')); +END IF; +RETURN NEW; +END; +$mw$; diff --git a/maintenance/postgres/compare_schemas.pl b/maintenance/postgres/compare_schemas.pl index 297e3af2..6f639eca 100644 --- a/maintenance/postgres/compare_schemas.pl +++ b/maintenance/postgres/compare_schemas.pl @@ -112,6 +112,8 @@ sub parse_sql { } elsif (/^ (\w+) $datatype$typeval$typeval2{0,3},?$/) { $info{$table}{column}{$1} = $2; + my $extra = $3 || ''; + $info{$table}{columnfull}{$1} = "$2$extra"; } elsif (/^ ($indextype)(?: (\w+))? \(([\w, \(\)]+)\),?$/) { $info{$table}{lc $1.'_name'} = $2 ? $2 : ''; @@ -128,6 +130,46 @@ sub parse_sql { } ## end of parse_sql +## Read in the parser test information +my $parsefile = '../parserTests.inc'; +open my $pfh, '<', $parsefile or die qq{Could not open "$parsefile": $!\n}; +my $stat = 0; +my %ptable; +while (<$pfh>) { + if (!$stat) { + if (/function listTables/) { + $stat = 1; + } + next; + } + $ptable{$1}=2 while /'(\w+)'/g; + last if /\);/; +} +close $pfh; + +my $OK_NOT_IN_PTABLE = ' +filearchive +logging +profiling +querycache_info +searchindex +trackbacks +transcache +user_newtalk +'; + +## Make sure all tables in main tables.sql are accounted for int the parsertest. +for my $table (sort keys %{$old{'../tables.sql'}}) { + $ptable{$table}++; + next if $ptable{$table} > 2; + next if $OK_NOT_IN_PTABLE =~ /\b$table\b/; + print qq{Table "$table" is in the schema, but not used inside of parserTest.inc\n}; +} +## Any that are used in ptables but no longer exist in the schema? +for my $table (sort grep { $ptable{$_} == 2 } keys %ptable) { + print qq{Table "$table" ($ptable{$table}) used in parserTest.inc, but not found in schema\n}; +} + for my $oldfile (@old) { ## Begin non-standard indent @@ -209,6 +251,171 @@ while (<$newfh>) { } } +## Which column types are okay to map from mysql to postgres? +my $COLMAP = q{ +## INTS: +tinyint SMALLINT +int INTEGER SERIAL +bigint BIGINT +real NUMERIC +float NUMERIC + +## TEXT: +varchar(32) TEXT +varchar(70) TEXT +varchar(255) TEXT +varchar TEXT +text TEXT +tinytext TEXT +ENUM TEXT + +## TIMESTAMPS: +varbinary(14) TIMESTAMPTZ +binary(14) TIMESTAMPTZ +datetime TIMESTAMPTZ +timestamp TIMESTAMPTZ + +## BYTEA: +mediumblob BYTEA + +## OTHER: +bool CHAR # Sigh + +}; +## Allow specific exceptions to the above +my $COLMAPOK = q{ +## User inputted text strings: +ar_comment tinyblob TEXT +fa_description tinyblob TEXT +img_description tinyblob TEXT +ipb_reason tinyblob TEXT +log_action varbinary(10) TEXT +oi_description tinyblob TEXT +rev_comment tinyblob TEXT +rc_log_action varbinary(255) TEXT +rc_log_type varbinary(255) TEXT + +## Simple text-only strings: +ar_flags tinyblob TEXT +fa_minor_mime varbinary(32) TEXT +fa_storage_group varbinary(16) TEXT # Just 'deleted' for now, should stay plain text +fa_storage_key varbinary(64) TEXT # sha1 plus text extension +ipb_address tinyblob TEXT # IP address or username +ipb_range_end tinyblob TEXT # hexadecimal +ipb_range_start tinyblob TEXT # hexadecimal +img_minor_mime varbinary(32) TEXT +img_sha1 varbinary(32) TEXT +job_cmd varbinary(60) TEXT # Should we limit to 60 as well? +keyname varbinary(255) TEXT # No tablename prefix (objectcache) +ll_lang varbinary(20) TEXT # Language code +log_params blob TEXT # LF separated list of args +log_type varbinary(10) TEXT +oi_minor_mime varbinary(32) TEXT +oi_sha1 varbinary(32) TEXT +old_flags tinyblob TEXT +old_text mediumblob TEXT +page_restrictions tinyblob TEXT # CSV string +pf_server varchar(30) TEXT +pr_level varbinary(60) TEXT +pr_type varbinary(60) TEXT +pt_create_perm varbinary(60) TEXT +pt_reason tinyblob TEXT +qc_type varbinary(32) TEXT +qcc_type varbinary(32) TEXT +qci_type varbinary(32) TEXT +rc_params blob TEXT +ug_group varbinary(16) TEXT +user_email_token binary(32) TEXT +user_ip varbinary(40) TEXT +user_newpassword tinyblob TEXT +user_options blob TEXT +user_password tinyblob TEXT +user_token binary(32) TEXT + +## Text URLs: +el_index blob TEXT +el_to blob TEXT +iw_url blob TEXT +tb_url blob TEXT +tc_url varbinary(255) TEXT + +## Deprecated or not yet used: +ar_text mediumblob TEXT +job_params blob TEXT +log_deleted tinyint INTEGER # Not used yet, but keep it INTEGER for safety +rc_type tinyint CHAR + +## Number tweaking: +fa_bits int SMALLINT # bits per pixel +fa_height int SMALLINT +fa_width int SMALLINT # Hope we don't see an image this wide... +hc_id int BIGINT # Odd that site_stats is all bigint... +img_bits int SMALLINT # bits per image should stay sane +oi_bits int SMALLINT + +## True binary fields, usually due to gzdeflate and/or serialize: +math_inputhash varbinary(16) BYTEA +math_outputhash varbinary(16) BYTEA + +## Namespaces: not need for such a high range +ar_namespace int SMALLINT +job_namespace int SMALLINT +log_namespace int SMALLINT +page_namespace int SMALLINT +pl_namespace int SMALLINT +pt_namespace int SMALLINT +qc_namespace int SMALLINT +rc_namespace int SMALLINT +rd_namespace int SMALLINT +tl_namespace int SMALLINT +wl_namespace int SMALLINT + +## "Bools" +ar_minor_edit tinyint CHAR +iw_trans tinyint CHAR +page_is_new tinyint CHAR +page_is_redirect tinyint CHAR +rc_bot tinyint CHAR +rc_deleted tinyint CHAR +rc_minor tinyint CHAR +rc_new tinyint CHAR +rc_patrolled tinyint CHAR +rev_deleted tinyint CHAR +rev_minor_edit tinyint CHAR + +## Easy enough to change if a wiki ever does grow this big: +ss_good_articles bigint INTEGER +ss_total_edits bigint INTEGER +ss_total_pages bigint INTEGER +ss_total_views bigint INTEGER +ss_users bigint INTEGER + +## True IP - keep an eye on these, coders tend to make textual assumptions +rc_ip varbinary(40) CIDR # Want to keep an eye on this + +## Others: +tc_time int TIMESTAMPTZ + + +}; + +my %colmap; +for (split /\n/ => $COLMAP) { + next unless /^\w/; + s/(.*?)#.*/$1/; + my ($col,@maps) = split / +/, $_; + for (@maps) { + $colmap{$col}{$_} = 1; + } +} + +my %colmapok; +for (split /\n/ => $COLMAPOK) { + next unless /^\w/; + my ($col,$old,$new) = split / +/, $_; + $colmapok{$col}{$old}{$new} = 1; +} + ## Old but not new for my $t (sort keys %{$old{$oldfile}}) { if (!exists $new{$t} and !exists $ok{OLD}{$t}) { @@ -218,6 +425,7 @@ for my $t (sort keys %{$old{$oldfile}}) { next if exists $ok{OLD}{$t} and !$ok{OLD}{$t}; my $newt = exists $ok{OLD}{$t} ? $ok{OLD}{$t} : $t; my $oldcol = $old{$oldfile}{$t}{column}; + my $oldcolfull = $old{$oldfile}{$t}{columnfull}; my $newcol = $new{$newt}{column}; for my $c (keys %$oldcol) { if (!exists $newcol->{$c}) { @@ -225,11 +433,22 @@ for my $t (sort keys %{$old{$oldfile}}) { next; } } - for my $c (keys %$newcol) { + for my $c (sort keys %$newcol) { if (!exists $oldcol->{$c}) { print "Column $t.$c not in $oldfile\n"; next; } + ## Column types (roughly) match up? + my $new = $newcol->{$c}; + my $old = $oldcolfull->{$c}; + + ## Known exceptions: + next if exists $colmapok{$c}{$old}{$new}; + + $old =~ s/ENUM.*/ENUM/; + if (! exists $colmap{$old}{$new}) { + print "Column types for $t.$c do not match: $old does not map to $new\n"; + } } } ## New but not old: diff --git a/maintenance/postgres/mediawiki_mysql2postgres.pl b/maintenance/postgres/mediawiki_mysql2postgres.pl index 733af08f..75749dd5 100644 --- a/maintenance/postgres/mediawiki_mysql2postgres.pl +++ b/maintenance/postgres/mediawiki_mysql2postgres.pl @@ -1,7 +1,14 @@ #!/usr/bin/perl ## Convert data from a MySQL mediawiki database into a Postgres mediawiki database -## svn: $Id: mediawiki_mysql2postgres.pl 21254 2007-04-14 02:10:03Z greg $ +## svn: $Id: mediawiki_mysql2postgres.pl 26564 2007-10-10 01:24:18Z greg $ + +## NOTE: It is probably easier to dump your wiki using maintenance/dumpBackup.php +## and then import it with maintenance/importDump.php + +## If having UTF-8 problems, there are reports that adding --compatible=postgresql +## may help. + use strict; use warnings; @@ -175,7 +182,7 @@ $MYSQLSOCKET and $conninfo .= "\n-- socket $MYSQLSOCKET"; print qq{ -- Dump of MySQL Mediawiki tables for import into a Postgres Mediawiki schema -- Performed by the program: $0 --- Version: $VERSION (subversion }.q{$LastChangedRevision: 21254 $}.qq{) +-- Version: $VERSION (subversion }.q{$LastChangedRevision: 26564 $}.qq{) -- Author: Greg Sabino Mullane <greg\@turnstep.com> Comments welcome -- -- This file was created: $now diff --git a/maintenance/postgres/tables.sql b/maintenance/postgres/tables.sql index 8f99c84a..dc1d7e92 100644 --- a/maintenance/postgres/tables.sql +++ b/maintenance/postgres/tables.sql @@ -5,7 +5,7 @@ -- For information about each table, please see the notes in maintenance/tables.sql -- Please make sure all dollar-quoting uses $mw$ at the start of the line -- We can't use SERIAL everywhere: the sequence names are hard-coded into the PHP --- TODO: Change CHAR to BOOL (still needed as CHAR due to some PHP code) +-- TODO: Change CHAR/SMALLINT to BOOL (still needed as CHAR due to some PHP code) BEGIN; SET client_min_messages = 'ERROR'; @@ -18,9 +18,9 @@ CREATE TABLE mwuser ( -- replace reserved word 'user' user_password TEXT, user_newpassword TEXT, user_newpass_time TIMESTAMPTZ, - user_token CHAR(32), + user_token TEXT, user_email TEXT, - user_email_token CHAR(32), + user_email_token TEXT, user_email_token_expires TIMESTAMPTZ, user_email_authenticated TIMESTAMPTZ, user_options TEXT, @@ -55,8 +55,8 @@ CREATE TABLE page ( page_title TEXT NOT NULL, page_restrictions TEXT, page_counter BIGINT NOT NULL DEFAULT 0, - page_is_redirect CHAR NOT NULL DEFAULT 0, - page_is_new CHAR NOT NULL DEFAULT 0, + page_is_redirect SMALLINT NOT NULL DEFAULT 0, + page_is_new SMALLINT NOT NULL DEFAULT 0, page_random NUMERIC(15,14) NOT NULL DEFAULT RANDOM(), page_touched TIMESTAMPTZ, page_latest INTEGER NOT NULL, -- FK? @@ -91,8 +91,8 @@ CREATE TABLE revision ( rev_user INTEGER NOT NULL REFERENCES mwuser(user_id) ON DELETE RESTRICT, rev_user_text TEXT NOT NULL, rev_timestamp TIMESTAMPTZ NOT NULL, - rev_minor_edit CHAR NOT NULL DEFAULT '0', - rev_deleted CHAR NOT NULL DEFAULT '0', + rev_minor_edit SMALLINT NOT NULL DEFAULT 0, + rev_deleted SMALLINT NOT NULL DEFAULT 0, rev_len INTEGER NULL, rev_parent_id INTEGER NULL ); @@ -127,17 +127,17 @@ ALTER TABLE page_restrictions ADD CONSTRAINT page_restrictions_pk PRIMARY KEY (p CREATE TABLE archive ( ar_namespace SMALLINT NOT NULL, ar_title TEXT NOT NULL, - ar_text TEXT, + ar_text TEXT, -- technically should be bytea, but not used anymore ar_page_id INTEGER NULL, ar_comment TEXT, ar_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL, ar_user_text TEXT NOT NULL, ar_timestamp TIMESTAMPTZ NOT NULL, - ar_minor_edit CHAR NOT NULL DEFAULT '0', + ar_minor_edit SMALLINT NOT NULL DEFAULT 0, ar_flags TEXT, ar_rev_id INTEGER, ar_text_id INTEGER, - ar_deleted INTEGER NOT NULL DEFAULT 0, + ar_deleted SMALLINT NOT NULL DEFAULT 0, ar_len INTEGER NULL ); CREATE INDEX archive_name_title_timestamp ON archive (ar_namespace,ar_title,ar_timestamp); @@ -161,7 +161,7 @@ CREATE UNIQUE INDEX pagelink_unique ON pagelinks (pl_from,pl_namespace,pl_title) CREATE TABLE templatelinks ( tl_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE, - tl_namespace TEXT NOT NULL, + tl_namespace SMALLINT NOT NULL, tl_title TEXT NOT NULL ); CREATE UNIQUE INDEX templatelinks_unique ON templatelinks (tl_namespace,tl_title,tl_from); @@ -202,7 +202,7 @@ CREATE TABLE site_stats ( ss_row_id INTEGER NOT NULL UNIQUE, ss_total_views INTEGER DEFAULT 0, ss_total_edits INTEGER DEFAULT 0, - ss_good_articles INTEGER DEFAULT 0, + ss_good_articles INTEGER DEFAULT 0, ss_total_pages INTEGER DEFAULT -1, ss_users INTEGER DEFAULT -1, ss_admins INTEGER DEFAULT -1, @@ -222,15 +222,15 @@ CREATE TABLE ipblocks ( ipb_by INTEGER NOT NULL REFERENCES mwuser(user_id) ON DELETE CASCADE, ipb_reason TEXT NOT NULL, ipb_timestamp TIMESTAMPTZ NOT NULL, - ipb_auto CHAR NOT NULL DEFAULT '0', - ipb_anon_only CHAR NOT NULL DEFAULT '0', - ipb_create_account CHAR NOT NULL DEFAULT '1', - ipb_enable_autoblock CHAR NOT NULL DEFAULT '1', + ipb_auto SMALLINT NOT NULL DEFAULT 0, + ipb_anon_only SMALLINT NOT NULL DEFAULT 0, + ipb_create_account SMALLINT NOT NULL DEFAULT 1, + ipb_enable_autoblock SMALLINT NOT NULL DEFAULT 1, ipb_expiry TIMESTAMPTZ NOT NULL, ipb_range_start TEXT, ipb_range_end TEXT, - ipb_deleted INTEGER NOT NULL DEFAULT 0, - ipb_block_email CHAR NOT NULL DEFAULT '0' + ipb_deleted SMALLINT NOT NULL DEFAULT 0, + ipb_block_email SMALLINT NOT NULL DEFAULT 0 ); CREATE INDEX ipb_address ON ipblocks (ipb_address); @@ -243,7 +243,7 @@ CREATE TABLE image ( img_size INTEGER NOT NULL, img_width INTEGER NOT NULL, img_height INTEGER NOT NULL, - img_metadata TEXT, + img_metadata BYTEA NOT NULL DEFAULT '', img_bits SMALLINT, img_media_type TEXT, img_major_mime TEXT DEFAULT 'unknown', @@ -259,7 +259,7 @@ CREATE INDEX img_timestamp_idx ON image (img_timestamp); CREATE INDEX img_sha1 ON image (img_sha1); CREATE TABLE oldimage ( - oi_name TEXT NOT NULL REFERENCES image(img_name), + oi_name TEXT NOT NULL, oi_archive_name TEXT NOT NULL, oi_size INTEGER NOT NULL, oi_width INTEGER NOT NULL, @@ -273,9 +273,10 @@ CREATE TABLE oldimage ( oi_media_type TEXT NULL, oi_major_mime TEXT NOT NULL DEFAULT 'unknown', oi_minor_mime TEXT NOT NULL DEFAULT 'unknown', - oi_deleted CHAR NOT NULL DEFAULT '0', + oi_deleted SMALLINT NOT NULL DEFAULT 0, oi_sha1 TEXT NOT NULL DEFAULT '' ); +ALTER TABLE oldimage ADD CONSTRAINT oldimage_oi_name_fkey_cascade FOREIGN KEY (oi_name) REFERENCES image(img_name) ON DELETE CASCADE; CREATE INDEX oi_name_timestamp ON oldimage (oi_name,oi_timestamp); CREATE INDEX oi_name_archive_name ON oldimage (oi_name,oi_archive_name); CREATE INDEX oi_sha1 ON oldimage (oi_sha1); @@ -285,15 +286,15 @@ CREATE TABLE filearchive ( fa_id SERIAL NOT NULL PRIMARY KEY, fa_name TEXT NOT NULL, fa_archive_name TEXT, - fa_storage_group VARCHAR(16), - fa_storage_key CHAR(64), + fa_storage_group TEXT, + fa_storage_key TEXT, fa_deleted_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL, fa_deleted_timestamp TIMESTAMPTZ NOT NULL, fa_deleted_reason TEXT, - fa_size SMALLINT NOT NULL, - fa_width SMALLINT NOT NULL, - fa_height SMALLINT NOT NULL, - fa_metadata TEXT, + fa_size INTEGER NOT NULL, + fa_width INTEGER NOT NULL, + fa_height INTEGER NOT NULL, + fa_metadata BYTEA NOT NULL DEFAULT '', fa_bits SMALLINT, fa_media_type TEXT, fa_major_mime TEXT DEFAULT 'unknown', @@ -302,7 +303,7 @@ CREATE TABLE filearchive ( fa_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL, fa_user_text TEXT NOT NULL, fa_timestamp TIMESTAMPTZ, - fa_deleted INTEGER NOT NULL DEFAULT 0 + fa_deleted SMALLINT NOT NULL DEFAULT 0 ); CREATE INDEX fa_name_time ON filearchive (fa_name, fa_timestamp); CREATE INDEX fa_dupe ON filearchive (fa_storage_group, fa_storage_key); @@ -320,20 +321,20 @@ CREATE TABLE recentchanges ( rc_namespace SMALLINT NOT NULL, rc_title TEXT NOT NULL, rc_comment TEXT, - rc_minor CHAR NOT NULL DEFAULT '0', - rc_bot CHAR NOT NULL DEFAULT '0', - rc_new CHAR NOT NULL DEFAULT '0', + rc_minor SMALLINT NOT NULL DEFAULT 0, + rc_bot SMALLINT NOT NULL DEFAULT 0, + rc_new SMALLINT NOT NULL DEFAULT 0, rc_cur_id INTEGER NULL REFERENCES page(page_id) ON DELETE SET NULL, rc_this_oldid INTEGER NOT NULL, rc_last_oldid INTEGER NOT NULL, - rc_type CHAR NOT NULL DEFAULT '0', + rc_type SMALLINT NOT NULL DEFAULT 0, rc_moved_to_ns SMALLINT, rc_moved_to_title TEXT, - rc_patrolled CHAR NOT NULL DEFAULT '0', + rc_patrolled SMALLINT NOT NULL DEFAULT 0, rc_ip CIDR, rc_old_len INTEGER, rc_new_len INTEGER, - rc_deleted INTEGER NOT NULL DEFAULT 0, + rc_deleted SMALLINT NOT NULL DEFAULT 0, rc_logid INTEGER NOT NULL DEFAULT 0, rc_log_type TEXT, rc_log_action TEXT, @@ -365,16 +366,16 @@ CREATE TABLE math ( CREATE TABLE interwiki ( - iw_prefix TEXT NOT NULL UNIQUE, - iw_url TEXT NOT NULL, - iw_local CHAR NOT NULL, - iw_trans CHAR NOT NULL DEFAULT '0' + iw_prefix TEXT NOT NULL UNIQUE, + iw_url TEXT NOT NULL, + iw_local SMALLINT NOT NULL, + iw_trans SMALLINT NOT NULL DEFAULT 0 ); CREATE TABLE querycache ( qc_type TEXT NOT NULL, - qc_value SMALLINT NOT NULL, + qc_value INTEGER NOT NULL, qc_namespace SMALLINT NOT NULL, qc_title TEXT NOT NULL ); @@ -387,7 +388,7 @@ CREATE TABLE querycache_info ( CREATE TABLE querycachetwo ( qcc_type TEXT NOT NULL, - qcc_value SMALLINT NOT NULL DEFAULT 0, + qcc_value INTEGER NOT NULL DEFAULT 0, qcc_namespace INTEGER NOT NULL DEFAULT 0, qcc_title TEXT NOT NULL DEFAULT '', qcc_namespacetwo INTEGER NOT NULL DEFAULT 0, @@ -398,7 +399,7 @@ CREATE INDEX querycachetwo_title ON querycachetwo (qcc_type,qcc_namespace,q CREATE INDEX querycachetwo_titletwo ON querycachetwo (qcc_type,qcc_namespacetwo,qcc_titletwo); CREATE TABLE objectcache ( - keyname CHAR(255) UNIQUE, + keyname TEXT UNIQUE, value BYTEA NOT NULL DEFAULT '', exptime TIMESTAMPTZ NOT NULL ); @@ -422,7 +423,7 @@ CREATE TABLE logging ( log_title TEXT NOT NULL, log_comment TEXT, log_params TEXT, - log_deleted INTEGER NOT NULL DEFAULT 0 + log_deleted SMALLINT NOT NULL DEFAULT 0 ); CREATE INDEX logging_type_name ON logging (log_type, log_timestamp); CREATE INDEX logging_user_time ON logging (log_timestamp, log_user); @@ -451,15 +452,16 @@ CREATE TABLE job ( CREATE INDEX job_cmd_namespace_title ON job (job_cmd, job_namespace, job_title); -- Tsearch2 2 stuff. Will fail if we don't have proper access to the tsearch2 tables +-- Note: if version 8.3 or higher, we remove the 'default' arg ALTER TABLE page ADD titlevector tsvector; CREATE FUNCTION ts2_page_title() RETURNS TRIGGER LANGUAGE plpgsql AS $mw$ BEGIN IF TG_OP = 'INSERT' THEN - NEW.titlevector = to_tsvector('default',NEW.page_title); + NEW.titlevector = to_tsvector('default',REPLACE(NEW.page_title,'/',' ')); ELSIF NEW.page_title != OLD.page_title THEN - NEW.titlevector := to_tsvector('default',NEW.page_title); + NEW.titlevector := to_tsvector('default',REPLACE(NEW.page_title,'/',' ')); END IF; RETURN NEW; END; @@ -486,11 +488,12 @@ CREATE TRIGGER ts2_page_text BEFORE INSERT OR UPDATE ON pagecontent FOR EACH ROW EXECUTE PROCEDURE ts2_page_text(); -- These are added by the setup script due to version compatibility issues --- If using 8.1, switch from "gin" to "gist" --- CREATE INDEX ts2_page_title ON page USING gin(titlevector); --- CREATE INDEX ts2_page_text ON pagecontent USING gin(textvector); +-- If using 8.1, we switch from "gin" to "gist" -CREATE FUNCTION add_interwiki (TEXT,INT,CHAR) RETURNS INT LANGUAGE SQL AS +CREATE INDEX ts2_page_title ON page USING gin(titlevector); +CREATE INDEX ts2_page_text ON pagecontent USING gin(textvector); + +CREATE FUNCTION add_interwiki (TEXT,INT,SMALLINT) RETURNS INT LANGUAGE SQL AS $mw$ INSERT INTO interwiki (iw_prefix, iw_url, iw_local) VALUES ($1,$2,$3); SELECT 1; @@ -505,6 +508,16 @@ CREATE TABLE profiling ( ); CREATE UNIQUE INDEX pf_name_server ON profiling (pf_name, pf_server); +CREATE TABLE protected_titles ( + pt_namespace SMALLINT NOT NULL, + pt_title TEXT NOT NULL, + pt_user INTEGER NULL REFERENCES mwuser(user_id) ON DELETE SET NULL, + pt_reason TEXT NULL, + pt_timestamp TIMESTAMPTZ NOT NULL, + pt_expiry TIMESTAMPTZ NULL, + pt_create_perm TEXT NOT NULL DEFAULT '' +); +CREATE UNIQUE INDEX protected_titles_unique ON protected_titles(pt_namespace, pt_title); CREATE TABLE mediawiki_version ( type TEXT NOT NULL, @@ -525,6 +538,5 @@ CREATE TABLE mediawiki_version ( ); INSERT INTO mediawiki_version (type,mw_version,sql_version,sql_date) - VALUES ('Creation','??','$LastChangedRevision: 25527 $','$LastChangedDate: 2007-09-05 04:14:18 -0400 (Wed, 05 Sep 2007) $'); - + VALUES ('Creation','??','$LastChangedRevision: 30800 $','$LastChangedDate: 2008-02-10 08:50:38 -0800 (Sun, 10 Feb 2008) $'); diff --git a/maintenance/preprocessorFuzzTest.php b/maintenance/preprocessorFuzzTest.php new file mode 100644 index 00000000..d814c4fe --- /dev/null +++ b/maintenance/preprocessorFuzzTest.php @@ -0,0 +1,233 @@ +<?php + +require_once( dirname( __FILE__ ). '/../maintenance/commandLine.inc' ); + +$wgHooks['BeforeParserFetchTemplateAndtitle'][] = 'PPFuzzTester::templateHook'; + +class PPFuzzTester { + var $hairs = array( + '[[', ']]', '{{', '{{', '}}', '}}', '{{{', '}}}', + '<', '>', '<nowiki', '<gallery', '</nowiki>', '</gallery>', '<nOwIkI>', '</NoWiKi>', + '<!--' , '-->', + "\n==", "==\n", + '|', '=', "\n", ' ', "\t", "\x7f", + '~~', '~~~', '~~~~', 'subst:', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + + // extensions + //'<ref>', '</ref>', '<references/>', + ); + var $minLength = 0; + var $maxLength = 20; + var $maxTemplates = 5; + //var $outputTypes = array( 'OT_HTML', 'OT_WIKI', 'OT_PREPROCESS' ); + var $entryPoints = array( 'testSrvus', 'testPst', 'testPreprocess' ); + var $verbose = false; + static $currentTest = false; + + function execute() { + if ( !file_exists( 'results' ) ) { + mkdir( 'results' ); + } + if ( !is_dir( 'results' ) ) { + echo "Unable to create 'results' directory\n"; + exit( 1 ); + } + $overallStart = microtime( true ); + $reportInterval = 1000; + for ( $i = 1; true; $i++ ) { + $t = -microtime( true ); + try { + self::$currentTest = new PPFuzzTest( $this ); + self::$currentTest->execute(); + $passed = 'passed'; + } catch ( MWException $e ) { + $testReport = self::$currentTest->getReport(); + $exceptionReport = $e->getText(); + $hash = md5( $testReport ); + file_put_contents( "results/ppft-$hash.in", serialize( self::$currentTest ) ); + file_put_contents( "results/ppft-$hash.fail", + "Input:\n$testReport\n\nException report:\n$exceptionReport\n" ); + print "Test $hash failed\n"; + $passed = 'failed'; + } + $t += microtime( true ); + + if ( $this->verbose ) { + printf( "Test $passed in %.3f seconds\n", $t ); + print self::$currentTest->getReport(); + } + + $reportMetric = ( microtime( true ) - $overallStart ) / $i * $reportInterval; + if ( $reportMetric > 25 ) { + if ( substr( $reportInterval, 0, 1 ) === '1' ) { + $reportInterval /= 2; + } else { + $reportInterval /= 5; + } + } elseif ( $reportMetric < 4 ) { + if ( substr( $reportInterval, 0, 1 ) === '1' ) { + $reportInterval *= 5; + } else { + $reportInterval *= 2; + } + } + if ( $i % $reportInterval == 0 ) { + print "$i tests done\n"; + /* + $testReport = self::$currentTest->getReport(); + $filename = 'results/ppft-' . md5( $testReport ) . '.pass'; + file_put_contents( $filename, "Input:\n$testReport\n" );*/ + } + } + wfLogProfilingData(); + } + + function makeInputText( $max = false ) { + if ( $max === false ) { + $max = $this->maxLength; + } + $length = mt_rand( $this->minLength, $max ); + $s = ''; + for ( $i = 0; $i < $length; $i++ ) { + $hairIndex = mt_rand( 0, count( $this->hairs ) - 1 ); + $s .= $this->hairs[$hairIndex]; + } + // Send through the UTF-8 normaliser + // This resolves a few differences between the old preprocessor and the + // XML-based one, which doesn't like illegals and converts line endings. + // It's done by the MW UI, so it's a reasonably legitimate thing to do. + $s = UtfNormal::cleanUp( $s ); + return $s; + } + + function makeTitle() { + return Title::newFromText( mt_rand( 0, 1000000 ), mt_rand( 0, 10 ) ); + } + + /* + function pickOutputType() { + $count = count( $this->outputTypes ); + return $this->outputTypes[ mt_rand( 0, $count - 1 ) ]; + }*/ + + function pickEntryPoint() { + $count = count( $this->entryPoints ); + return $this->entryPoints[ mt_rand( 0, $count - 1 ) ]; + } +} + +class PPFuzzTest { + var $templates, $mainText, $title, $entryPoint, $output; + + function __construct( $tester ) { + global $wgMaxSigChars; + $this->parent = $tester; + $this->mainText = $tester->makeInputText(); + $this->title = $tester->makeTitle(); + //$this->outputType = $tester->pickOutputType(); + $this->entryPoint = $tester->pickEntryPoint(); + $this->nickname = $tester->makeInputText( $wgMaxSigChars + 10); + $this->fancySig = (bool)mt_rand( 0, 1 ); + $this->templates = array(); + } + + function templateHook( $title ) { + $titleText = $title->getPrefixedDBkey(); + + if ( !isset( $this->templates[$titleText] ) ) { + $finalTitle = $title; + if ( count( $this->templates ) >= $this->parent->maxTemplates ) { + // Too many templates + $text = false; + } else { + if ( !mt_rand( 0, 1 ) ) { + // Redirect + $finalTitle = $this->parent->makeTitle(); + } + if ( !mt_rand( 0, 5 ) ) { + // Doesn't exist + $text = false; + } else { + $text = $this->parent->makeInputText(); + } + } + $this->templates[$titleText] = array( + 'text' => $text, + 'finalTitle' => $finalTitle ); + } + return $this->templates[$titleText]; + } + + function execute() { + global $wgParser, $wgUser; + + $wgUser = new PPFuzzUser; + $wgUser->mName = 'Fuzz'; + $wgUser->mFrom = 'name'; + $wgUser->ppfz_test = $this; + + $options = new ParserOptions; + $options->setTemplateCallback( array( $this, 'templateHook' ) ); + $options->setTimestamp( wfTimestampNow() ); + $this->output = call_user_func( array( $wgParser, $this->entryPoint ), $this->mainText, $this->title->getPrefixedText(), $options ); + return $this->output; + } + + function getReport() { + $s = "Title: " . $this->title->getPrefixedDBkey() . "\n" . +// "Output type: {$this->outputType}\n" . + "Entry point: {$this->entryPoint}\n" . + "User: " . ( $this->fancySig ? 'fancy' : 'no-fancy' ) . ' ' . var_export( $this->nickname, true ) . "\n" . + "Main text: " . var_export( $this->mainText, true ) . "\n"; + foreach ( $this->templates as $titleText => $template ) { + $finalTitle = $template['finalTitle']; + if ( $finalTitle != $titleText ) { + $s .= "[[$titleText]] -> [[$finalTitle]]: " . var_export( $template['text'], true ) . "\n"; + } else { + $s .= "[[$titleText]]: " . var_export( $template['text'], true ) . "\n"; + } + } + $s .= "Output: " . var_export( $this->output, true ) . "\n"; + return $s; + } +} + +class PPFuzzUser extends User { + var $ppfz_test; + + function load() { + if ( $this->mDataLoaded ) { + return; + } + $this->mDataLoaded = true; + $this->loadDefaults( $this->mName ); + } + + function getOption( $option, $defaultOverride = '' ) { + if ( $option === 'fancysig' ) { + return $this->ppfz_test->fancySig; + } elseif ( $option === 'nickname' ) { + return $this->ppfz_test->nickname; + } else { + return parent::getOption( $option, $defaultOverride ); + } + } +} + +ini_set( 'memory_limit', '50M' ); +if ( isset( $args[0] ) ) { + $testText = file_get_contents( $args[0] ); + if ( !$testText ) { + print "File not found\n"; + exit( 1 ); + } + $test = unserialize( $testText ); + $result = $test->execute(); + print "Test passed.\n"; +} else { + $tester = new PPFuzzTester; + $tester->verbose = isset( $options['verbose'] ); + $tester->execute(); +} diff --git a/maintenance/rebuildInterwiki.inc b/maintenance/rebuildInterwiki.inc index d85612bd..a14f8897 100644 --- a/maintenance/rebuildInterwiki.inc +++ b/maintenance/rebuildInterwiki.inc @@ -28,7 +28,7 @@ class Site { } } -function getRebuildInterwikiSQL() { +function makeInterwikiSQL( $destDir ) { global $langlist, $languageAliases, $prefixRewrites; # Multi-language sites @@ -41,6 +41,7 @@ function getRebuildInterwikiSQL() { 'wikinews' => new Site( 'wikinews', 'n', 'wikinews.org' ), 'wikisource' => new Site( 'wikisource', 's', 'wikisource.org' ), 'wikimedia' => new Site( 'wikimedia', 'chapter', 'wikimedia.org' ), + 'wikiversity' => new Site( 'wikiversity', 'v', 'wikiversity.org' ), ); # List of language prefixes likely to be found in multi-language sites @@ -123,10 +124,10 @@ function getRebuildInterwikiSQL() { } } - $sql = "-- Generated by rebuildInterwiki.php"; foreach ( $dblist as $db ) { + $sql = "-- Generated by rebuildInterwiki.php"; if ( isset( $specials[$db] ) ) { # Special wiki # Has interwiki links and interlanguage links to wikipedia @@ -211,10 +212,10 @@ function getRebuildInterwikiSQL() { foreach ( $extraLinks as $link ){ $sql .= makeLink( $link, $first, $db ); } - $sql .= ";\n\n"; + $sql .= ";\n"; } + file_put_contents( "$destDir/$db.sql", $sql ); } - return $sql; } # ------------------------------------------------------------------------------------------ diff --git a/maintenance/rebuildInterwiki.php b/maintenance/rebuildInterwiki.php index bc0a0008..08968421 100644 --- a/maintenance/rebuildInterwiki.php +++ b/maintenance/rebuildInterwiki.php @@ -9,22 +9,18 @@ /** */ $oldCwd = getcwd(); -$optionsWithArgs = array( "o" ); +$optionsWithArgs = array( "d" ); include_once( "commandLine.inc" ); include_once( "rebuildInterwiki.inc" ); chdir( $oldCwd ); -$sql = getRebuildInterwikiSQL(); - # Output -if ( isset( $options['o'] ) ) { - # To file specified with -o - $file = fopen( $options['o'], "w" ); - fwrite( $file, $sql ); - fclose( $file ); +if ( isset( $options['d'] ) ) { + $destDir = $options['d']; } else { - # To stdout - print $sql; + $destDir = '/home/wikipedia/conf/interwiki/sql'; } +echo "Making new interwiki SQL files in $destDir\n"; +makeInterwikiSQL( $destDir ); diff --git a/maintenance/rebuildall.php b/maintenance/rebuildall.php index 002e6fad..1c2647b2 100644 --- a/maintenance/rebuildall.php +++ b/maintenance/rebuildall.php @@ -22,8 +22,7 @@ rebuildTextIndex( $database ); createTextIndex( $database ); print "\n\n** Rebuilding recentchanges table:\n"; -rebuildRecentChangesTablePass1(); -rebuildRecentChangesTablePass2(); +rebuildRecentChangesTable(); # Doesn't work anymore # rebuildLinkTables(); diff --git a/maintenance/rebuildmessages.php b/maintenance/rebuildmessages.php new file mode 100644 index 00000000..dea70ef8 --- /dev/null +++ b/maintenance/rebuildmessages.php @@ -0,0 +1,17 @@ +<?php + +require_once( 'commandLine.inc' ); + +if( $wgLocalDatabases ) { + $databases = $wgLocalDatabases; +} else { + $databases = array( $wgDBname ); +} + +foreach( $databases as $db ) { + echo "Deleting message cache for {$db}... "; + $wgMessageCache->mMemc->delete( "{$db}:messages" ); + if( $wgEnableSidebarCache ) + $wgMessageCache->mMemc->delete( "{$db}:sidebar" ); + echo "Deleted\n"; +}
\ No newline at end of file diff --git a/maintenance/rebuildrecentchanges.inc b/maintenance/rebuildrecentchanges.inc index 51018153..8b2c7805 100644 --- a/maintenance/rebuildrecentchanges.inc +++ b/maintenance/rebuildrecentchanges.inc @@ -6,12 +6,18 @@ * @addtogroup Maintenance */ +/** Public entry; more passes might come in! :) */ +function rebuildRecentChangesTable() { + rebuildRecentChangesTablePass1(); + rebuildRecentChangesTablePass2(); + rebuildRecentChangesTablePass3(); +} + /** */ function rebuildRecentChangesTablePass1() { $fname = 'rebuildRecentChangesTablePass1'; $dbw = wfGetDB( DB_MASTER ); - extract( $dbw->tableNames( 'recentchanges', 'cur', 'old' ) ); $dbw->delete( 'recentchanges', '*' ); @@ -103,15 +109,15 @@ function rebuildRecentChangesTablePass3() $dbw = wfGetDB( DB_MASTER ); - list ($recentchanges, $usergroups) = $dbw->tableNamesN( 'recentchanges', 'user_groups' ); + list($recentchanges,$usergroups,$user) = $dbw->tableNamesN( 'recentchanges', 'user_groups', 'user' ); $botgroups = $autopatrolgroups = array(); foreach( $wgGroupPermissions as $group => $rights ) { if( isset( $rights['bot'] ) && $rights['bot'] == true ) { - $botgroups[] = "'" . $dbw->strencode( $group ) . "'"; + $botgroups[] = $dbw->addQuotes( $group ); } if( $wgUseRCPatrol && isset( $rights['autopatrol'] ) && $rights['autopatrol'] == true ) { - $autopatrolgroups[] = "'" . $dbw->strencode( $group ) . "'"; + $autopatrolgroups[] = $dbw->addQuotes( $group ); } } # Flag our recent bot edits @@ -121,20 +127,19 @@ function rebuildRecentChangesTablePass3() print( "Flagging bot account edits...\n" ); - # Find all users in RC that are bots - $sql = "SELECT DISTINCT rc_user FROM $recentchanges " . - "LEFT JOIN $usergroups ON rc_user=ug_user " . - "WHERE ug_group IN($botwhere)"; + # Find all users that are bots + $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . + "WHERE ug_group IN($botwhere) AND user_id = ug_user"; $res = $dbw->query( $sql, DB_MASTER ); while( $obj = $dbw->fetchObject( $res ) ) { - $botusers[] = $obj->rc_user; + $botusers[] = $dbw->addQuotes( $obj->user_name ); } # Fill in the rc_bot field if( !empty($botusers) ) { $botwhere = implode(',',$botusers); $sql2 = "UPDATE $recentchanges SET rc_bot=1 " . - "WHERE rc_user IN($botwhere)"; + "WHERE rc_user_text IN($botwhere)"; $dbw->query( $sql2 ); } } @@ -146,20 +151,19 @@ function rebuildRecentChangesTablePass3() print( "Flagging auto-patrolled edits...\n" ); # Find all users in RC with autopatrol rights - $sql = "SELECT DISTINCT rc_user FROM $recentchanges " . - "LEFT JOIN $usergroups ON rc_user=ug_user " . - "WHERE ug_group IN($patrolwhere)"; + $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . + "WHERE ug_group IN($patrolwhere) AND user_id = ug_user"; $res = $dbw->query( $sql, DB_MASTER ); while( $obj = $dbw->fetchObject( $res ) ) { - $patrolusers[] = $obj->rc_user; + $patrolusers[] = $dbw->addQuotes( $obj->user_name ); } # Fill in the rc_patrolled field if( !empty($patrolusers) ) { $patrolwhere = implode(',',$patrolusers); $sql2 = "UPDATE $recentchanges SET rc_patrolled=1 " . - "WHERE rc_user IN($patrolwhere)"; + "WHERE rc_user_text IN($patrolwhere)"; $dbw->query( $sql2 ); } } diff --git a/maintenance/rebuildrecentchanges.php b/maintenance/rebuildrecentchanges.php index a94780e2..3c455c55 100644 --- a/maintenance/rebuildrecentchanges.php +++ b/maintenance/rebuildrecentchanges.php @@ -15,9 +15,7 @@ $wgTitle = Title::newFromText( "Rebuild recent changes script" ); $wgDBuser = $wgDBadminuser; $wgDBpassword = $wgDBadminpassword; -rebuildRecentChangesTablePass1(); -rebuildRecentChangesTablePass2(); -rebuildRecentChangesTablePass3(); // flag bot edits +rebuildRecentChangesTable(); print "Done.\n"; exit(); diff --git a/maintenance/refreshLinks.inc b/maintenance/refreshLinks.inc index 9e4eea8d..e89db8aa 100644 --- a/maintenance/refreshLinks.inc +++ b/maintenance/refreshLinks.inc @@ -19,7 +19,7 @@ function refreshLinks( $start, $newOnly = false, $maxLag = false, $end = 0, $red $wgUser->setOption('math', MW_MATH_SOURCE); # Don't generate extension images (e.g. Timeline) - $wgParser->mTagHooks = array(); + $wgParser->clearTagHooks(); # Don't generate thumbnail images $wgUseImageResize = false; diff --git a/maintenance/refreshLinks.php b/maintenance/refreshLinks.php index 34ca53c5..67fb77ae 100644 --- a/maintenance/refreshLinks.php +++ b/maintenance/refreshLinks.php @@ -1,18 +1,32 @@ <?php /** - * @todo document * @addtogroup Maintenance */ /** */ $optionsWithArgs = array( 'm', 'e' ); + require_once( "commandLine.inc" ); require_once( "refreshLinks.inc" ); +if( isset( $options['help'] ) ) { + echo <<<TEXT +Usage: php refreshLinks.php [<start>] [-e <end>] [-m <maxlag>] [--help] + + --help : This help message + --dfn-only : Delete links from nonexistent articles only + -m <number> : Maximum replication lag + <start> : First page id to refresh + -e <number> : Last page id to refresh + +TEXT; + exit(0); +} + error_reporting( E_ALL & (~E_NOTICE) ); if ( !$options['dfn-only'] ) { - if ($args[0]) { + if ( isset( $args[0] ) ) { $start = (int)$args[0]; } else { $start = 1; diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php index 4539be05..799092e1 100644 --- a/maintenance/runJobs.php +++ b/maintenance/runJobs.php @@ -1,6 +1,6 @@ <?php -$optionsWithArgs = array( 'maxjobs' ); +$optionsWithArgs = array( 'maxjobs', 'type' ); $wgUseNormalUser = true; require_once( 'commandLine.inc' ); require_once( "$IP/includes/JobQueue.php" ); @@ -20,15 +20,15 @@ $wgTitle = Title::newFromText( 'RunJobs.php' ); $dbw = wfGetDB( DB_MASTER ); $n = 0; -$conds = array(); +$conds = ''; if ($type !== false) - $conds = array('job_cmd' => $type); + $conds = "job_cmd = " . $dbw->addQuotes($type); while ( $dbw->selectField( 'job', 'count(*)', $conds, 'runJobs.php' ) ) { $offset=0; for (;;) { $job = ($type == false) ? - Job::pop($offset, $type) + Job::pop($offset) : Job::pop_type($type); if ($job == false) diff --git a/maintenance/storage/compressOld.inc b/maintenance/storage/compressOld.inc index 2da015b0..421f5304 100644 --- a/maintenance/storage/compressOld.inc +++ b/maintenance/storage/compressOld.inc @@ -15,7 +15,7 @@ function compressOldPages( $start = 0, $extdb = '' ) { print "Starting from old_id $start...\n"; $dbw = wfGetDB( DB_MASTER ); do { - $res = $dbw->select( 'text', array( 'old_id','old_flags','old_namespace','old_title','old_text' ), + $res = $dbw->select( 'text', array( 'old_id','old_flags','old_text' ), "old_id>=$start", $fname, array( 'ORDER BY' => 'old_id', 'LIMIT' => $chunksize, 'FOR UPDATE' ) ); if( $dbw->numRows( $res ) == 0 ) { break; diff --git a/maintenance/tables.sql b/maintenance/tables.sql index 72a779a9..6282f522 100644 --- a/maintenance/tables.sql +++ b/maintenance/tables.sql @@ -1168,4 +1168,17 @@ CREATE TABLE /*$wgDBprefix*/page_restrictions ( KEY pr_cascade (pr_cascade) ) /*$wgDBTableOptions*/; +-- Protected titles - nonexistent pages that have been protected +CREATE TABLE /*$wgDBprefix*/protected_titles ( + pt_namespace int NOT NULL, + pt_title varchar(255) NOT NULL, + pt_user int unsigned NOT NULL, + pt_reason tinyblob, + pt_timestamp binary(14) NOT NULL, + pt_expiry varbinary(14) NOT NULL default '', + pt_create_perm varbinary(60) NOT NULL, + PRIMARY KEY (pt_namespace,pt_title), + KEY pt_timestamp (pt_timestamp) +) /*$wgDBTableOptions*/; + -- vim: sw=2 sts=2 et diff --git a/maintenance/testRunner.postgres.sql b/maintenance/testRunner.postgres.sql new file mode 100644 index 00000000..c15300b5 --- /dev/null +++ b/maintenance/testRunner.postgres.sql @@ -0,0 +1,30 @@ +-- +-- Optional tables for parserTests recording mode +-- With --record option, success data will be saved to these tables, +-- and comparisons of what's changed from the previous run will be +-- displayed at the end of each run. +-- +-- This file is for the Postgres version of the tables +-- + +-- Note: "if exists" will not work on older versions of Postgres +DROP TABLE IF EXISTS testitem; +DROP TABLE IF EXISTS testrun; +DROP SEQUENCE IF EXISTS testrun_id_seq; + +CREATE SEQUENCE testrun_id_seq; +CREATE TABLE testrun ( + tr_id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('testrun_id_seq'), + tr_date TIMESTAMPTZ, + tr_mw_version TEXT, + tr_php_version TEXT, + tr_db_version TEXT, + tr_uname TEXT +); + +CREATE TABLE testitem ( + ti_run INTEGER NOT NULL REFERENCES testrun(tr_id) ON DELETE CASCADE, + ti_name TEXT NOT NULL, + ti_success SMALLINT NOT NULL +); +CREATE UNIQUE INDEX testitem_uniq ON testitem(ti_run, ti_name); diff --git a/maintenance/updateRestrictions.php b/maintenance/updateRestrictions.php index 9503d0b7..c8cebfc4 100644 --- a/maintenance/updateRestrictions.php +++ b/maintenance/updateRestrictions.php @@ -56,7 +56,7 @@ function migrate_page_restrictions( $db ) { # We use insert() and not replace() as Article.php replaces # page_restrictions with '' when protected in the restrictions table if ( count( $batch ) ) { - $db->insert( 'page_restrictions', $batch, __FUNCTION__ ); + $db->insert( 'page_restrictions', $batch, __FUNCTION__, array( 'IGNORE' ) ); } $blockStart += BATCH_SIZE; $blockEnd += BATCH_SIZE; diff --git a/maintenance/updateSpecialPages.php b/maintenance/updateSpecialPages.php index 424e20d9..5e0f2ceb 100644 --- a/maintenance/updateSpecialPages.php +++ b/maintenance/updateSpecialPages.php @@ -5,14 +5,15 @@ $options = array('only','help'); require_once( 'commandLine.inc' ); -require_once( 'SpecialPage.php' ); -require_once( 'QueryPage.php' ); +require_once( "$IP/includes/SpecialPage.php" ); +require_once( "$IP/includes/QueryPage.php" ); if(@$options['help']) { print "usage:updateSpecialPages.php [--help] [--only=page]\n"; print " --help : this help message\n"; print " --list : list special pages names\n"; print " --only=page : only update 'page'. Ex: --only=BrokenRedirects\n"; + print " --override : update even pages which have had updates disabled\n"; wfDie(); } @@ -28,7 +29,7 @@ foreach ( $wgQueryPages as $page ) { continue; } - if ( $wgDisableQueryPageUpdate && in_array( $special, $wgDisableQueryPageUpdate ) ) { + if ( !isset( $options['override'] ) && $wgDisableQueryPageUpdate && in_array( $special, $wgDisableQueryPageUpdate ) ) { printf("%-30s disabled\n", $special); continue; } diff --git a/maintenance/updaters.inc b/maintenance/updaters.inc index 3c972044..9d7e32cb 100644 --- a/maintenance/updaters.inc +++ b/maintenance/updaters.inc @@ -16,74 +16,120 @@ require_once 'deleteDefaultMessages.php'; # Extension updates require_once( "$IP/includes/Hooks.php" ); -$wgRenamedTables = array( -# from to patch file -# array( 'group', 'groups', 'patch-rename-group.sql' ), -); - -$wgNewTables = array( -# table patch file (in maintenance/archives) - array( 'hitcounter', 'patch-hitcounter.sql' ), - array( 'querycache', 'patch-querycache.sql' ), - array( 'objectcache', 'patch-objectcache.sql' ), - array( 'categorylinks', 'patch-categorylinks.sql' ), - array( 'logging', 'patch-logging.sql' ), - array( 'user_newtalk', 'patch-usernewtalk2.sql' ), - array( 'transcache', 'patch-transcache.sql' ), - array( 'trackbacks', 'patch-trackbacks.sql' ), - array( 'externallinks', 'patch-externallinks.sql' ), - array( 'job', 'patch-job.sql' ), - array( 'langlinks', 'patch-langlinks.sql' ), - array( 'querycache_info', 'patch-querycacheinfo.sql' ), - array( 'filearchive', 'patch-filearchive.sql' ), - array( 'querycachetwo', 'patch-querycachetwo.sql' ), - array( 'redirect', 'patch-redirect.sql' ), +/** + * List of update functions to call on a MySQL-based MediaWiki installation, + * in sequence. First item is function name, rest are parameters to pass. + */ +$wgMysqlUpdates = array( + // 1.2 + // update_passwords obsolete + array( 'add_field', 'ipblocks', 'ipb_id', 'patch-ipblocks.sql' ), + array( 'add_field', 'ipblocks', 'ipb_expiry', 'patch-ipb_expiry.sql' ), + array( 'do_interwiki_update' ), + array( 'do_index_update' ), + // do_linkscc_update obsolete + array( 'add_table', 'hitcounter', 'patch-hitcounter.sql' ), + array( 'add_field', 'recentchanges', 'rc_type', 'patch-rc_type.sql' ), + + // 1.3 + array( 'add_field', 'user', 'user_real_name', 'patch-user-realname.sql' ), + array( 'add_table', 'querycache', 'patch-querycache.sql' ), + array( 'add_table', 'objectcache', 'patch-objectcache.sql' ), + array( 'add_table', 'categorylinks', 'patch-categorylinks.sql' ), + // do_linkscc_1_3_update obsolete + array( 'do_old_links_update' ), + array( 'add_field', 'recentchanges', 'rc_ip', 'patch-rc_ip.sql' ), + + // 1.4 + array( 'do_image_name_unique_update' ), + array( 'add_field', 'recentchanges', 'rc_id', 'patch-rc_id.sql' ), + array( 'add_field', 'recentchanges', 'rc_patrolled', 'patch-rc-patrol.sql' ), + array( 'add_table', 'logging', 'patch-logging.sql' ), + // do_user_rights_update obsolete + array( 'add_field', 'user', 'user_token', 'patch-user_token.sql' ), + // old, old_articleid, patch-remove-old-title-namespace.sql obsolete + // user_groups, patch-userlevels.sql obsolete + // do_group_update() obsolete + array( 'do_watchlist_update' ), + array( 'do_user_update' ), + // do_copy_newtalk_to_watchlist obsolete + + // 1.5 + array( 'do_schema_restructuring' ), + array( 'add_field', 'logging', 'log_params', 'patch-log_params.sql' ), + array( 'do_logging_encoding' ), + array( 'add_field', 'archive', 'ar_rev_id', 'patch-archive-rev_id.sql' ), + array( 'add_field', 'page', 'page_len', 'patch-page_len.sql' ), + array( 'do_inverse_timestamp' ), + array( 'do_text_id' ), + array( 'add_field', 'revision', 'rev_deleted', 'patch-rev_deleted.sql' ), + array( 'add_field', 'image', 'img_width', 'patch-img_width.sql' ), + array( 'add_field', 'image', 'img_metadata', 'patch-img_metadata.sql' ), + array( 'add_field', 'user', 'user_email_token', 'patch-user_email_token.sql' ), + array( 'add_field', 'archive', 'ar_text_id', 'patch-archive-text_id.sql' ), + array( 'do_namespace_size' ), + array( 'add_field', 'image', 'img_media_type', 'patch-img_media_type.sql' ), + array( 'do_pagelinks_update' ), + array( 'do_drop_img_type' ), + array( 'do_user_unique_update' ), + array( 'do_user_groups_update' ), + array( 'add_field', 'site_stats', 'ss_total_pages', 'patch-ss_total_articles.sql' ), + array( 'add_table', 'user_newtalk', 'patch-usernewtalk2.sql' ), + array( 'add_table', 'transcache', 'patch-transcache.sql' ), + array( 'add_field', 'interwiki', 'iw_trans', 'patch-interwiki-trans.sql' ), + array( 'add_table', 'trackbacks', 'patch-trackbacks.sql' ), + + // 1.6 + array( 'do_watchlist_null' ), + // do_image_index_update obsolete + array( 'do_logging_timestamp_index' ), + array( 'add_field', 'ipblocks', 'ipb_range_start', 'patch-ipb_range_start.sql' ), + array( 'do_page_random_update' ), + array( 'add_field', 'user', 'user_registration','patch-user_registration.sql' ), + array( 'do_templatelinks_update' ), + array( 'add_table', 'externallinks', 'patch-externallinks.sql' ), + array( 'add_table', 'job', 'patch-job.sql' ), + array( 'add_field', 'site_stats', 'ss_images', 'patch-ss_images.sql' ), + array( 'add_table', 'langlinks', 'patch-langlinks.sql' ), + array( 'add_table', 'querycache_info', 'patch-querycacheinfo.sql' ), + array( 'add_table', 'filearchive', 'patch-filearchive.sql' ), + array( 'add_field', 'ipblocks', 'ipb_anon_only', 'patch-ipb_anon_only.sql' ), + array( 'do_rc_indices_update' ), + + // 1.9 + array( 'add_field', 'user', 'user_newpass_time', 'patch-user_newpass_time.sql' ), + array( 'add_table', 'redirect', 'patch-redirect.sql' ), + array( 'add_table', 'querycachetwo', 'patch-querycachetwo.sql' ), + array( 'add_field', 'ipblocks', 'ipb_enable_autoblock', 'patch-ipb_optional_autoblock.sql' ), + array( 'do_backlinking_indices_update' ), + array( 'add_field', 'recentchanges', 'rc_old_len', 'patch-rc_len.sql' ), + array( 'add_field', 'user', 'user_editcount', 'patch-user_editcount.sql' ), + + // 1.10 + array( 'do_restrictions_update' ), + array( 'add_field', 'logging', 'log_id', 'patch-log_id.sql' ), + array( 'add_field', 'revision', 'rev_parent_id', 'patch-rev_parent_id.sql' ), + array( 'add_field', 'page_restrictions', 'pr_id', 'patch-page_restrictions_sortkey.sql' ), + array( 'add_field', 'revision', 'rev_len', 'patch-rev_len.sql' ), + array( 'add_field', 'recentchanges', 'rc_deleted', 'patch-rc_deleted.sql' ), + array( 'add_field', 'logging', 'log_deleted', 'patch-log_deleted.sql' ), + array( 'add_field', 'archive', 'ar_deleted', 'patch-ar_deleted.sql' ), + array( 'add_field', 'ipblocks', 'ipb_deleted', 'patch-ipb_deleted.sql' ), + array( 'add_field', 'filearchive', 'fa_deleted', 'patch-fa_deleted.sql' ), + array( 'add_field', 'archive', 'ar_len', 'patch-ar_len.sql' ), + + // 1.11 + array( 'add_field', 'ipblocks', 'ipb_block_email', 'patch-ipb_emailban.sql' ), + array( 'do_categorylinks_indices_update' ), + array( 'add_field', 'oldimage', 'oi_metadata', 'patch-oi_metadata.sql'), + array( 'do_archive_user_index' ), + array( 'do_image_user_index' ), + array( 'do_oldimage_user_index' ), + array( 'add_field', 'archive', 'ar_page_id', 'patch-archive-page_id.sql'), + array( 'add_field', 'image', 'img_sha1', 'patch-img_sha1.sql' ), + array( 'add_table', 'protected_titles', 'patch-protected_titles.sql' ), ); -$wgNewFields = array( -# table field patch file (in maintenance/archives) - array( 'ipblocks', 'ipb_id', 'patch-ipblocks.sql' ), - array( 'ipblocks', 'ipb_expiry', 'patch-ipb_expiry.sql' ), - array( 'recentchanges', 'rc_type', 'patch-rc_type.sql' ), - array( 'recentchanges', 'rc_ip', 'patch-rc_ip.sql' ), - array( 'recentchanges', 'rc_id', 'patch-rc_id.sql' ), - array( 'recentchanges', 'rc_patrolled', 'patch-rc-patrol.sql' ), - array( 'recentchanges', 'rc_old_len', 'patch-rc_len.sql' ), - array( 'user', 'user_real_name', 'patch-user-realname.sql' ), - array( 'user', 'user_token', 'patch-user_token.sql' ), - array( 'user', 'user_email_token', 'patch-user_email_token.sql' ), - array( 'user', 'user_registration','patch-user_registration.sql' ), - array( 'logging', 'log_params', 'patch-log_params.sql' ), - array( 'archive', 'ar_rev_id', 'patch-archive-rev_id.sql' ), - array( 'archive', 'ar_text_id', 'patch-archive-text_id.sql' ), - array( 'page', 'page_len', 'patch-page_len.sql' ), - array( 'revision', 'rev_deleted', 'patch-rev_deleted.sql' ), - array( 'image', 'img_width', 'patch-img_width.sql' ), - array( 'image', 'img_metadata', 'patch-img_metadata.sql' ), - array( 'image', 'img_media_type', 'patch-img_media_type.sql' ), - array( 'site_stats', 'ss_total_pages', 'patch-ss_total_articles.sql' ), - array( 'interwiki', 'iw_trans', 'patch-interwiki-trans.sql' ), - array( 'ipblocks', 'ipb_range_start', 'patch-ipb_range_start.sql' ), - array( 'site_stats', 'ss_images', 'patch-ss_images.sql' ), - array( 'ipblocks', 'ipb_anon_only', 'patch-ipb_anon_only.sql' ), - array( 'ipblocks', 'ipb_enable_autoblock', 'patch-ipb_optional_autoblock.sql' ), - array( 'user', 'user_newpass_time','patch-user_newpass_time.sql' ), - array( 'user', 'user_editcount', 'patch-user_editcount.sql' ), - array( 'recentchanges', 'rc_deleted', 'patch-rc_deleted.sql' ), - array( 'logging', 'log_id', 'patch-log_id.sql' ), - array( 'logging', 'log_deleted', 'patch-log_deleted.sql' ), - array( 'archive', 'ar_deleted', 'patch-ar_deleted.sql' ), - array( 'ipblocks', 'ipb_deleted', 'patch-ipb_deleted.sql' ), - array( 'filearchive', 'fa_deleted', 'patch-fa_deleted.sql' ), - array( 'revision', 'rev_len', 'patch-rev_len.sql' ), - array( 'archive', 'ar_len', 'patch-ar_len.sql' ), - array( 'revision', 'rev_parent_id', 'patch-rev_parent_id.sql' ), - array( 'page_restrictions', 'pr_id', 'patch-page_restrictions_sortkey.sql' ), - array( 'ipblocks', 'ipb_block_email', 'patch-ipb_emailban.sql' ), - array( 'oldimage', 'oi_metadata', 'patch-oi_metadata.sql'), - array( 'archive', 'ar_page_id', 'patch-archive-page_id.sql'), - array( 'image', 'img_sha1', 'patch-img_sha1.sql' ), -); # For extensions only, should be populated via hooks # $wgDBtype should be checked to specifiy the proper file @@ -943,26 +989,16 @@ function do_all_updates( $shared = false, $purge = true ) { do_postgres_updates(); return; } - - # Rename tables - foreach ( $wgRenamedTables as $tableRecord ) { - rename_table( $tableRecord[0], $tableRecord[1], $tableRecord[2] ); - } - - # Add missing tables - foreach ( $wgNewTables as $tableRecord ) { - add_table( $tableRecord[0], $tableRecord[1] ); - flush(); - } - - # Add missing fields - foreach ( $wgNewFields as $fieldRecord ) { - if ( $fieldRecord[0] != 'user' || $doUser ) { - add_field( $fieldRecord[0], $fieldRecord[1], $fieldRecord[2] ); - } + + # Run core updates in sequence... + global $wgMysqlUpdates; + foreach( $wgMysqlUpdates as $params ) { + $func = array_shift( $params ); + call_user_func_array( $func, $params ); flush(); } + /// @fixme clean up this mess too! global $wgExtNewTables, $wgExtNewFields, $wgExtNewIndexes; # Add missing extension tables foreach ( $wgExtNewTables as $tableRecord ) { @@ -982,54 +1018,6 @@ function do_all_updates( $shared = false, $purge = true ) { flush(); } - # Do schema updates which require special handling - do_interwiki_update(); flush(); - do_index_update(); flush(); - do_old_links_update(); flush(); - do_image_name_unique_update(); flush(); - do_watchlist_update(); flush(); - if ( $doUser ) { - do_user_update(); flush(); - } -###### do_copy_newtalk_to_watchlist(); flush(); - do_logging_encoding(); flush(); - - do_schema_restructuring(); flush(); - do_inverse_timestamp(); flush(); - do_text_id(); flush(); - do_namespace_size(); flush(); - - do_pagelinks_update(); flush(); - do_templatelinks_update(); flush(); // after pagelinks - - do_drop_img_type(); flush(); - - if ( $doUser ) { - do_user_unique_update(); flush(); - } - do_user_groups_update(); flush(); - - do_watchlist_null(); flush(); - - //do_image_index_update(); flush(); - - do_logging_timestamp_index(); flush(); - - do_page_random_update(); flush(); - - do_rc_indices_update(); flush(); - - do_backlinking_indices_update(); flush(); - - do_categorylinks_indices_update(); flush(); - - do_restrictions_update(); flush (); - - do_archive_user_index(); flush (); - - do_image_user_index(); flush (); - - do_oldimage_user_index(); flush (); echo "Deleting old default messages (this may take a long time!)..."; flush(); deleteDefaultMessages(); @@ -1260,6 +1248,10 @@ END; function do_postgres_updates() { global $wgDatabase, $wgVersion, $wgDBmwschema, $wgDBts2schema, $wgShowExceptionDetails, $wgDBuser; + ## Gather version numbers in case we need them + $version = $wgDatabase->getServerVersion(); ## long string + $numver = $wgDatabase->numeric_version; ## X.Y e.g. 8.3 + $wgShowExceptionDetails = 1; # Just in case their LocalSettings.php does not have this: @@ -1268,7 +1260,7 @@ function do_postgres_updates() { # Verify that this user is configured correctly $safeuser = $wgDatabase->addQuotes($wgDBuser); - $SQL = "SELECT array_to_string(useconfig,'*') FROM pg_user WHERE usename = $safeuser"; + $SQL = "SELECT array_to_string(useconfig,'*') FROM pg_catalog.pg_user WHERE usename = $safeuser"; $config = pg_fetch_result( $wgDatabase->doQuery( $SQL ), 0, 0 ); $conf = array(); foreach( explode( '*', $config ) as $c ) { @@ -1276,17 +1268,24 @@ function do_postgres_updates() { $conf[$x] = $y; } $newpath = array(); - if( !array_key_exists( 'search_path', $conf ) or strpos( $conf['search_path'],$wgDBmwschema ) === false ) { - print "Adding in schema \"$wgDBmwschema\" to search_path for user \"$wgDBuser\"\n"; - $newpath[$wgDBmwschema] = 1; + if( $wgDBmwschema === 'mediawiki' ) { + if (!array_key_exists( 'search_path', $conf ) or strpos( $conf['search_path'],$wgDBmwschema ) === false ) { + echo "Adding in schema \"$wgDBmwschema\" to search_path for user \"$wgDBuser\"\n"; + $newpath[$wgDBmwschema] = 1; + } } if( !array_key_exists( 'search_path', $conf ) or strpos( $conf['search_path'],$wgDBts2schema ) === false ) { - print "Adding in schema \"$wgDBts2schema\" to search_path for user \"$wgDBuser\"\n"; + echo "Adding in schema \"$wgDBts2schema\" to search_path for user \"$wgDBuser\"\n"; $newpath[$wgDBts2schema] = 1; } $searchpath = implode( ',', array_keys( $newpath ) ); if( strlen( $searchpath ) ) { $wgDatabase->doQuery( "ALTER USER $wgDBuser SET search_path = $searchpath" ); + $wgDatabase->doQuery( "SET search_path = $searchpath" ); + } + else { + $path = $conf['search_path']; + echo "... search_path for user \"$wgDBuser\" looks correct ($path)\n"; } $goodconf = array( 'client_min_messages' => 'error', @@ -1296,8 +1295,12 @@ function do_postgres_updates() { foreach( array_keys( $goodconf ) AS $key ) { $value = $goodconf[$key]; if( !array_key_exists( $key, $conf ) or $conf[$key] !== $value ) { - print "Setting $key to '$value' for user \"$wgDBuser\"\n"; + echo "Setting $key to '$value' for user \"$wgDBuser\"\n"; $wgDatabase->doQuery( "ALTER USER $wgDBuser SET $key = '$value'" ); + $wgDatabase->doQuery( "SET $key = '$value'" ); + } + else { + echo "... default value of \"$key\" is correctly set to \"$value\" for user \"$wgDBuser\"\n"; } } @@ -1313,32 +1316,34 @@ function do_postgres_updates() { array("querycachetwo", "patch-querycachetwo.sql"), array("page_restrictions", "patch-page_restrictions.sql"), array("profiling", "patch-profiling.sql"), + array("protected_titles", "patch-protected_titles.sql"), array("redirect", "patch-redirect.sql"), ); $newcols = array( + array("archive", "ar_deleted", "SMALLINT NOT NULL DEFAULT 0"), array("archive", "ar_len", "INTEGER"), array("archive", "ar_page_id", "INTEGER"), array("image", "img_sha1", "TEXT NOT NULL DEFAULT ''"), array("ipblocks", "ipb_anon_only", "CHAR NOT NULL DEFAULT '0'"), array("ipblocks", "ipb_block_email", "CHAR NOT NULL DEFAULT '0'"), array("ipblocks", "ipb_create_account", "CHAR NOT NULL DEFAULT '1'"), - array("ipblocks", "ipb_deleted", "INTEGER NOT NULL DEFAULT 0"), + array("ipblocks", "ipb_deleted", "SMALLINT NOT NULL DEFAULT 0"), array("ipblocks", "ipb_enable_autoblock", "CHAR NOT NULL DEFAULT '1'"), - array("filearchive", "fa_deleted", "INTEGER NOT NULL DEFAULT 0"), - array("logging", "log_deleted", "INTEGER NOT NULL DEFAULT 0"), + array("filearchive", "fa_deleted", "SMALLINT NOT NULL DEFAULT 0"), + array("logging", "log_deleted", "SMALLINT NOT NULL DEFAULT 0"), array("logging", "log_id", "INTEGER NOT NULL PRIMARY KEY DEFAULT nextval('log_log_id_seq')"), array("logging", "log_params", "TEXT"), array("mwuser", "user_editcount", "INTEGER"), array("mwuser", "user_newpass_time", "TIMESTAMPTZ"), - array("oldimage", "oi_deleted", "CHAR NOT NULL DEFAULT '0'"), + array("oldimage", "oi_deleted", "SMALLINT NOT NULL DEFAULT 0"), array("oldimage", "oi_metadata", "BYTEA NOT NULL DEFAULT ''"), array("oldimage", "oi_media_type", "TEXT"), array("oldimage", "oi_major_mime", "TEXT NOT NULL DEFAULT 'unknown'"), array("oldimage", "oi_minor_mime", "TEXT NOT NULL DEFAULT 'unknown'"), array("oldimage", "oi_sha1", "TEXT NOT NULL DEFAULT ''"), array("page_restrictions", "pr_id", "INTEGER NOT NULL UNIQUE DEFAULT nextval('pr_id_val')"), - array("recentchanges", "rc_deleted", "INTEGER NOT NULL DEFAULT 0"), + array("recentchanges", "rc_deleted", "SMALLINT NOT NULL DEFAULT 0"), array("recentchanges", "rc_log_action", "TEXT"), array("recentchanges", "rc_log_type", "TEXT"), array("recentchanges", "rc_logid", "INTEGER NOT NULL DEFAULT 0"), @@ -1346,21 +1351,55 @@ function do_postgres_updates() { array("recentchanges", "rc_old_len", "INTEGER"), array("recentchanges", "rc_params", "TEXT"), array("revision", "rev_len", "INTEGER"), + array("revision", "rev_deleted", "SMALLINT NOT NULL DEFAULT 0"), ); - # table, column, desired type, USING clause if needed + # table, column, desired type, USING clause if needed (with new default if needed) $typechanges = array( - array("image", "img_size", "int4", ""), - array("image", "img_width", "int4", ""), - array("image", "img_height", "int4", ""), - array("ipblocks", "ipb_address", "text", "ipb_address::text"), - array("math", "math_inputhash", "bytea", "decode(math_inputhash,'escape')"), - array("math", "math_outputhash", "bytea", "decode(math_outputhash,'escape')"), - array("oldimage", "oi_size", "int4", ""), - array("oldimage", "oi_width", "int4", ""), - array("oldimage", "oi_height", "int4", ""), - array("user_newtalk", "user_ip", "text", "host(user_ip)"), + array("archive", "ar_deleted", "smallint", ""), + array("archive", "ar_minor_edit", "smallint", "ar_minor_edit::smallint DEFAULT 0"), + array("filearchive", "fa_deleted", "smallint", ""), + array("filearchive", "fa_height", "integer", ""), + array("filearchive", "fa_metadata", "bytea", "decode(fa_metadata,'escape')"), + array("filearchive", "fa_size", "integer", ""), + array("filearchive", "fa_width", "integer", ""), + array("filearchive", "fa_storage_group","text", ""), + array("filearchive", "fa_storage_key", "text", ""), + array("image", "img_metadata", "bytea", "decode(img_metadata,'escape')"), + array("image", "img_size", "integer", ""), + array("image", "img_width", "integer", ""), + array("image", "img_height", "integer", ""), + array("interwiki", "iw_local", "smallint", "iw_local::smallint DEFAULT 0"), + array("interwiki", "iw_trans", "smallint", "iw_trans::smallint DEFAULT 0"), + array("ipblocks", "ipb_auto", "smallint", "ipb_auto::smallint DEFAULT 0"), + array("ipblocks", "ipb_anon_only", "smallint", "ipb_anon_only::smallint DEFAULT 0"), + array("ipblocks", "ipb_create_account", "smallint", "ipb_create_account::smallint DEFAULT 1"), + array("ipblocks", "ipb_enable_autoblock", "smallint", "ipb_enable_autoblock::smallint DEFAULT 1"), + array("ipblocks", "ipb_block_email", "smallint", "ipb_block_email::smallint DEFAULT 0"), + array("ipblocks", "ipb_address", "text", "ipb_address::text"), + array("ipblocks", "ipb_deleted", "smallint", ""), + array("math", "math_inputhash", "bytea", "decode(math_inputhash,'escape')"), + array("math", "math_outputhash", "bytea", "decode(math_outputhash,'escape')"), + array("mwuser", "user_token", "text", ""), + array("mwuser", "user_email_token","text", ""), + array("objectcache", "keyname", "text", ""), + array("oldimage", "oi_height", "integer", ""), + array("oldimage", "oi_size", "integer", ""), + array("oldimage", "oi_width", "integer", ""), + array("page", "page_is_redirect","smallint", "page_is_redirect::smallint DEFAULT 0"), + array("page", "page_is_new", "smallint", "page_is_new::smallint DEFAULT 0"), + array("querycache", "qc_value", "integer", ""), + array("querycachetwo","qcc_value", "integer", ""), + array("recentchanges","rc_bot", "smallint", "rc_bot::smallint DEFAULT 0"), + array("recentchanges","rc_deleted", "smallint", ""), + array("recentchanges","rc_minor", "smallint", "rc_minor::smallint DEFAULT 0"), + array("recentchanges","rc_new", "smallint", "rc_new::smallint DEFAULT 0"), + array("recentchanges","rc_type", "smallint", "rc_type::smallint DEFAULT 0"), + array("recentchanges","rc_patrolled", "smallint", "rc_patrolled::smallint DEFAULT 0"), + array("revision", "rev_minor_edit", "smallint", "rev_minor_edit::smallint DEFAULT 0"), + array("templatelinks","tl_namespace", "smallint", "tl_namespace::smallint"), + array("user_newtalk", "user_ip", "text", "host(user_ip)"), ); $newindexes = array( @@ -1375,48 +1414,48 @@ function do_postgres_updates() { foreach ($newsequences as $ns) { if ($wgDatabase->sequenceExists($ns)) { - echo "... sequence $ns already exists\n"; + echo "... sequence \"$ns\" already exists\n"; continue; } - echo "... create sequence $ns\n"; + echo "Creating sequence \"$ns\"\n"; $wgDatabase->query("CREATE SEQUENCE $ns"); } foreach ($newtables as $nt) { if ($wgDatabase->tableExists($nt[0])) { - echo "... table $nt[0] already exists\n"; + echo "... table \"$nt[0]\" already exists\n"; continue; } - echo "... create table $nt[0]\n"; + echo "Creating table \"$nt[0]\"\n"; dbsource(archive($nt[1])); } ## Needed before newcols if ($wgDatabase->tableExists("archive2")) { - echo "... convert archive2 back to normal archive table\n"; + echo "Converting \"archive2\" back to normal archive table\n"; if ($wgDatabase->ruleExists("archive", "archive_insert")) { - echo "... drop rule archive_insert\n"; + echo "Dropping rule \"archive_insert\"\n"; $wgDatabase->query("DROP RULE archive_insert ON archive"); } if ($wgDatabase->ruleExists("archive", "archive_delete")) { - echo "... drop rule archive_delete\n"; + echo "Dropping rule \"archive_delete\"\n"; $wgDatabase->query("DROP RULE archive_delete ON archive"); } - dbsource(archive("patch-remove-archive2.sql")); - } else - echo "... obsolete archive2 not present\n"; + } + else + echo "... obsolete table \"archive2\" does not exist\n"; foreach ($newcols as $nc) { $fi = $wgDatabase->fieldInfo($nc[0], $nc[1]); if (!is_null($fi)) { - echo "... column $nc[0].$nc[1] already exists\n"; + echo "... column \"$nc[0].$nc[1]\" already exists\n"; continue; } - echo "... add column $nc[0].$nc[1]\n"; + echo "Adding column \"$nc[0].$nc[1]\"\n"; $wgDatabase->query("ALTER TABLE $nc[0] ADD $nc[1] $nc[2]"); } @@ -1428,11 +1467,17 @@ function do_postgres_updates() { } if ($fi->type() === $tc[2]) - echo "... $tc[0].$tc[1] is already $tc[2]\n"; + echo "... column \"$tc[0].$tc[1]\" is already of type \"$tc[2]\"\n"; else { - echo "... change $tc[0].$tc[1] from {$fi->type()} to $tc[2]\n"; + echo "Changing column type of \"$tc[0].$tc[1]\" from \"{$fi->type()}\" to \"$tc[2]\"\n"; $sql = "ALTER TABLE $tc[0] ALTER $tc[1] TYPE $tc[2]"; if (strlen($tc[3])) { + $default = array(); + if (preg_match( '/DEFAULT (.+)/', $tc[3], $default)) { + $sqldef = "ALTER TABLE $tc[0] ALTER $tc[1] SET DEFAULT $default[1]"; + $wgDatabase->query($sqldef); + $tc[3] = preg_replace( '/\s*DEFAULT .+/', '', $tc[3]); + } $sql .= " USING $tc[3]"; } $sql .= ";\nCOMMIT;\n"; @@ -1440,91 +1485,120 @@ function do_postgres_updates() { } } + if ($wgDatabase->fieldInfo('oldimage','oi_deleted')->type() !== 'smallint') { + echo "Changing \"oldimage.oi_deleted\" to type \"smallint\"\n"; + $wgDatabase->query( "ALTER TABLE oldimage ALTER oi_deleted DROP DEFAULT" ); + $wgDatabase->query( "ALTER TABLE oldimage ALTER oi_deleted TYPE SMALLINT USING (oi_deleted::smallint)" ); + $wgDatabase->query( "ALTER TABLE oldimage ALTER oi_deleted SET DEFAULT 0" ); + } + else + echo "... column \"oldimage.oi_deleted\" is already of type \"smallint\"\n"; + + foreach ($newindexes as $ni) { if (pg_index_exists($ni[0], $ni[1])) { - echo "... index $ni[1] on $ni[0] already exists\n"; + echo "... index \"$ni[1]\" on table \"$ni[0]\" already exists\n"; continue; } - $wgDatabase->query("CREATE INDEX $ni[1] ON $ni[0] $ni[2]"); - echo "create index $ni[1]\n"; + echo "Creating index \"$ni[1]\" on table \"$ni[0]\" $ni[2]\n"; + $wgDatabase->query( "CREATE INDEX $ni[1] ON $ni[0] $ni[2]" ); } foreach ($newrules as $nr) { if ($wgDatabase->ruleExists($nr[0], $nr[1])) { - echo "... rule $nr[1] on $nr[0] already exists\n"; + echo "... rule \"$nr[1]\" on table \"$nr[0]\" already exists\n"; continue; } + echo "Adding rule \"$nr[1]\" to table \"$nr[0]\"\n"; dbsource(archive($nr[2])); } + if ($wgDatabase->hasConstraint("oldimage_oi_name_fkey")) { + echo "Making foriegn key on table \"oldimage\" (to image) a cascade delete\n"; + $wgDatabase->query( "ALTER TABLE oldimage DROP CONSTRAINT oldimage_oi_name_fkey" ); + $wgDatabase->query( "ALTER TABLE oldimage ADD CONSTRAINT oldimage_oi_name_fkey_cascade ". + "FOREIGN KEY (oi_name) REFERENCES image(img_name) ON DELETE CASCADE" ); + } + else + echo "... table \"oldimage\" has correct cascade delete foreign key to image\n"; + if (!$wgDatabase->triggerExists("page", "page_deleted")) { - echo "... create page_deleted trigger\n"; + echo "Adding function and trigger \"page_deleted\" to table \"page\"\n"; dbsource(archive('patch-page_deleted.sql')); } + else + echo "... table \"page\" has \"page_deleted\" trigger\n"; $fi = $wgDatabase->fieldInfo("recentchanges", "rc_cur_id"); if (!$fi->nullable()) { - echo "... remove NOT NULL constraint on recentchanges.rc_cur_id\n"; + echo "Removing NOT NULL constraint from \"recentchanges.rc_cur_id\"\n"; dbsource(archive('patch-rc_cur_id-not-null.sql')); } + else + echo "... column \"recentchanges.rc_cur_id\" has a NOT NULL constraint\n"; $pu = pg_describe_index("pagelink_unique"); if (!is_null($pu) && ($pu[0] != "pl_from" || $pu[1] != "pl_namespace" || $pu[2] != "pl_title")) { - echo "... dropping obsolete pagelink_unique index\n"; + echo "Dropping obsolete version of index \"pagelink_unique index\"\n"; $wgDatabase->query("DROP INDEX pagelink_unique"); $pu = null; - } else - echo "... obsolete pagelink_unique index not present\n"; + } + else + echo "... obsolete version of index \"pagelink_unique index\" does not exist\n"; if (is_null($pu)) { - echo "... adding new pagelink_unique index\n"; + echo "Creating index \"pagelink_unique index\"\n"; $wgDatabase->query("CREATE UNIQUE INDEX pagelink_unique ON pagelinks (pl_from,pl_namespace,pl_title)"); - } else - echo "... already have current pagelink_unique index\n"; + } + else + echo "... index \"pagelink_unique_index\" aready exists\n"; if (pg_fkey_deltype("revision_rev_user_fkey") == 'r') { - echo "... revision_rev_user_fkey is already ON DELETE RESTRICT\n"; - } else { - echo "... change revision_rev_user_fkey to ON DELETE RESTRICT\n"; + echo "... constraint \"revision_rev_user_fkey\" is ON DELETE RESTRICT\n"; + } + else { + echo "Changing constraint \"revision_rev_user_fkey\" to ON DELETE RESTRICT\n"; dbsource(archive('patch-revision_rev_user_fkey.sql')); } - if (is_null($wgDatabase->fieldInfo("archive", "ar_deleted"))) { - echo "... add archive.ar_deleted\n"; - dbsource(archive("patch-archive-ar_deleted.sql")); - } else - echo "... archive.ar_deleted already exists\n"; - global $wgExtNewTables, $wgExtPGNewFields, $wgExtNewIndexes; # Add missing extension tables foreach ( $wgExtNewTables as $nt ) { if ($wgDatabase->tableExists($nt[0])) { - echo "... table $nt[0] already exists\n"; + echo "... table \"$nt[0]\" already exists\n"; continue; } - - echo "... create table $nt[0]\n"; + echo "Creating table \"$nt[0]\"\n"; dbsource($nt[1]); } # Add missing extension fields foreach ( $wgExtPGNewFields as $nc ) { $fi = $wgDatabase->fieldInfo($nc[0], $nc[1]); if (!is_null($fi)) { - echo "... column $nc[0].$nc[1] already exists\n"; + echo "... column \"$nc[0].$nc[1]\" already exists\n"; continue; } - - echo "... add column $nc[0].$nc[1]\n"; - $wgDatabase->query("ALTER TABLE $nc[0] ADD $nc[1] $nc[2]"); + echo "Adding column \"$nc[0].$nc[1]\"\n"; + $wgDatabase->query( "ALTER TABLE $nc[0] ADD $nc[1] $nc[2]" ); } # Add missing extension indexes foreach ( $wgExtNewIndexes as $ni ) { if (pg_index_exists($ni[0], $ni[1])) { - echo "... index $ni[1] on $ni[0] already exists\n"; + echo "... index \"$ni[1]\" on table \"$ni[0]\" already exists\n"; continue; } + echo "Creating index \"$ni[1]\" on table \"$ni[0]\"\n"; dbsource($ni[2]); } + # Tweak the page_title tsearch2 trigger to filter out slashes + # This is create or replace, so harmless to call if not needed + dbsource(archive('patch-ts2pagetitle.sql')); + + ## If the server is 8.3 or higher, rewrite teh tsearch2 triggers + ## in case they have the old 'default' versions + if ( $numver >= 8.3 ) + dbsource(archive('patch-tsearch2funcs.sql')); + return; } diff --git a/maintenance/wikipedia-interwiki.sql b/maintenance/wikipedia-interwiki.sql index 9e97967b..cfef8562 100644 --- a/maintenance/wikipedia-interwiki.sql +++ b/maintenance/wikipedia-interwiki.sql @@ -230,7 +230,7 @@ REPLACE INTO /*$wgDBprefix*/interwiki (iw_prefix,iw_url,iw_local) VALUES ('yue','http://zh-yue.wikipedia.org/wiki/$1',1), ('za','http://za.wikipedia.org/wiki/$1',1), ('zh-cfr','http://zh-min-nan.wikipedia.org/wiki/$1',1), -('zh-classical','http://pam.wikipedia.org/wiki/$1',1), +('zh-classical','http://zh-classical.wikipedia.org/wiki/$1',1), ('zh-cn','http://zh.wikipedia.org/wiki/$1',1), ('zh','http://zh.wikipedia.org/wiki/$1',1), ('zh-min-nan','http://zh-min-nan.wikipedia.org/wiki/$1',1), |