diff options
Diffstat (limited to 'maintenance/storage')
-rw-r--r-- | maintenance/storage/blobs.sql | 3 | ||||
-rw-r--r-- | maintenance/storage/checkStorage.php | 26 | ||||
-rw-r--r-- | maintenance/storage/compressOld.php | 42 | ||||
-rw-r--r-- | maintenance/storage/drop_content_model_info.sql | 7 | ||||
-rw-r--r-- | maintenance/storage/dumpRev.php | 15 | ||||
-rw-r--r-- | maintenance/storage/fixBug20757.php | 18 | ||||
-rwxr-xr-x | maintenance/storage/make-blobs | 4 | ||||
-rw-r--r-- | maintenance/storage/moveToExternal.php | 8 | ||||
-rw-r--r-- | maintenance/storage/orphanStats.php | 15 | ||||
-rw-r--r-- | maintenance/storage/recompressTracked.php | 47 | ||||
-rw-r--r-- | maintenance/storage/resolveStubs.php | 7 | ||||
-rw-r--r-- | maintenance/storage/storageTypeStats.php | 5 | ||||
-rw-r--r-- | maintenance/storage/testCompression.php | 10 | ||||
-rw-r--r-- | maintenance/storage/trackBlobs.php | 12 |
14 files changed, 130 insertions, 89 deletions
diff --git a/maintenance/storage/blobs.sql b/maintenance/storage/blobs.sql index 623dd7bf..979e68a9 100644 --- a/maintenance/storage/blobs.sql +++ b/maintenance/storage/blobs.sql @@ -4,5 +4,4 @@ CREATE TABLE /*$wgDBprefix*/blobs ( blob_id integer UNSIGNED NOT NULL AUTO_INCREMENT, blob_text longblob, PRIMARY KEY (blob_id) -) ENGINE=MyISAM MAX_ROWS=100000000 AVG_ROW_LENGTH=100000; - +) ENGINE=InnoDB; diff --git a/maintenance/storage/checkStorage.php b/maintenance/storage/checkStorage.php index 6c669bfa..03dc113a 100644 --- a/maintenance/storage/checkStorage.php +++ b/maintenance/storage/checkStorage.php @@ -22,7 +22,7 @@ */ if ( !defined( 'MEDIAWIKI' ) ) { - require_once( __DIR__ . '/../commandLine.inc' ); + require_once __DIR__ . '/../commandLine.inc'; $cs = new CheckStorage; $fix = isset( $options['fix'] ); @@ -38,14 +38,16 @@ if ( !defined( 'MEDIAWIKI' ) ) { // ---------------------------------------------------------------------------------- /** + * Maintenance script to do various checks on external storage. + * * @ingroup Maintenance ExternalStorage */ class CheckStorage { const CONCAT_HEADER = 'O:27:"concatenatedgziphistoryblob"'; - var $oldIdMap, $errors; - var $dbStore = null; + public $oldIdMap, $errors; + public $dbStore = null; - var $errorDescriptions = array( + public $errorDescriptions = array( 'restore text' => 'Damaged text, need to be restored from a backup', 'restore revision' => 'Damaged revision row, need to be restored from a backup', 'unfixable' => 'Unexpected errors with no automated fixing method', @@ -73,7 +75,7 @@ class CheckStorage { 'fixable' => array(), ); - for ( $chunkStart = 1 ; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) { + for ( $chunkStart = 1; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) { $chunkEnd = $chunkStart + $chunkSize - 1; // print "$chunkStart of $maxRevId\n"; @@ -441,18 +443,26 @@ class CheckStorage { function importRevision( &$revision, &$importer ) { $id = $revision->getID(); - $text = $revision->getText(); + $content = $revision->getContent( Revision::RAW ); + $id = $id ? $id : ''; + + if ( $content === null ) { + echo "Revision $id is broken, we have no content available\n"; + return; + } + + $text = $content->serialize(); if ( $text === '' ) { // This is what happens if the revision was broken at the time the // dump was made. Unfortunately, it also happens if the revision was // legitimately blank, so there's no way to tell the difference. To // be safe, we'll skip it and leave it broken - $id = $id ? $id : ''; + echo "Revision $id is blank in the dump, may have been broken before export\n"; return; } - if ( !$id ) { + if ( !$id ) { // No ID, can't import echo "No id tag in revision, can't import\n"; return; diff --git a/maintenance/storage/compressOld.php b/maintenance/storage/compressOld.php index 4594db71..8cb55487 100644 --- a/maintenance/storage/compressOld.php +++ b/maintenance/storage/compressOld.php @@ -41,14 +41,19 @@ * @ingroup Maintenance ExternalStorage */ -require_once( __DIR__ . '/../Maintenance.php' ); +require_once __DIR__ . '/../Maintenance.php'; +/** + * Maintenance script that compress the text of a wiki. + * + * @ingroup Maintenance ExternalStorage + */ class CompressOld extends Maintenance { /** * @todo document */ const LS_INDIVIDUAL = 0; - const LS_CHUNKED = 1; + const LS_CHUNKED = 1; public function __construct() { parent::__construct(); @@ -108,9 +113,9 @@ class CompressOld extends Maintenance { $this->output( "Starting from old_id $start...\n" ); $dbw = wfGetDB( DB_MASTER ); do { - $res = $dbw->select( 'text', array( 'old_id','old_flags','old_text' ), + $res = $dbw->select( 'text', array( 'old_id', 'old_flags', 'old_text' ), "old_id>=$start", __METHOD__, array( 'ORDER BY' => 'old_id', 'LIMIT' => $chunksize, 'FOR UPDATE' ) ); - if( $dbw->numRows( $res ) == 0 ) { + if ( $res->numRows() == 0 ) { break; } $last = $start; @@ -121,7 +126,7 @@ class CompressOld extends Maintenance { } $start = $last + 1; # Deletion may leave long empty stretches $this->output( "$start...\n" ); - } while( true ); + } while ( true ); } /** @@ -218,7 +223,7 @@ class CompressOld extends Maintenance { } $conds[] = "rev_timestamp>'" . $beginDate . "'"; } - if ( $endDate ) { + if ( $endDate ) { if ( !preg_match( '/^\d{14}$/', $endDate ) ) { $this->error( "Invalid end date \"$endDate\"\n" ); return false; @@ -249,9 +254,9 @@ class CompressOld extends Maintenance { # Get the page row $pageRes = $dbr->select( 'page', - array('page_id', 'page_namespace', 'page_title','page_latest'), - $pageConds + array('page_id' => $pageId), __METHOD__ ); - if ( $dbr->numRows( $pageRes ) == 0 ) { + array( 'page_id', 'page_namespace', 'page_title', 'page_latest' ), + $pageConds + array( 'page_id' => $pageId ), __METHOD__ ); + if ( $pageRes->numRows() == 0 ) { continue; } $pageRow = $dbr->fetchObject( $pageRes ); @@ -277,7 +282,7 @@ class CompressOld extends Maintenance { $revs[] = $revRow; } - if ( count( $revs ) < 2) { + if ( count( $revs ) < 2 ) { # No revisions matching, no further processing $this->output( "\n" ); continue; @@ -346,21 +351,22 @@ class CompressOld extends Maintenance { if ( $extdb != "" ) { # Move blob objects to External Storage $stored = $storeObj->store( $extdb, serialize( $chunk )); - if ($stored === false) { - $this->error( "Unable to store object" ); + if ( $stored === false ) { + $this->error( "Unable to store object" ); return false; } # Store External Storage URLs instead of Stub placeholders - foreach ($stubs as $stub) { - if ($stub===false) + foreach ( $stubs as $stub ) { + if ( $stub === false ) { continue; + } # $stored should provide base path to a BLOB - $url = $stored."/".$stub->getHash(); + $url = $stored . "/" . $stub->getHash(); $dbw->update( 'text', array( /* SET */ 'old_text' => $url, 'old_flags' => 'external,utf-8', - ), array ( /* WHERE */ + ), array( /* WHERE */ 'old_id' => $stub->getReferrer(), ) ); @@ -382,7 +388,7 @@ class CompressOld extends Maintenance { if ( $stubs[$j] !== false && $revs[$i + $j]->rev_text_id != $primaryOldid ) { $dbw->update( 'text', array( /* SET */ - 'old_text' => serialize($stubs[$j]), + 'old_text' => serialize( $stubs[$j] ), 'old_flags' => 'object,utf-8', ), array( /* WHERE */ 'old_id' => $revs[$i + $j]->rev_text_id @@ -406,4 +412,4 @@ class CompressOld extends Maintenance { } $maintClass = 'CompressOld'; -require_once( RUN_MAINTENANCE_IF_MAIN ); +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/storage/drop_content_model_info.sql b/maintenance/storage/drop_content_model_info.sql new file mode 100644 index 00000000..7bd9aba9 --- /dev/null +++ b/maintenance/storage/drop_content_model_info.sql @@ -0,0 +1,7 @@ +ALTER TABLE /*$wgDBprefix*/archive DROP COLUMN ar_content_model; +ALTER TABLE /*$wgDBprefix*/archive DROP COLUMN ar_content_format; + +ALTER TABLE /*$wgDBprefix*/revision DROP COLUMN rev_content_model; +ALTER TABLE /*$wgDBprefix*/revision DROP COLUMN rev_content_format; + +ALTER TABLE /*$wgDBprefix*/page DROP COLUMN page_content_model; diff --git a/maintenance/storage/dumpRev.php b/maintenance/storage/dumpRev.php index 6020f22e..f12bbd1b 100644 --- a/maintenance/storage/dumpRev.php +++ b/maintenance/storage/dumpRev.php @@ -1,5 +1,7 @@ <?php /** + * Get the text of a revision, resolving external storage if needed. + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -15,11 +17,18 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html * + * @file * @ingroup Maintenance ExternalStorage */ -require_once( __DIR__ . '/../Maintenance.php' ); +require_once __DIR__ . '/../Maintenance.php'; +/** + * Maintenance script that gets the text of a revision, + * resolving external storage if needed. + * + * @ingroup Maintenance ExternalStorage + */ class DumpRev extends Maintenance { public function __construct() { parent::__construct(); @@ -37,7 +46,7 @@ class DumpRev extends Maintenance { $this->error( "Row not found", true ); } - $flags = explode( ',', $row->old_flags ); + $flags = explode( ',', $row->old_flags ); $text = $row->old_text; if ( in_array( 'external', $flags ) ) { $this->output( "External $text\n" ); @@ -76,4 +85,4 @@ class DumpRev extends Maintenance { } $maintClass = "DumpRev"; -require_once( RUN_MAINTENANCE_IF_MAIN ); +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/storage/fixBug20757.php b/maintenance/storage/fixBug20757.php index 52ee825c..101aa068 100644 --- a/maintenance/storage/fixBug20757.php +++ b/maintenance/storage/fixBug20757.php @@ -21,13 +21,18 @@ * @ingroup Maintenance ExternalStorage */ -require_once( __DIR__ . '/../Maintenance.php' ); +require_once __DIR__ . '/../Maintenance.php'; +/** + * Maintenance script to fix bug 20757. + * + * @ingroup Maintenance ExternalStorage + */ class FixBug20757 extends Maintenance { - var $batchSize = 10000; - var $mapCache = array(); - var $mapCacheSize = 0; - var $maxMapCacheSize = 1000000; + public $batchSize = 10000; + public $mapCache = array(); + public $mapCacheSize = 0; + public $maxMapCacheSize = 1000000; function __construct() { parent::__construct(); @@ -343,5 +348,4 @@ class FixBug20757 extends Maintenance { } $maintClass = 'FixBug20757'; -require_once( RUN_MAINTENANCE_IF_MAIN ); - +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/storage/make-blobs b/maintenance/storage/make-blobs index 36cf9ced..16dcb672 100755 --- a/maintenance/storage/make-blobs +++ b/maintenance/storage/make-blobs @@ -6,11 +6,9 @@ if [ -z $2 ];then fi if [ -z $3 ]; then table=blobs -else +else table=$3 fi echo "CREATE DATABASE $2" | mysql -u wikiadmin -p`wikiadmin_pass` -h $1 && \ sed "s/blobs\>/$table/" blobs.sql | mysql -u wikiadmin -p`wikiadmin_pass` -h $1 $2 - - diff --git a/maintenance/storage/moveToExternal.php b/maintenance/storage/moveToExternal.php index 2dcc25c2..348fb773 100644 --- a/maintenance/storage/moveToExternal.php +++ b/maintenance/storage/moveToExternal.php @@ -24,9 +24,9 @@ define( 'REPORTING_INTERVAL', 1 ); if ( !defined( 'MEDIAWIKI' ) ) { - require_once( __DIR__ . '/../commandLine.inc' ); - require_once( __DIR__ . '/../../includes/ExternalStoreDB.php' ); - require_once( 'resolveStubs.php' ); + require_once __DIR__ . '/../commandLine.inc'; + require_once __DIR__ . '/../../includes/externalstore/ExternalStoreDB.php'; + require_once 'resolveStubs.php'; $fname = 'moveToExternal'; @@ -124,5 +124,3 @@ function moveToExternal( $cluster, $maxID, $minID = 1 ) { } } } - - diff --git a/maintenance/storage/orphanStats.php b/maintenance/storage/orphanStats.php index 82ee135b..1df1501e 100644 --- a/maintenance/storage/orphanStats.php +++ b/maintenance/storage/orphanStats.php @@ -1,7 +1,6 @@ <?php - /** - * Show some statistics on the blob_orphans table, created with trackBlobs.php + * Show some statistics on the blob_orphans table, created with trackBlobs.php. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,10 +17,18 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html * + * @file * @ingroup Maintenance ExternalStorage */ -require_once( __DIR__ . '/../Maintenance.php' ); +require_once __DIR__ . '/../Maintenance.php'; + +/** + * Maintenance script that shows some statistics on the blob_orphans table, + * created with trackBlobs.php. + * + * @ingroup Maintenance ExternalStorage + */ class OrphanStats extends Maintenance { public function __construct() { parent::__construct(); @@ -67,4 +74,4 @@ class OrphanStats extends Maintenance { } $maintClass = "OrphanStats"; -require_once( RUN_MAINTENANCE_IF_MAIN ); +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/storage/recompressTracked.php b/maintenance/storage/recompressTracked.php index 4098077f..b2663165 100644 --- a/maintenance/storage/recompressTracked.php +++ b/maintenance/storage/recompressTracked.php @@ -23,7 +23,7 @@ */ $optionsWithArgs = RecompressTracked::getOptionsWithArgs(); -require( __DIR__ . '/../commandLine.inc' ); +require __DIR__ . '/../commandLine.inc'; if ( count( $args ) < 1 ) { echo "Usage: php recompressTracked.php [options] <cluster> [... <cluster>...] @@ -42,20 +42,26 @@ Options: $job = RecompressTracked::newFromCommandLine( $args, $options ); $job->execute(); +/** + * Maintenance script that moves blobs indexed by trackBlobs.php to a specified + * list of destination clusters, and recompresses them in the process. + * + * @ingroup Maintenance ExternalStorage + */ class RecompressTracked { - var $destClusters; - var $batchSize = 1000; - var $orphanBatchSize = 1000; - var $reportingInterval = 10; - var $numProcs = 1; - var $useDiff, $pageBlobClass, $orphanBlobClass; - var $slavePipes, $slaveProcs, $prevSlaveId; - var $copyOnly = false; - var $isChild = false; - var $slaveId = false; - var $noCount = false; - var $debugLog, $infoLog, $criticalLog; - var $store; + public $destClusters; + public $batchSize = 1000; + public $orphanBatchSize = 1000; + public $reportingInterval = 10; + public $numProcs = 1; + public $useDiff, $pageBlobClass, $orphanBlobClass; + public $slavePipes, $slaveProcs, $prevSlaveId; + public $copyOnly = false; + public $isChild = false; + public $slaveId = false; + public $noCount = false; + public $debugLog, $infoLog, $criticalLog; + public $store; static $optionsWithArgs = array( 'procs', 'slave-id', 'debug-log', 'info-log', 'critical-log' ); static $cmdLineOptionMap = array( @@ -273,7 +279,7 @@ class RecompressTracked { */ function dispatchToSlave( $slaveId, $args ) { $args = (array)$args; - $cmd = implode( ' ', $args ); + $cmd = implode( ' ', $args ); fwrite( $this->slavePipes[$slaveId], "$cmd\n" ); } @@ -517,7 +523,7 @@ class RecompressTracked { * * Write the new URL to the text table and set the bt_moved flag. * - * This is done in a single transaction to provide restartable behaviour + * This is done in a single transaction to provide restartable behavior * without data loss. * * The transaction is kept short to reduce locking. @@ -670,10 +676,10 @@ class RecompressTracked { * Class to represent a recompression operation for a single CGZ blob */ class CgzCopyTransaction { - var $parent; - var $blobClass; - var $cgz; - var $referrers; + public $parent; + public $blobClass; + public $cgz; + public $referrers; /** * Create a transaction from a RecompressTracked object @@ -803,4 +809,3 @@ class CgzCopyTransaction { } } } - diff --git a/maintenance/storage/resolveStubs.php b/maintenance/storage/resolveStubs.php index 7e288e13..e47d6407 100644 --- a/maintenance/storage/resolveStubs.php +++ b/maintenance/storage/resolveStubs.php @@ -1,7 +1,7 @@ <?php /** - * Script to convert history stubs that point to an external row to direct - * external pointers. + * Convert history stubs that point to an external row to direct external + * pointers. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ define( 'REPORTING_INTERVAL', 100 ); if ( !defined( 'MEDIAWIKI' ) ) { $optionsWithArgs = array( 'm' ); - require_once( __DIR__ . '/../commandLine.inc' ); + require_once __DIR__ . '/../commandLine.inc'; resolveStubs(); } @@ -110,4 +110,3 @@ function resolveStub( $id, $stubText, $flags ) { ), $fname ); } - diff --git a/maintenance/storage/storageTypeStats.php b/maintenance/storage/storageTypeStats.php index 1afecc4e..e33057f6 100644 --- a/maintenance/storage/storageTypeStats.php +++ b/maintenance/storage/storageTypeStats.php @@ -19,7 +19,7 @@ * @ingroup Maintenance ExternalStorage */ -require_once( __DIR__ . '/../Maintenance.php' ); +require_once __DIR__ . '/../Maintenance.php'; class StorageTypeStats extends Maintenance { function execute() { @@ -112,5 +112,4 @@ SQL; } $maintClass = 'StorageTypeStats'; -require_once( RUN_MAINTENANCE_IF_MAIN ); - +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/storage/testCompression.php b/maintenance/storage/testCompression.php index 998ebe48..fdc28d9b 100644 --- a/maintenance/storage/testCompression.php +++ b/maintenance/storage/testCompression.php @@ -1,5 +1,7 @@ <?php /** + * Test revision text compression and decompression. + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -16,12 +18,11 @@ * http://www.gnu.org/copyleft/gpl.html * * @file - * @ingroup Maintenance - * @see wfWaitForSlaves() + * @ingroup Maintenance ExternalStorage */ $optionsWithArgs = array( 'start', 'limit', 'type' ); -require( __DIR__ . '/../commandLine.inc' ); +require __DIR__ . '/../commandLine.inc'; if ( !isset( $args[0] ) ) { echo "Usage: php testCompression.php [--type=<type>] [--start=<start-date>] [--limit=<num-revs>] <page-title>\n"; @@ -65,7 +66,7 @@ $uncompressedSize = 0; $t = -microtime( true ); foreach ( $res as $row ) { $revision = new Revision( $row ); - $text = $revision->getText(); + $text = $revision->getSerializedData(); $uncompressedSize += strlen( $text ); $hashes[$row->rev_id] = md5( $text ); $keys[$row->rev_id] = $blob->addItem( $text ); @@ -98,4 +99,3 @@ foreach ( $keys as $id => $key ) { } $t += microtime( true ); printf( "Decompression time: %5.2f ms\n", $t * 1000 ); - diff --git a/maintenance/storage/trackBlobs.php b/maintenance/storage/trackBlobs.php index 214168a8..7857dd95 100644 --- a/maintenance/storage/trackBlobs.php +++ b/maintenance/storage/trackBlobs.php @@ -22,7 +22,7 @@ * @see wfWaitForSlaves() */ -require( __DIR__ . '/../commandLine.inc' ); +require __DIR__ . '/../commandLine.inc'; if ( count( $args ) < 1 ) { @@ -37,12 +37,12 @@ $tracker->run(); echo "All done.\n"; class TrackBlobs { - var $clusters, $textClause; - var $doBlobOrphans; - var $trackedBlobs = array(); + public $clusters, $textClause; + public $doBlobOrphans; + public $trackedBlobs = array(); - var $batchSize = 1000; - var $reportingInterval = 10; + public $batchSize = 1000; + public $reportingInterval = 10; function __construct( $clusters ) { $this->clusters = $clusters; |