MediaWiki 1.7.1 wiederhergestellt

8 files changed, 1080 insertions, 0 deletions

diff --git a/maintenance/storage/blobs.sql b/maintenance/storage/blobs.sql
new file mode 100644
index 00000000..5782ac47
--- /dev/null
+++ b/maintenance/storage/blobs.sql
@@ -0,0 +1,8 @@
+-- Blobs table for external storage
+
+CREATE TABLE /*$wgDBprefix*/blobs (
+	blob_id int(8) NOT NULL AUTO_INCREMENT,
+	blob_text mediumtext,
+	PRIMARY KEY  (blob_id)
+) TYPE=InnoDB;
+
diff --git a/maintenance/storage/checkStorage.php b/maintenance/storage/checkStorage.php
new file mode 100644
index 00000000..a83d2744
--- /dev/null
+++ b/maintenance/storage/checkStorage.php
@@ -0,0 +1,468 @@
+<?php

+

+/**

+ * Fsck for MediaWiki

+ */

+

+define( 'CONCAT_HEADER', 'O:27:"concatenatedgziphistoryblob"' );

+

+if ( !defined( 'MEDIAWIKI' ) ) {

+	require_once( dirname(__FILE__) . '/../commandLine.inc' );

+	require_once( 'ExternalStore.php' );

+	require_once( 'ExternalStoreDB.php' );

+	require_once( 'SpecialImport.php' );

+

+	$cs = new CheckStorage;

+	$fix = isset( $options['fix'] );

+	if ( isset( $args[0] ) ) {

+		$xml = $args[0];

+	} else {

+		$xml = false;

+	}

+	$cs->check( $fix, $xml );

+}

+

+

+//----------------------------------------------------------------------------------

+

+class CheckStorage

+{

+	var $oldIdMap, $errors;

+	var $dbStore = null;

+

+	var $errorDescriptions = array(

+		'restore text' => 'Damaged text, need to be restored from a backup',

+		'restore revision' => 'Damaged revision row, need to be restored from a backup',

+		'unfixable' => 'Unexpected errors with no automated fixing method',

+		'fixed' => 'Errors already fixed',

+		'fixable' => 'Errors which would already be fixed if --fix was specified',

+	);		

+

+	function check( $fix = false, $xml = '' ) {

+		$fname = 'checkStorage';

+		$dbr =& wfGetDB( DB_SLAVE );

+		if ( $fix ) {

+			$dbw =& wfGetDB( DB_MASTER );

+			print "Checking, will fix errors if possible...\n";

+		} else {

+			print "Checking...\n";

+		}

+		$maxRevId = $dbr->selectField( 'revision', 'MAX(rev_id)', false, $fname );

+		$chunkSize = 1000;

+		$flagStats = array();

+		$objectStats = array();

+		$knownFlags = array( 'external', 'gzip', 'object', 'utf-8' );

+		$this->errors = array(

+			'restore text' => array(),

+			'restore revision' => array(),

+			'unfixable' => array(),

+			'fixed' => array(),

+			'fixable' => array(),

+		);

+

+		for ( $chunkStart = 1 ; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) {

+			$chunkEnd = $chunkStart + $chunkSize - 1;

+			//print "$chunkStart of $maxRevId\n";

+

+			// Fetch revision rows

+			$this->oldIdMap = array();

+			$dbr->ping();		

+			$res = $dbr->select( 'revision', array( 'rev_id', 'rev_text_id' ), 

+				array( "rev_id BETWEEN $chunkStart AND $chunkEnd" ), $fname );

+			while ( $row = $dbr->fetchObject( $res ) ) {

+				$this->oldIdMap[$row->rev_id] = $row->rev_text_id;

+			}

+			$dbr->freeResult( $res );

+

+			if ( !count( $this->oldIdMap ) ) {

+				continue;

+			}

+

+			// Fetch old_flags

+			$missingTextRows = array_flip( $this->oldIdMap );

+			$externalRevs = array();

+			$objectRevs = array();

+			$res = $dbr->select( 'text', array( 'old_id', 'old_flags' ), 

+				'old_id IN (' . implode( ',', $this->oldIdMap ) . ')', $fname );

+			while ( $row = $dbr->fetchObject( $res ) ) {

+				$flags = $row->old_flags;

+				$id = $row->old_id;

+

+				// Create flagStats row if it doesn't exist

+				$flagStats = $flagStats + array( $flags => 0 );

+				// Increment counter

+				$flagStats[$flags]++;

+

+				// Not missing

+				unset( $missingTextRows[$row->old_id] );

+

+				// Check for external or object

+				if ( $flags == '' ) {

+					$flagArray = array();

+				} else {

+					$flagArray = explode( ',', $flags );

+				}

+				if ( in_array( 'external', $flagArray ) ) {

+					$externalRevs[] = $id;

+				} elseif ( in_array( 'object', $flagArray ) ) {

+					$objectRevs[] = $id;

+				}

+

+				// Check for unrecognised flags

+				if ( $flags == '0' ) {

+					// This is a known bug from 2004

+					// It's safe to just erase the old_flags field

+					if ( $fix ) {

+						$this->error( 'fixed', "Warning: old_flags set to 0", $id );

+						$dbw->ping();

+						$dbw->update( 'text', array( 'old_flags' => '' ), 

+							array( 'old_id' => $id ), $fname );

+						echo "Fixed\n";

+					} else {

+						$this->error( 'fixable', "Warning: old_flags set to 0", $id );

+					}

+				} elseif ( count( array_diff( $flagArray, $knownFlags ) ) ) {

+					$this->error( 'unfixable', "Error: invalid flags field \"$flags\"", $id );

+				}

+			}

+			$dbr->freeResult( $res );

+

+			// Output errors for any missing text rows

+			foreach ( $missingTextRows as $oldId => $revId ) {

+				$this->error( 'restore revision', "Error: missing text row", $oldId );

+			}

+

+			// Verify external revisions

+			$externalConcatBlobs = array();

+			$externalNormalBlobs = array();

+			if ( count( $externalRevs ) ) {

+				$res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ), 

+					array( 'old_id IN (' . implode( ',', $externalRevs ) . ')' ), $fname );

+				while ( $row = $dbr->fetchObject( $res ) ) {

+					$urlParts = explode( '://', $row->old_text, 2 );

+					if ( count( $urlParts ) !== 2 || $urlParts[1] == '' ) {

+						$this->error( 'restore text', "Error: invalid URL \"{$row->old_text}\"", $row->old_id );

+						continue;

+					}

+					list( $proto, $path ) = $urlParts;

+					if ( $proto != 'DB' ) {

+						$this->error( 'restore text', "Error: invalid external protocol \"$proto\"", $row->old_id );

+						continue;

+					}

+					$path = explode( '/', $row->old_text );

+					$cluster = $path[2];

+					$id = $path[3];

+					if ( isset( $path[4] ) ) {

+						$externalConcatBlobs[$cluster][$id][] = $row->old_id;

+					} else {

+						$externalNormalBlobs[$cluster][$id][] = $row->old_id;

+					}

+				}

+				$dbr->freeResult( $res );

+			}

+

+			// Check external concat blobs for the right header

+			$this->checkExternalConcatBlobs( $externalConcatBlobs );

+			

+			// Check external normal blobs for existence

+			if ( count( $externalNormalBlobs ) ) {

+				if ( is_null( $this->dbStore ) ) {

+					$this->dbStore = new ExternalStoreDB;

+				}

+				foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) {

+					$blobIds = array_keys( $xBlobIds );

+					$extDb =& $this->dbStore->getSlave( $cluster );

+					$blobsTable = $this->dbStore->getTable( $extDb );

+					$res = $extDb->select( $blobsTable, 

+						array( 'blob_id' ), 

+						array( 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ), $fname );

+					while ( $row = $extDb->fetchObject( $res ) ) {

+						unset( $xBlobIds[$row->blob_id] );

+					}

+					$extDb->freeResult( $res );

+					// Print errors for missing blobs rows

+					foreach ( $xBlobIds as $blobId => $oldId ) {

+						$this->error( 'restore text', "Error: missing target $blobId for one-part ES URL", $oldId );

+					}

+				}

+			}

+

+			// Check local objects

+			$dbr->ping();

+			$concatBlobs = array();

+			$curIds = array();

+			if ( count( $objectRevs ) ) {

+				$headerLength = 300;

+				$res = $dbr->select( 'text', array( 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ), 

+					array( 'old_id IN (' . implode( ',', $objectRevs ) . ')' ), $fname );

+				while ( $row = $dbr->fetchObject( $res ) ) {

+					$oldId = $row->old_id;

+					if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) {

+						$this->error( 'restore text', "Error: invalid object header", $oldId );

+						continue;

+					}

+

+					$className = strtolower( $matches[2] );

+					if ( strlen( $className ) != $matches[1] ) {

+						$this->error( 'restore text', "Error: invalid object header, wrong class name length", $oldId );

+						continue;

+					}

+

+					$objectStats = $objectStats + array( $className => 0 );

+					$objectStats[$className]++;

+

+					switch ( $className ) {

+						case 'concatenatedgziphistoryblob':

+							// Good

+							break;

+						case 'historyblobstub':

+						case 'historyblobcurstub':

+							if ( strlen( $row->header ) == $headerLength ) {

+								$this->error( 'unfixable', "Error: overlong stub header", $oldId );

+								continue;

+							}

+							$stubObj = unserialize( $row->header );

+							if ( !is_object( $stubObj ) ) {

+								$this->error( 'restore text', "Error: unable to unserialize stub object", $oldId );

+								continue;

+							}

+							if ( $className == 'historyblobstub' ) {

+								$concatBlobs[$stubObj->mOldId][] = $oldId;

+							} else {

+								$curIds[$stubObj->mCurId][] = $oldId;

+							}

+							break;

+						default:

+							$this->error( 'unfixable', "Error: unrecognised object class \"$className\"", $oldId );

+					}

+				}

+				$dbr->freeResult( $res );

+			}

+

+			// Check local concat blob validity

+			$externalConcatBlobs = array();

+			if ( count( $concatBlobs ) ) {

+				$headerLength = 300;

+				$res = $dbr->select( 'text', array( 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ), 

+					array( 'old_id IN (' . implode( ',', array_keys( $concatBlobs ) ) . ')' ), $fname );

+				while ( $row = $dbr->fetchObject( $res ) ) {

+					$flags = explode( ',', $row->old_flags );

+					if ( in_array( 'external', $flags ) ) {

+						// Concat blob is in external storage?

+						if ( in_array( 'object', $flags ) ) {

+							$urlParts = explode( '/', $row->header );

+							if ( $urlParts[0] != 'DB:' ) {

+								$this->error( 'unfixable', "Error: unrecognised external storage type \"{$urlParts[0]}", $row->old_id );

+							} else {

+								$cluster = $urlParts[2];

+								$id = $urlParts[3];

+								if ( !isset( $externalConcatBlobs[$cluster][$id] ) ) {

+									$externalConcatBlobs[$cluster][$id] = array();

+								}

+								$externalConcatBlobs[$cluster][$id] = array_merge( 

+									$externalConcatBlobs[$cluster][$id], $concatBlobs[$row->old_id]

+								);

+							}

+						} else {

+							$this->error( 'unfixable', "Error: invalid flags \"{$row->old_flags}\" on concat bulk row {$row->old_id}",

+								$concatBlobs[$row->old_id] );

+						}

+					} elseif ( strcasecmp( substr( $row->header, 0, strlen( CONCAT_HEADER ) ), CONCAT_HEADER ) ) {

+						$this->error( 'restore text', "Error: Incorrect object header for concat bulk row {$row->old_id}", 

+							$concatBlobs[$row->old_id] );

+					} # else good

+

+					unset( $concatBlobs[$row->old_id] );

+				}

+				$dbr->freeResult( $res );

+			}

+

+			// Check targets of unresolved stubs

+			$this->checkExternalConcatBlobs( $externalConcatBlobs );

+

+			// next chunk

+		}

+

+		print "\n\nErrors:\n";

+		foreach( $this->errors as $name => $errors ) {

+			if ( count( $errors ) ) {

+				$description = $this->errorDescriptions[$name];

+				echo "$description: " . implode( ',', array_keys( $errors ) ) . "\n";

+			}

+		}

+

+		if ( count( $this->errors['restore text'] ) && $fix ) {

+			if ( (string)$xml !== '' ) {

+				$this->restoreText( array_keys( $this->errors['restore text'] ), $xml );

+			} else {

+				echo "Can't fix text, no XML backup specified\n";

+			}

+		}

+

+		print "\nFlag statistics:\n";

+		$total = array_sum( $flagStats );

+		foreach ( $flagStats as $flag => $count ) {

+			printf( "%-30s %10d %5.2f%%\n", $flag, $count, $count / $total * 100 );

+		}

+		print "\nLocal object statistics:\n";

+		$total = array_sum( $objectStats );

+		foreach ( $objectStats as $className => $count ) {

+			printf( "%-30s %10d %5.2f%%\n", $className, $count, $count / $total * 100 );

+		}

+	}

+

+

+	function error( $type, $msg, $ids ) {

+		if ( is_array( $ids ) && count( $ids ) == 1 ) {

+			$ids = reset( $ids );

+		}

+		if ( is_array( $ids ) ) {

+			$revIds = array();

+			foreach ( $ids as $id ) {

+				$revIds = array_merge( $revIds, array_keys( $this->oldIdMap, $id ) );

+			}

+			print "$msg in text rows " . implode( ', ', $ids ) . 

+				", revisions " . implode( ', ', $revIds ) . "\n";

+		} else {

+			$id = $ids;

+			$revIds = array_keys( $this->oldIdMap, $id );

+			if ( count( $revIds ) == 1 ) {

+				print "$msg in old_id $id, rev_id {$revIds[0]}\n";

+			} else {

+				print "$msg in old_id $id, revisions " . implode( ', ', $revIds ) . "\n";

+			}

+		}

+		$this->errors[$type] = $this->errors[$type] + array_flip( $revIds );

+	}

+

+	function checkExternalConcatBlobs( $externalConcatBlobs ) {

+		$fname = 'CheckStorage::checkExternalConcatBlobs';

+		if ( !count( $externalConcatBlobs ) ) {

+			return;

+		}

+

+		if ( is_null( $this->dbStore ) ) {

+			$this->dbStore = new ExternalStoreDB;

+		}

+		

+		foreach ( $externalConcatBlobs as $cluster => $oldIds ) {

+			$blobIds = array_keys( $oldIds );

+			$extDb =& $this->dbStore->getSlave( $cluster );

+			$blobsTable = $this->dbStore->getTable( $extDb );

+			$headerLength = strlen( CONCAT_HEADER );

+			$res = $extDb->select( $blobsTable, 

+				array( 'blob_id', "LEFT(blob_text, $headerLength) AS header" ), 

+				array( 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ), $fname );

+			while ( $row = $extDb->fetchObject( $res ) ) {

+				if ( strcasecmp( $row->header, CONCAT_HEADER ) ) {

+					$this->error( 'restore text', "Error: invalid header on target $cluster/{$row->blob_id} of two-part ES URL", 

+						$oldIds[$row->blob_id] );

+				}

+				unset( $oldIds[$row->blob_id] );

+

+			}

+			$extDb->freeResult( $res );

+

+			// Print errors for missing blobs rows

+			foreach ( $oldIds as $blobId => $oldIds ) {

+				$this->error( 'restore text', "Error: missing target $cluster/$blobId for two-part ES URL", $oldIds );

+			}

+		}

+	}

+

+	function restoreText( $revIds, $xml ) {

+		global $wgTmpDirectory, $wgDBname;

+

+		if ( !count( $revIds ) ) {

+			return;

+		}

+

+		print "Restoring text from XML backup...\n";

+

+		$revFileName = "$wgTmpDirectory/broken-revlist-$wgDBname";

+		$filteredXmlFileName = "$wgTmpDirectory/filtered-$wgDBname.xml";

+		

+		// Write revision list

+		if ( !file_put_contents( $revFileName, implode( "\n", $revIds ) ) ) {

+			echo "Error writing revision list, can't restore text\n";

+			return;

+		}

+

+		// Run mwdumper

+		echo "Filtering XML dump...\n";

+		$exitStatus = 0;

+		passthru( 'mwdumper ' . 

+			wfEscapeShellArg( 

+				"--output=file:$filteredXmlFileName",

+				"--filter=revlist:$revFileName",

+				$xml

+			), $exitStatus

+		);

+

+		if ( $exitStatus ) {

+			echo "mwdumper died with exit status $exitStatus\n";

+			return;

+		}

+

+		$file = fopen( $filteredXmlFileName, 'r' );

+		if ( !$file ) {

+			echo "Unable to open filtered XML file\n";

+			return;

+		}

+

+		$dbr =& wfGetDB( DB_SLAVE );

+		$dbw =& wfGetDB( DB_MASTER );

+		$dbr->ping();

+		$dbw->ping();

+		

+		$source = new ImportStreamSource( $file );

+		$importer = new WikiImporter( $source );

+		$importer->setRevisionCallback( array( &$this, 'importRevision' ) );

+		$importer->doImport();

+	}

+

+	function importRevision( &$revision, &$importer ) {

+		$fname = 'CheckStorage::importRevision';

+

+		$id = $revision->getID();

+		$text = $revision->getText();

+		if ( $text === '' ) {

+			// This is what happens if the revision was broken at the time the 

+			// dump was made. Unfortunately, it also happens if the revision was 

+			// legitimately blank, so there's no way to tell the difference. To

+			// be safe, we'll skip it and leave it broken

+			$id = $id ? $id : '';

+			echo "Revision $id is blank in the dump, may have been broken before export\n";

+			return;

+		}

+

+		if ( !$id )  {

+			// No ID, can't import

+			echo "No id tag in revision, can't import\n";

+			return;

+		}

+

+		// Find text row again

+		$dbr =& wfGetDB( DB_SLAVE );

+		$oldId = $dbr->selectField( 'revision', 'rev_text_id', array( 'rev_id' => $id ), $fname );

+		if ( !$oldId ) {

+			echo "Missing revision row for rev_id $id\n";

+			return;

+		}

+

+		// Compress the text

+		$flags = Revision::compressRevisionText( $text );

+

+		// Update the text row

+		$dbw->update( 'text', 

+			array( 'old_flags' => $flags, 'old_text' => $text ),

+			array( 'old_id' => $oldId ),

+			$fname, array( 'LIMIT' => 1 )

+		);

+

+		// Remove it from the unfixed list and add it to the fixed list

+		unset( $this->errors['restore text'][$id] );

+		$this->errors['fixed'][$id] = true;

+	}

+}

+?>

diff --git a/maintenance/storage/compressOld.inc b/maintenance/storage/compressOld.inc
new file mode 100644
index 00000000..b7d7094f
--- /dev/null
+++ b/maintenance/storage/compressOld.inc
@@ -0,0 +1,300 @@
+<?php
+/**
+ * @package MediaWiki
+ * @subpackage Maintenance
+ */
+
+/** */
+require_once( 'Revision.php' );
+require_once( 'ExternalStoreDB.php' );
+
+/** @todo document */
+function compressOldPages( $start = 0, $extdb = '' ) {
+	$fname = 'compressOldPages';
+
+	$chunksize = 50;
+	print "Starting from old_id $start...\n";
+	$dbw =& wfGetDB( DB_MASTER );
+	do {
+		$end = $start + $chunksize;
+		$res = $dbw->select( 'text', array( 'old_id','old_flags','old_namespace','old_title','old_text' ),
+			"old_id>=$start", $fname, array( 'ORDER BY' => 'old_id', 'LIMIT' => $chunksize, 'FOR UPDATE' ) );
+		if( $dbw->numRows( $res ) == 0 ) {
+			break;
+		}
+		$last = $start;
+		while( $row = $dbw->fetchObject( $res ) ) {
+			# print "  {$row->old_id} - {$row->old_namespace}:{$row->old_title}\n";
+			compressPage( $row, $extdb );
+			$last = $row->old_id;
+		}
+		$dbw->freeResult( $res );
+		$start = $last + 1; # Deletion may leave long empty stretches
+		print "$start...\n";
+	} while( true );
+}
+
+/** @todo document */
+function compressPage( $row, $extdb ) {
+	$fname = 'compressPage';
+	if ( false !== strpos( $row->old_flags, 'gzip' ) || false !== strpos( $row->old_flags, 'object' ) ) {
+		#print "Already compressed row {$row->old_id}\n";
+		return false;
+	}
+	$dbw =& wfGetDB( DB_MASTER );
+	$flags = $row->old_flags ? "{$row->old_flags},gzip" : "gzip";
+	$compress = gzdeflate( $row->old_text );
+
+	# Store in external storage if required
+	if ( $extdb !== '' ) {
+		$storeObj = new ExternalStoreDB;
+		$compress = $storeObj->store( $extdb, $compress );
+		if ( $compress === false ) {
+			print "Unable to store object\n";
+			return false;
+		}
+	}
+
+	# Update text row
+	$dbw->update( 'text',
+		array( /* SET */
+			'old_flags' => $flags,
+			'old_text' => $compress
+		), array( /* WHERE */
+			'old_id' => $row->old_id
+		), $fname, 'LIMIT 1'
+	);
+	return true;
+}
+
+define( 'LS_INDIVIDUAL', 0 );
+define( 'LS_CHUNKED', 1 );
+
+/** @todo document */
+function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorThreshold, $beginDate,
+	$endDate, $extdb="", $maxPageId = false )
+{
+	$fname = 'compressWithConcat';
+	$loadStyle = LS_CHUNKED;
+
+	$dbr =& wfGetDB( DB_SLAVE );
+	$dbw =& wfGetDB( DB_MASTER );
+
+	# Set up external storage
+	if ( $extdb != '' ) {
+		$storeObj = new ExternalStoreDB;
+	}
+
+	# Get all articles by page_id
+	if ( !$maxPageId ) {
+		$maxPageId = $dbr->selectField( 'page', 'max(page_id)', '', $fname );
+	}
+	print "Starting from $startId of $maxPageId\n";
+	$pageConds = array();
+
+	/*
+	if ( $exclude_ns0 ) {
+		print "Excluding main namespace\n";
+		$pageConds[] = 'page_namespace<>0';
+	}
+	if ( $queryExtra ) {
+                $pageConds[] = $queryExtra;
+	}
+	 */
+
+	# For each article, get a list of revisions which fit the criteria
+	
+	# No recompression, use a condition on old_flags
+	# Don't compress object type entities, because that might produce data loss when
+	# overwriting bulk storage concat rows. Don't compress external references, because
+	# the script doesn't yet delete rows from external storage.
+	$conds = array(
+		"old_flags NOT LIKE '%object%' AND old_flags NOT LIKE '%external%'");
+
+	if ( $beginDate ) {
+		$conds[] = "rev_timestamp>'" . $beginDate . "'";
+	}
+	if ( $endDate )  {
+		$conds[] = "rev_timestamp<'" . $endDate . "'";
+	}
+	if ( $loadStyle == LS_CHUNKED ) {
+		$tables = array( 'revision', 'text' );
+		$fields = array( 'rev_id', 'rev_text_id', 'old_flags', 'old_text' );
+		$conds[] = 'rev_text_id=old_id';
+		$revLoadOptions = 'FOR UPDATE';
+	} else {
+		$tables = array( 'revision' );
+		$fields = array( 'rev_id', 'rev_text_id' );
+		$revLoadOptions = array();
+	}
+
+	# Don't work with current revisions
+	# Don't lock the page table for update either -- TS 2006-04-04
+	#$tables[] = 'page';
+	#$conds[] = 'page_id=rev_page AND rev_id != page_latest';
+
+	$oldReadsSinceLastSlaveWait = 0;	#check slave lag periodically
+	$totalMatchingRevisions = 0;
+	$masterPos = false;
+	for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) {
+		wfWaitForSlaves( 5 );
+
+		# Wake up
+		$dbr->ping();		
+
+		# Get the page row
+		$pageRes = $dbr->select( 'page', 
+			array('page_id', 'page_namespace', 'page_title','page_latest'),
+			$pageConds + array('page_id' => $pageId), $fname );
+		if ( $dbr->numRows( $pageRes ) == 0 ) {
+			continue;
+		}
+		$pageRow = $dbr->fetchObject( $pageRes );
+
+		# Display progress
+		$titleObj = Title::makeTitle( $pageRow->page_namespace, $pageRow->page_title );
+		print "$pageId\t" . $titleObj->getPrefixedDBkey() . " ";
+
+		# Load revisions
+		$revRes = $dbw->select( $tables, $fields,
+			array( 
+				'rev_page' => $pageRow->page_id, 
+				# Don't operate on the current revision
+				# Use < instead of <> in case the current revision has changed 
+				# since the page select, which wasn't locking
+				'rev_id < ' . $pageRow->page_latest
+			) + $conds,
+			$fname,
+			$revLoadOptions
+		);
+		$revs = array();
+		while ( $revRow = $dbw->fetchObject( $revRes ) ) {
+			$revs[] = $revRow;
+		}
+
+		if ( count( $revs ) < 2) {
+			# No revisions matching, no further processing
+			print "\n";
+			continue;
+		}
+
+		# For each chunk
+		$i = 0;
+		while ( $i < count( $revs ) ) {
+			if ( $i < count( $revs ) - $maxChunkSize ) {
+				$thisChunkSize = $maxChunkSize;
+			} else {
+				$thisChunkSize = count( $revs ) - $i;
+			}
+
+			$chunk = new ConcatenatedGzipHistoryBlob();
+			$stubs = array();
+			$dbw->begin();
+			$usedChunk = false;
+			$primaryOldid = $revs[$i]->rev_text_id;
+
+			# Get the text of each revision and add it to the object
+			for ( $j = 0; $j < $thisChunkSize && $chunk->isHappy( $maxChunkFactor, $factorThreshold ); $j++ ) {
+				$oldid = $revs[$i + $j]->rev_text_id;
+
+				# Get text
+				if ( $loadStyle == LS_INDIVIDUAL ) {
+					$textRow = $dbw->selectRow( 'text',
+						array( 'old_flags', 'old_text' ),
+						array( 'old_id' => $oldid ),
+						$fname,
+						'FOR UPDATE'
+					);
+					$text = Revision::getRevisionText( $textRow );
+				} else {
+					$text = Revision::getRevisionText( $revs[$i + $j] );
+				}
+
+				if ( $text === false ) {
+					print "\nError, unable to get text in old_id $oldid\n";
+					#$dbw->delete( 'old', array( 'old_id' => $oldid ) );
+				}
+
+				if ( $extdb == "" && $j == 0 ) {
+					$chunk->setText( $text );
+					print '.';
+				} else {
+					# Don't make a stub if it's going to be longer than the article
+					# Stubs are typically about 100 bytes
+					if ( strlen( $text ) < 120 ) {
+						$stub = false;
+						print 'x';
+					} else {
+						$stub = $chunk->addItem( $text );
+						$stub->setLocation( $primaryOldid );
+						$stub->setReferrer( $oldid );
+						print '.';
+						$usedChunk = true;
+					}
+					$stubs[$j] = $stub;
+				}
+			}
+			$thisChunkSize = $j;
+
+			# If we couldn't actually use any stubs because the pages were too small, do nothing
+			if ( $usedChunk ) {
+				if ( $extdb != "" ) {
+					# Move blob objects to External Storage
+					$stored = $storeObj->store( $extdb, serialize( $chunk ));
+					if ($stored === false) {
+						print "Unable to store object\n";
+						return false;
+					}
+					# Store External Storage URLs instead of Stub placeholders
+					foreach ($stubs as $stub) {
+						if ($stub===false)
+							continue;
+						# $stored should provide base path to a BLOB
+						$url = $stored."/".$stub->getHash();
+						$dbw->update( 'text',
+							array( /* SET */
+								'old_text' => $url,
+								'old_flags' => 'external,utf-8',
+							), array ( /* WHERE */
+								'old_id' => $stub->getReferrer(),
+							)
+						);
+					}
+				} else {
+					# Store the main object locally
+					$dbw->update( 'text',
+						array( /* SET */
+							'old_text' => serialize( $chunk ),
+							'old_flags' => 'object,utf-8',
+						), array( /* WHERE */
+							'old_id' => $primaryOldid
+						)
+					);
+
+					# Store the stub objects
+					for ( $j = 1; $j < $thisChunkSize; $j++ ) {
+						# Skip if not compressing
+						if ( $stubs[$j] !== false ) {
+							$dbw->update( 'text',
+								array( /* SET */
+									'old_text' => serialize($stubs[$j]),
+									'old_flags' => 'object,utf-8',
+								), array( /* WHERE */
+									'old_id' => $revs[$i + $j]->rev_text_id
+								)
+							);
+						}
+					}
+				}
+			}
+			# Done, next
+			print "/";
+			$dbw->commit();
+			$i += $thisChunkSize;
+			wfWaitForSlaves( 5 );
+		}
+		print "\n";
+	}
+	return true;
+}
+?>
diff --git a/maintenance/storage/compressOld.php b/maintenance/storage/compressOld.php
new file mode 100644
index 00000000..d597f1df
--- /dev/null
+++ b/maintenance/storage/compressOld.php
@@ -0,0 +1,82 @@
+<?php
+/**
+ * Compress the text of a wiki
+ *
+ * @package MediaWiki
+ * @subpackage Maintenance
+ */
+
+/** */
+
+/**
+ * Usage:
+ *
+ * Non-wikimedia
+ * php compressOld.php [options...]
+ *
+ * Wikimedia
+ * php compressOld.php <database> [options...]
+ *
+ * Options are:
+ *  -t <type>           set compression type to either:
+ *                          gzip: compress revisions independently
+ *                          concat: concatenate revisions and compress in chunks (default)
+ *  -c <chunk-size>     maximum number of revisions in a concat chunk
+ *  -b <begin-date>     earliest date to check for uncompressed revisions
+ *  -e <end-date>       latest revision date to compress
+ *  -s <start-id>       the old_id to start from
+ *  -f <max-factor>     the maximum ratio of compressed chunk bytes to uncompressed avg. revision bytes
+ *  -h <threshold>      is a minimum number of KB, where <max-factor> cuts in
+ *  --extdb <cluster>   store specified revisions in an external cluster (untested)
+ *
+ */
+
+$optionsWithArgs = array( 't', 'c', 's', 'f', 'h', 'extdb', 'endid' );
+require_once( "../commandLine.inc" );
+require_once( "compressOld.inc" );
+
+if( !function_exists( "gzdeflate" ) ) {
+	print "You must enable zlib support in PHP to compress old revisions!\n";
+	print "Please see http://www.php.net/manual/en/ref.zlib.php\n\n";
+	wfDie();
+}
+
+$defaults = array(
+	't' => 'concat',
+	'c' => 20,
+	's' => 0,
+	'f' => 5,
+	'h' => 100,
+	'b' => '',
+    'e' => '',
+    'extdb' => '',
+    'endid' => false,
+);
+
+$options = $options + $defaults;
+
+if ( $options['t'] != 'concat' && $options['t'] != 'gzip' ) {
+	print "Type \"{$options['t']}\" not supported\n";
+}
+
+if ( $options['extdb'] != '' ) {
+	print "Compressing database $wgDBname to external cluster {$options['extdb']}\n" . str_repeat('-', 76) . "\n\n";
+} else {
+	print "Compressing database $wgDBname\n" . str_repeat('-', 76) . "\n\n";
+}
+
+$success = true;
+if ( $options['t'] == 'concat' ) {
+    $success = compressWithConcat( $options['s'], $options['c'], $options['f'], $options['h'], $options['b'],
+        $options['e'], $options['extdb'], $options['endid'] );
+} else {
+	compressOldPages( $options['s'], $options['extdb'] );
+}
+
+if ( $success ) {
+	print "Done.\n";
+}
+
+exit();
+
+?>
diff --git a/maintenance/storage/dumpRev.php b/maintenance/storage/dumpRev.php
new file mode 100644
index 00000000..4d0ccb58
--- /dev/null
+++ b/maintenance/storage/dumpRev.php
@@ -0,0 +1,14 @@
+<?php
+
+require_once( 'commandLine.inc' );
+$dbr =& wfGetDB( DB_SLAVE );
+$row = $dbr->selectRow( 'old', array( 'old_flags', 'old_text' ), array( 'old_id' => $args[0] ) );
+$obj = unserialize( $row->old_text );
+
+if ( get_class( $obj ) == 'concatenatedgziphistoryblob' ) {
+	print_r( array_keys( $obj->mItems ) );
+} else {
+	var_dump( $obj );
+}
+
+?>
diff --git a/maintenance/storage/make-blobs b/maintenance/storage/make-blobs
new file mode 100755
index 00000000..9eb7e83e
--- /dev/null
+++ b/maintenance/storage/make-blobs
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ X$2 == X ];then
+	echo 'Usage: make-blobs <server> <db>'
+	exit 1
+fi
+
+echo "CREATE DATABASE $2" | mysql -u wikiadmin -p`wikiadmin_pass` -h $1 && \
+mysql -u wikiadmin -p`wikiadmin_pass` -h $1 $2 < blobs.sql
+	
+	
diff --git a/maintenance/storage/moveToExternal.php b/maintenance/storage/moveToExternal.php
new file mode 100644
index 00000000..0b46f70b
--- /dev/null
+++ b/maintenance/storage/moveToExternal.php
@@ -0,0 +1,97 @@
+<?php
+
+define( 'REPORTING_INTERVAL', 100 );
+
+if ( !defined( 'MEDIAWIKI' ) ) {
+	$optionsWithArgs = array( 'm' );
+
+	require_once( '../commandLine.inc' );
+	require_once( 'ExternalStoreDB.php' );
+	require_once( 'resolveStubs.php' );
+
+	$fname = 'moveToExternal';
+
+	if ( !isset( $args[0] ) ) {
+		print "Usage: php moveToExternal.php [-m <maxid>] <cluster>\n";
+		exit;
+	}
+
+	$cluster = $args[0];
+	$dbw =& wfGetDB( DB_MASTER );
+
+	if ( isset( $options['m'] ) ) {
+		$maxID = $options['m'];
+	} else {
+		$maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
+	}
+
+	moveToExternal( $cluster, $maxID );
+}
+
+
+
+function moveToExternal( $cluster, $maxID ) {
+	$fname = 'moveToExternal';
+	$dbw =& wfGetDB( DB_MASTER );
+
+	print "Moving $maxID text rows to external storage\n";
+	$ext = new ExternalStoreDB;
+	for ( $id = 1; $id <= $maxID; $id++ ) {
+		if ( !($id % REPORTING_INTERVAL) ) {
+			print "$id\n";
+			wfWaitForSlaves( 5 );
+		}
+		$row = $dbw->selectRow( 'text', array( 'old_flags', 'old_text' ),
+			array(
+				'old_id' => $id,
+				"old_flags NOT LIKE '%external%'",
+			), $fname );
+		if ( !$row ) {
+			# Non-existent or already done
+			continue;
+		}
+
+		# Resolve stubs
+		$text = $row->old_text;
+		if ( $row->old_flags === '' ) {
+			$flags = 'external';
+		} else {
+			$flags = "{$row->old_flags},external";
+		}
+		
+		if ( strpos( $flags, 'object' ) !== false ) {
+			$obj = unserialize( $text );
+			$className = strtolower( get_class( $obj ) );
+			if ( $className == 'historyblobstub' ) {
+				resolveStub( $id, $row->old_text, $row->old_flags );
+				continue;
+			} elseif ( $className == 'historyblobcurstub' ) {
+				$text = gzdeflate( $obj->getText() );
+				$flags = 'utf-8,gzip,external';
+			} elseif ( $className == 'concatenatedgziphistoryblob' ) {
+				// Do nothing
+			} else {
+				print "Warning: unrecognised object class \"$className\"\n";
+				continue;
+			}
+		}
+
+		if ( strlen( $text ) < 100 ) {
+			// Don't move tiny revisions
+			continue;
+		}
+
+		#print "Storing "  . strlen( $text ) . " bytes to $url\n";
+
+		$url = $ext->store( $cluster, $text );
+		if ( !$url ) {
+			print "Error writing to external storage\n";
+			exit;
+		}
+		$dbw->update( 'text',
+			array( 'old_flags' => $flags, 'old_text' => $url ),
+			array( 'old_id' => $id ), $fname );
+	}
+}
+
+?>
diff --git a/maintenance/storage/resolveStubs.php b/maintenance/storage/resolveStubs.php
new file mode 100644
index 00000000..e93d5c97
--- /dev/null
+++ b/maintenance/storage/resolveStubs.php
@@ -0,0 +1,100 @@
+<?php
+
+define( 'REPORTING_INTERVAL', 100 );
+
+if ( !defined( 'MEDIAWIKI' ) ) {
+	$optionsWithArgs = array( 'm' );
+
+	require_once( '../commandLine.inc' );
+	require_once( 'includes/ExternalStoreDB.php' );
+
+	resolveStubs();
+}
+
+/**
+ * Convert history stubs that point to an external row to direct
+ * external pointers
+ */
+function resolveStubs() {
+	$fname = 'resolveStubs';
+
+	$dbr =& wfGetDB( DB_SLAVE );
+	$dbw =& wfGetDB( DB_MASTER );
+	$maxID = $dbr->selectField( 'text', 'MAX(old_id)', false, $fname );
+	$blockSize = 10000;
+	$numBlocks = intval( $maxID / $blockSize ) + 1;
+
+	for ( $b = 0; $b < $numBlocks; $b++ ) {
+		wfWaitForSlaves( 5 );
+		
+		printf( "%5.2f%%\n", $b / $numBlocks * 100 );
+		$start = intval($maxID / $numBlocks) * $b + 1;
+		$end = intval($maxID / $numBlocks) * ($b + 1);
+		$stubs = array();
+		$flagsArray = array();
+
+		
+		$res = $dbr->select( 'text', array( 'old_id', 'old_text', 'old_flags' ),
+			"old_id>=$start AND old_id<=$end " .
+			# Using a more restrictive flag set for now, until I do some more analysis -- TS
+			#"AND old_flags LIKE '%object%' AND old_flags NOT LIKE '%external%' ".
+			
+			"AND old_flags='object' " .
+			"AND old_text LIKE 'O:15:\"historyblobstub\"%'", $fname );
+		while ( $row = $dbr->fetchObject( $res ) ) {
+			resolveStub( $row->old_id, $row->old_text, $row->old_flags );
+		}
+		$dbr->freeResult( $res );
+
+		
+	}
+	print "100%\n";
+}
+
+/**
+ * Resolve a history stub
+ */
+function resolveStub( $id, $stubText, $flags ) {
+	$fname = 'resolveStub';
+
+	$stub = unserialize( $stubText );
+	$flags = explode( ',', $flags );
+
+	$dbr =& wfGetDB( DB_SLAVE );
+	$dbw =& wfGetDB( DB_MASTER );
+
+	if ( strtolower( get_class( $stub ) ) !== 'historyblobstub' ) {
+		print "Error found object of class " . get_class( $stub ) . ", expecting historyblobstub\n";
+		return;
+	}
+
+	# Get the (maybe) external row
+	$externalRow = $dbr->selectRow( 'text', array( 'old_text' ),
+		array( 'old_id' => $stub->mOldId, "old_flags LIKE '%external%'" ),
+		$fname
+	);
+
+	if ( !$externalRow ) {
+		# Object wasn't external
+		return;
+	}
+
+	# Preserve the legacy encoding flag, but switch from object to external
+	if ( in_array( 'utf-8', $flags ) ) {
+		$newFlags = 'external,utf-8';
+	} else {
+		$newFlags = 'external';
+	}
+
+	# Update the row
+	$dbw->update( 'text',
+		array( /* SET */
+			'old_flags' => $newFlags,
+			'old_text' => $externalRow->old_text . '/' . $stub->mHash
+		),
+		array( /* WHERE */
+			'old_id' => $id
+		), $fname
+	);
+}
+?>