1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
<?php
/**
* Move revision's text to external storage
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup Maintenance ExternalStorage
*/
define( 'REPORTING_INTERVAL', 1 );
if ( !defined( 'MEDIAWIKI' ) ) {
require_once __DIR__ . '/../commandLine.inc';
require_once 'resolveStubs.php';
$fname = 'moveToExternal';
if ( !isset( $args[0] ) ) {
print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <cluster>\n";
exit;
}
$cluster = $args[0];
$dbw = wfGetDB( DB_MASTER );
if ( isset( $options['e'] ) ) {
$maxID = $options['e'];
} else {
$maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
}
$minID = isset( $options['s'] ) ? $options['s'] : 1;
moveToExternal( $cluster, $maxID, $minID );
}
function moveToExternal( $cluster, $maxID, $minID = 1 ) {
$fname = 'moveToExternal';
$dbw = wfGetDB( DB_MASTER );
$dbr = wfGetDB( DB_SLAVE );
$count = $maxID - $minID + 1;
$blockSize = 1000;
$numBlocks = ceil( $count / $blockSize );
print "Moving text rows from $minID to $maxID to external storage\n";
$ext = new ExternalStoreDB;
$numMoved = 0;
for ( $block = 0; $block < $numBlocks; $block++ ) {
$blockStart = $block * $blockSize + $minID;
$blockEnd = $blockStart + $blockSize - 1;
if ( !( $block % REPORTING_INTERVAL ) ) {
print "oldid=$blockStart, moved=$numMoved\n";
wfWaitForSlaves();
}
$res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ),
array(
"old_id BETWEEN $blockStart AND $blockEnd",
'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
), $fname );
foreach ( $res as $row ) {
# Resolve stubs
$text = $row->old_text;
$id = $row->old_id;
if ( $row->old_flags === '' ) {
$flags = 'external';
} else {
$flags = "{$row->old_flags},external";
}
if ( strpos( $flags, 'object' ) !== false ) {
$obj = unserialize( $text );
$className = strtolower( get_class( $obj ) );
if ( $className == 'historyblobstub' ) {
# resolveStub( $id, $row->old_text, $row->old_flags );
# $numStubs++;
continue;
} elseif ( $className == 'historyblobcurstub' ) {
$text = gzdeflate( $obj->getText() );
$flags = 'utf-8,gzip,external';
} elseif ( $className == 'concatenatedgziphistoryblob' ) {
// Do nothing
} else {
print "Warning: unrecognised object class \"$className\"\n";
continue;
}
} else {
$className = false;
}
if ( strlen( $text ) < 100 && $className === false ) {
// Don't move tiny revisions
continue;
}
# print "Storing " . strlen( $text ) . " bytes to $url\n";
# print "old_id=$id\n";
$url = $ext->store( $cluster, $text );
if ( !$url ) {
print "Error writing to external storage\n";
exit;
}
$dbw->update( 'text',
array( 'old_flags' => $flags, 'old_text' => $url ),
array( 'old_id' => $id ), $fname );
$numMoved++;
}
}
}
|
