1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
|
<?php
/**
* @defgroup ExternalStorage ExternalStorage
*/
/**
* Interface for data storage in external repositories.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
/**
* Constructor class for key/value blob data kept in external repositories.
*
* Objects in external stores are defined by a special URL. The URL is of
* the form "<store protocol>://<location>/<object name>". The protocol is used
* to determine what ExternalStoreMedium class is used. The location identifies
* particular storage instances or database clusters for store class to use.
*
* When an object is inserted into a store, the calling code uses a partial URL of
* the form "<store protocol>://<location>" and receives the full object URL on success.
* This is useful since object names can be sequential IDs, UUIDs, or hashes.
* Callers are not responsible for unique name generation.
*
* External repositories might be populated by maintenance/async
* scripts, thus partial moving of data may be possible, as well
* as the possibility to have any storage format (i.e. for archives).
*
* @ingroup ExternalStorage
*/
class ExternalStore {
/**
* Get an external store object of the given type, with the given parameters
*
* @param string $proto Type of external storage, should be a value in $wgExternalStores
* @param array $params Associative array of ExternalStoreMedium parameters
* @return ExternalStoreMedium|bool The store class or false on error
*/
public static function getStoreObject( $proto, array $params = array() ) {
global $wgExternalStores;
if ( !$wgExternalStores || !in_array( $proto, $wgExternalStores ) ) {
return false; // protocol not enabled
}
$class = 'ExternalStore' . ucfirst( $proto );
// Any custom modules should be added to $wgAutoLoadClasses for on-demand loading
return class_exists( $class ) ? new $class( $params ) : false;
}
/**
* Fetch data from given URL
*
* @param string $url The URL of the text to get
* @param array $params Associative array of ExternalStoreMedium parameters
* @return string|bool The text stored or false on error
* @throws MWException
*/
public static function fetchFromURL( $url, array $params = array() ) {
$parts = explode( '://', $url, 2 );
if ( count( $parts ) != 2 ) {
return false; // invalid URL
}
list( $proto, $path ) = $parts;
if ( $path == '' ) { // bad URL
return false;
}
$store = self::getStoreObject( $proto, $params );
if ( $store === false ) {
return false;
}
return $store->fetchFromURL( $url );
}
/**
* Fetch data from multiple URLs with a minimum of round trips
*
* @param array $urls The URLs of the text to get
* @return array Map from url to its data. Data is either string when found
* or false on failure.
*/
public static function batchFetchFromURLs( array $urls ) {
$batches = array();
foreach ( $urls as $url ) {
$scheme = parse_url( $url, PHP_URL_SCHEME );
if ( $scheme ) {
$batches[$scheme][] = $url;
}
}
$retval = array();
foreach ( $batches as $proto => $batchedUrls ) {
$store = self::getStoreObject( $proto );
if ( $store === false ) {
continue;
}
$retval += $store->batchFetchFromURLs( $batchedUrls );
}
// invalid, not found, db dead, etc.
$missing = array_diff( $urls, array_keys( $retval ) );
if ( $missing ) {
foreach ( $missing as $url ) {
$retval[$url] = false;
}
}
return $retval;
}
/**
* Store a data item to an external store, identified by a partial URL
* The protocol part is used to identify the class, the rest is passed to the
* class itself as a parameter.
*
* @param string $url A partial external store URL ("<store type>://<location>")
* @param $data string
* @param array $params Associative array of ExternalStoreMedium parameters
* @return string|bool The URL of the stored data item, or false on error
* @throws MWException
*/
public static function insert( $url, $data, array $params = array() ) {
$parts = explode( '://', $url, 2 );
if ( count( $parts ) != 2 ) {
return false; // invalid URL
}
list( $proto, $path ) = $parts;
if ( $path == '' ) { // bad URL
return false;
}
$store = self::getStoreObject( $proto, $params );
if ( $store === false ) {
return false;
} else {
return $store->store( $path, $data );
}
}
/**
* Like insert() above, but does more of the work for us.
* This function does not need a url param, it builds it by
* itself. It also fails-over to the next possible clusters
* provided by $wgDefaultExternalStore.
*
* @param string $data
* @param array $params Associative array of ExternalStoreMedium parameters
* @return string|bool The URL of the stored data item, or false on error
* @throws MWException
*/
public static function insertToDefault( $data, array $params = array() ) {
global $wgDefaultExternalStore;
return self::insertWithFallback( (array)$wgDefaultExternalStore, $data, $params );
}
/**
* Like insert() above, but does more of the work for us.
* This function does not need a url param, it builds it by
* itself. It also fails-over to the next possible clusters
* as provided in the first parameter.
*
* @param array $tryStores refer to $wgDefaultExternalStore
* @param string $data
* @param array $params Associative array of ExternalStoreMedium parameters
* @return string|bool The URL of the stored data item, or false on error
* @throws MWException
*/
public static function insertWithFallback( array $tryStores, $data, array $params = array() ) {
$error = false;
while ( count( $tryStores ) > 0 ) {
$index = mt_rand( 0, count( $tryStores ) - 1 );
$storeUrl = $tryStores[$index];
wfDebug( __METHOD__ . ": trying $storeUrl\n" );
list( $proto, $path ) = explode( '://', $storeUrl, 2 );
$store = self::getStoreObject( $proto, $params );
if ( $store === false ) {
throw new MWException( "Invalid external storage protocol - $storeUrl" );
}
try {
$url = $store->store( $path, $data ); // Try to save the object
} catch ( MWException $error ) {
$url = false;
}
if ( strlen( $url ) ) {
return $url; // Done!
} else {
unset( $tryStores[$index] ); // Don't try this one again!
$tryStores = array_values( $tryStores ); // Must have consecutive keys
wfDebugLog( 'ExternalStorage',
"Unable to store text to external storage $storeUrl" );
}
}
// All stores failed
if ( $error ) {
throw $error; // rethrow the last error
} else {
throw new MWException( "Unable to store text to external storage" );
}
}
/**
* @param $data string
* @param $wiki string
* @return string|bool The URL of the stored data item, or false on error
* @throws MWException
*/
public static function insertToForeignDefault( $data, $wiki ) {
return self::insertToDefault( $data, array( 'wiki' => $wiki ) );
}
}
|
