summaryrefslogtreecommitdiff
path: root/includes/filebackend
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2013-08-12 09:28:15 +0200
committerPierre Schmitz <pierre@archlinux.de>2013-08-12 09:28:15 +0200
commit08aa4418c30cfc18ccc69a0f0f9cb9e17be6c196 (patch)
tree577a29fb579188d16003a209ce2a2e9c5b0aa2bd /includes/filebackend
parentcacc939b34e315b85e2d72997811eb6677996cc1 (diff)
Update to MediaWiki 1.21.1
Diffstat (limited to 'includes/filebackend')
-rw-r--r--includes/filebackend/FSFile.php49
-rw-r--r--includes/filebackend/FSFileBackend.php341
-rw-r--r--includes/filebackend/FileBackend.php370
-rw-r--r--includes/filebackend/FileBackendGroup.php10
-rw-r--r--includes/filebackend/FileBackendMultiWrite.php91
-rw-r--r--includes/filebackend/FileBackendStore.php502
-rw-r--r--includes/filebackend/FileOp.php332
-rw-r--r--includes/filebackend/FileOpBatch.php62
-rw-r--r--includes/filebackend/README208
-rw-r--r--includes/filebackend/SwiftFileBackend.php483
-rw-r--r--includes/filebackend/TempFSFile.php13
-rw-r--r--includes/filebackend/filejournal/DBFileJournal.php32
-rw-r--r--includes/filebackend/filejournal/FileJournal.php65
-rw-r--r--includes/filebackend/lockmanager/DBLockManager.php227
-rw-r--r--includes/filebackend/lockmanager/FSLockManager.php42
-rw-r--r--includes/filebackend/lockmanager/LSLockManager.php8
-rw-r--r--includes/filebackend/lockmanager/LockManager.php324
-rw-r--r--includes/filebackend/lockmanager/LockManagerGroup.php54
-rw-r--r--includes/filebackend/lockmanager/MemcLockManager.php94
-rw-r--r--includes/filebackend/lockmanager/QuorumLockManager.php230
-rw-r--r--includes/filebackend/lockmanager/ScopedLock.php102
21 files changed, 2405 insertions, 1234 deletions
diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php
index e07c99d4..7d0dbd52 100644
--- a/includes/filebackend/FSFile.php
+++ b/includes/filebackend/FSFile.php
@@ -28,11 +28,12 @@
*/
class FSFile {
protected $path; // path to file
+ private $sha1Base36 = null; // File Sha1Base36
/**
* Sets up the file object
*
- * @param $path string Path to temporary file on local disk
+ * @param string $path Path to temporary file on local disk
* @throws MWException
*/
public function __construct( $path ) {
@@ -86,8 +87,8 @@ class FSFile {
/**
* Guess the MIME type from the file contents alone
- *
- * @return string
+ *
+ * @return string
*/
public function getMimeType() {
return MimeMagic::singleton()->guessMimeType( $this->path, false );
@@ -104,7 +105,7 @@ class FSFile {
*/
public function getProps( $ext = true ) {
wfProfileIn( __METHOD__ );
- wfDebug( __METHOD__.": Getting file info for $this->path\n" );
+ wfDebug( __METHOD__ . ": Getting file info for $this->path\n" );
$info = self::placeholderProps();
$info['fileExists'] = $this->exists();
@@ -131,7 +132,7 @@ class FSFile {
# Height, width and metadata
$handler = MediaHandler::getHandler( $info['mime'] );
if ( $handler ) {
- $tempImage = (object)array();
+ $tempImage = (object)array(); // XXX (hack for File object)
$info['metadata'] = $handler->getMetadata( $tempImage, $this->path );
$gis = $handler->getImageSize( $tempImage, $this->path, $info['metadata'] );
if ( is_array( $gis ) ) {
@@ -140,9 +141,9 @@ class FSFile {
}
$info['sha1'] = $this->getSha1Base36();
- wfDebug(__METHOD__.": $this->path loaded, {$info['size']} bytes, {$info['mime']}.\n");
+ wfDebug( __METHOD__ . ": $this->path loaded, {$info['size']} bytes, {$info['mime']}.\n" );
} else {
- wfDebug(__METHOD__.": $this->path NOT FOUND!\n");
+ wfDebug( __METHOD__ . ": $this->path NOT FOUND!\n" );
}
wfProfileOut( __METHOD__ );
@@ -193,25 +194,32 @@ class FSFile {
* 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36
* fairly neatly.
*
+ * @param $recache bool
* @return bool|string False on failure
*/
- public function getSha1Base36() {
+ public function getSha1Base36( $recache = false ) {
wfProfileIn( __METHOD__ );
+ if ( $this->sha1Base36 !== null && !$recache ) {
+ wfProfileOut( __METHOD__ );
+ return $this->sha1Base36;
+ }
+
wfSuppressWarnings();
- $hash = sha1_file( $this->path );
+ $this->sha1Base36 = sha1_file( $this->path );
wfRestoreWarnings();
- if ( $hash !== false ) {
- $hash = wfBaseConvert( $hash, 16, 36, 31 );
+
+ if ( $this->sha1Base36 !== false ) {
+ $this->sha1Base36 = wfBaseConvert( $this->sha1Base36, 16, 36, 31 );
}
wfProfileOut( __METHOD__ );
- return $hash;
+ return $this->sha1Base36;
}
/**
* Get the final file extension from a file system path
- *
+ *
* @param $path string
* @return string
*/
@@ -223,7 +231,7 @@ class FSFile {
/**
* Get an associative array containing information about a file in the local filesystem.
*
- * @param $path String: absolute local filesystem path
+ * @param string $path absolute local filesystem path
* @param $ext Mixed: the file extension, or true to extract it from the filename.
* Set it to false to ignore the extension.
*
@@ -242,11 +250,18 @@ class FSFile {
* fairly neatly.
*
* @param $path string
+ * @param $recache bool
*
* @return bool|string False on failure
*/
- public static function getSha1Base36FromPath( $path ) {
- $fsFile = new self( $path );
- return $fsFile->getSha1Base36();
+ public static function getSha1Base36FromPath( $path, $recache = false ) {
+ static $sha1Base36 = array();
+
+ if ( !isset( $sha1Base36[$path] ) || $recache ) {
+ $fsFile = new self( $path );
+ $sha1Base36[$path] = $fsFile->getSha1Base36();
+ }
+
+ return $sha1Base36[$path];
}
}
diff --git a/includes/filebackend/FSFileBackend.php b/includes/filebackend/FSFileBackend.php
index 93495340..c9769989 100644
--- a/includes/filebackend/FSFileBackend.php
+++ b/includes/filebackend/FSFileBackend.php
@@ -46,6 +46,7 @@ class FSFileBackend extends FileBackendStore {
protected $fileOwner; // string; required OS username to own files
protected $currentUser; // string; OS username running this script
+ /** @var Array */
protected $hadWarningErrors = array();
/**
@@ -69,7 +70,7 @@ class FSFileBackend extends FileBackendStore {
if ( isset( $config['containerPaths'] ) ) {
$this->containerPaths = (array)$config['containerPaths'];
foreach ( $this->containerPaths as &$path ) {
- $path = rtrim( $path, '/' ); // remove trailing slash
+ $path = rtrim( $path, '/' ); // remove trailing slash
}
}
@@ -101,7 +102,7 @@ class FSFileBackend extends FileBackendStore {
/**
* Sanity check a relative file system path for validity
*
- * @param $path string Normalized relative path
+ * @param string $path Normalized relative path
* @return bool
*/
protected function isLegalRelPath( $path ) {
@@ -136,7 +137,7 @@ class FSFileBackend extends FileBackendStore {
/**
* Get the absolute file system path for a storage path
*
- * @param $storagePath string Storage path
+ * @param string $storagePath Storage path
* @return string|null
*/
protected function resolveToFSPath( $storagePath ) {
@@ -144,7 +145,7 @@ class FSFileBackend extends FileBackendStore {
if ( $relPath === null ) {
return null; // invalid
}
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $storagePath );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $storagePath );
$fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
if ( $relPath != '' ) {
$fsPath .= "/{$relPath}";
@@ -178,10 +179,10 @@ class FSFileBackend extends FileBackendStore {
}
/**
- * @see FileBackendStore::doStoreInternal()
+ * @see FileBackendStore::doCreateInternal()
* @return Status
*/
- protected function doStoreInternal( array $params ) {
+ protected function doCreateInternal( array $params ) {
$status = Status::newGood();
$dest = $this->resolveToFSPath( $params['dst'] );
@@ -190,27 +191,74 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- if ( file_exists( $dest ) ) {
- if ( !empty( $params['overwrite'] ) ) {
- $ok = unlink( $dest );
- if ( !$ok ) {
- $status->fatal( 'backend-fail-delete', $params['dst'] );
- return $status;
- }
- } else {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ if ( !empty( $params['async'] ) ) { // deferred
+ $tempFile = TempFSFile::factory( 'create_', 'tmp' );
+ if ( !$tempFile ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
+ return $status;
+ }
+ $this->trapWarnings();
+ $bytes = file_put_contents( $tempFile->getPath(), $params['content'] );
+ $this->untrapWarnings();
+ if ( $bytes === false ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
+ return $status;
+ }
+ $cmd = implode( ' ', array(
+ wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite)
+ wfEscapeShellArg( $this->cleanPathSlashes( $tempFile->getPath() ) ),
+ wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
+ ) );
+ $status->value = new FSFileOpHandle( $this, $params, 'Create', $cmd, $dest );
+ $tempFile->bind( $status->value );
+ } else { // immediate write
+ $this->trapWarnings();
+ $bytes = file_put_contents( $dest, $params['content'] );
+ $this->untrapWarnings();
+ if ( $bytes === false ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
return $status;
}
+ $this->chmod( $dest );
+ }
+
+ return $status;
+ }
+
+ /**
+ * @see FSFileBackend::doExecuteOpHandlesInternal()
+ */
+ protected function _getResponseCreate( $errors, Status $status, array $params, $cmd ) {
+ if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
+ trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
+ }
+ }
+
+ /**
+ * @see FileBackendStore::doStoreInternal()
+ * @return Status
+ */
+ protected function doStoreInternal( array $params ) {
+ $status = Status::newGood();
+
+ $dest = $this->resolveToFSPath( $params['dst'] );
+ if ( $dest === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+ return $status;
}
if ( !empty( $params['async'] ) ) { // deferred
- $cmd = implode( ' ', array( wfIsWindows() ? 'COPY' : 'cp',
+ $cmd = implode( ' ', array(
+ wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite)
wfEscapeShellArg( $this->cleanPathSlashes( $params['src'] ) ),
wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
) );
$status->value = new FSFileOpHandle( $this, $params, 'Store', $cmd, $dest );
} else { // immediate write
+ $this->trapWarnings();
$ok = copy( $params['src'], $dest );
+ $this->untrapWarnings();
// In some cases (at least over NFS), copy() returns true when it fails
if ( !$ok || ( filesize( $params['src'] ) !== filesize( $dest ) ) ) {
if ( $ok ) { // PHP bug
@@ -255,31 +303,30 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- if ( file_exists( $dest ) ) {
- if ( !empty( $params['overwrite'] ) ) {
- $ok = unlink( $dest );
- if ( !$ok ) {
- $status->fatal( 'backend-fail-delete', $params['dst'] );
- return $status;
- }
- } else {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
+ if ( !is_file( $source ) ) {
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-copy', $params['src'] );
}
+ return $status; // do nothing; either OK or bad status
}
if ( !empty( $params['async'] ) ) { // deferred
- $cmd = implode( ' ', array( wfIsWindows() ? 'COPY' : 'cp',
+ $cmd = implode( ' ', array(
+ wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite)
wfEscapeShellArg( $this->cleanPathSlashes( $source ) ),
wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
) );
$status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd, $dest );
} else { // immediate write
+ $this->trapWarnings();
$ok = copy( $source, $dest );
+ $this->untrapWarnings();
// In some cases (at least over NFS), copy() returns true when it fails
if ( !$ok || ( filesize( $source ) !== filesize( $dest ) ) ) {
if ( $ok ) { // PHP bug
+ $this->trapWarnings();
unlink( $dest ); // remove broken file
+ $this->untrapWarnings();
trigger_error( __METHOD__ . ": copy() failed but returned true." );
}
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
@@ -320,30 +367,24 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- if ( file_exists( $dest ) ) {
- if ( !empty( $params['overwrite'] ) ) {
- // Windows does not support moving over existing files
- if ( wfIsWindows() ) {
- $ok = unlink( $dest );
- if ( !$ok ) {
- $status->fatal( 'backend-fail-delete', $params['dst'] );
- return $status;
- }
- }
- } else {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
+ if ( !is_file( $source ) ) {
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-move', $params['src'] );
}
+ return $status; // do nothing; either OK or bad status
}
if ( !empty( $params['async'] ) ) { // deferred
- $cmd = implode( ' ', array( wfIsWindows() ? 'MOVE' : 'mv',
+ $cmd = implode( ' ', array(
+ wfIsWindows() ? 'MOVE /Y' : 'mv', // (overwrite)
wfEscapeShellArg( $this->cleanPathSlashes( $source ) ),
wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
) );
$status->value = new FSFileOpHandle( $this, $params, 'Move', $cmd );
} else { // immediate write
+ $this->trapWarnings();
$ok = rename( $source, $dest );
+ $this->untrapWarnings();
clearstatcache(); // file no longer at source
if ( !$ok ) {
$status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
@@ -385,12 +426,15 @@ class FSFileBackend extends FileBackendStore {
}
if ( !empty( $params['async'] ) ) { // deferred
- $cmd = implode( ' ', array( wfIsWindows() ? 'DEL' : 'unlink',
+ $cmd = implode( ' ', array(
+ wfIsWindows() ? 'DEL' : 'unlink',
wfEscapeShellArg( $this->cleanPathSlashes( $source ) )
) );
$status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd );
} else { // immediate write
+ $this->trapWarnings();
$ok = unlink( $source );
+ $this->untrapWarnings();
if ( !$ok ) {
$status->fatal( 'backend-fail-delete', $params['src'] );
return $status;
@@ -411,89 +455,27 @@ class FSFileBackend extends FileBackendStore {
}
/**
- * @see FileBackendStore::doCreateInternal()
- * @return Status
- */
- protected function doCreateInternal( array $params ) {
- $status = Status::newGood();
-
- $dest = $this->resolveToFSPath( $params['dst'] );
- if ( $dest === null ) {
- $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
- return $status;
- }
-
- if ( file_exists( $dest ) ) {
- if ( !empty( $params['overwrite'] ) ) {
- $ok = unlink( $dest );
- if ( !$ok ) {
- $status->fatal( 'backend-fail-delete', $params['dst'] );
- return $status;
- }
- } else {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
- }
-
- if ( !empty( $params['async'] ) ) { // deferred
- $tempFile = TempFSFile::factory( 'create_', 'tmp' );
- if ( !$tempFile ) {
- $status->fatal( 'backend-fail-create', $params['dst'] );
- return $status;
- }
- $bytes = file_put_contents( $tempFile->getPath(), $params['content'] );
- if ( $bytes === false ) {
- $status->fatal( 'backend-fail-create', $params['dst'] );
- return $status;
- }
- $cmd = implode( ' ', array( wfIsWindows() ? 'COPY' : 'cp',
- wfEscapeShellArg( $this->cleanPathSlashes( $tempFile->getPath() ) ),
- wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
- ) );
- $status->value = new FSFileOpHandle( $this, $params, 'Create', $cmd, $dest );
- $tempFile->bind( $status->value );
- } else { // immediate write
- $bytes = file_put_contents( $dest, $params['content'] );
- if ( $bytes === false ) {
- $status->fatal( 'backend-fail-create', $params['dst'] );
- return $status;
- }
- $this->chmod( $dest );
- }
-
- return $status;
- }
-
- /**
- * @see FSFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseCreate( $errors, Status $status, array $params, $cmd ) {
- if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
- $status->fatal( 'backend-fail-create', $params['dst'] );
- trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
- }
- }
-
- /**
* @see FileBackendStore::doPrepareInternal()
* @return Status
*/
protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
$dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
$existed = is_dir( $dir ); // already there?
- if ( !wfMkdirParents( $dir ) ) { // make directory and its parents
+ // Create the directory and its parents as needed...
+ $this->trapWarnings();
+ if ( !wfMkdirParents( $dir ) ) {
$status->fatal( 'directorycreateerror', $params['dir'] ); // fails on races
} elseif ( !is_writable( $dir ) ) {
$status->fatal( 'directoryreadonlyerror', $params['dir'] );
} elseif ( !is_readable( $dir ) ) {
$status->fatal( 'directorynotreadableerror', $params['dir'] );
}
+ $this->untrapWarnings();
+ // Respect any 'noAccess' or 'noListing' flags...
if ( is_dir( $dir ) && !$existed ) {
- // Respect any 'noAccess' or 'noListing' flags...
$status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) );
}
return $status;
@@ -505,24 +487,26 @@ class FSFileBackend extends FileBackendStore {
*/
protected function doSecureInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
$dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
// Seed new directories with a blank index.html, to prevent crawling...
if ( !empty( $params['noListing'] ) && !file_exists( "{$dir}/index.html" ) ) {
+ $this->trapWarnings();
$bytes = file_put_contents( "{$dir}/index.html", $this->indexHtmlPrivate() );
+ $this->untrapWarnings();
if ( $bytes === false ) {
$status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' );
- return $status;
}
}
// Add a .htaccess file to the root of the container...
if ( !empty( $params['noAccess'] ) && !file_exists( "{$contRoot}/.htaccess" ) ) {
+ $this->trapWarnings();
$bytes = file_put_contents( "{$contRoot}/.htaccess", $this->htaccessPrivate() );
+ $this->untrapWarnings();
if ( $bytes === false ) {
$storeDir = "mwstore://{$this->name}/{$shortCont}";
$status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" );
- return $status;
}
}
return $status;
@@ -534,25 +518,27 @@ class FSFileBackend extends FileBackendStore {
*/
protected function doPublishInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
$dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
// Unseed new directories with a blank index.html, to allow crawling...
if ( !empty( $params['listing'] ) && is_file( "{$dir}/index.html" ) ) {
$exists = ( file_get_contents( "{$dir}/index.html" ) === $this->indexHtmlPrivate() );
+ $this->trapWarnings();
if ( $exists && !unlink( "{$dir}/index.html" ) ) { // reverse secure()
$status->fatal( 'backend-fail-delete', $params['dir'] . '/index.html' );
- return $status;
}
+ $this->untrapWarnings();
}
// Remove the .htaccess file from the root of the container...
if ( !empty( $params['access'] ) && is_file( "{$contRoot}/.htaccess" ) ) {
$exists = ( file_get_contents( "{$contRoot}/.htaccess" ) === $this->htaccessPrivate() );
+ $this->trapWarnings();
if ( $exists && !unlink( "{$contRoot}/.htaccess" ) ) { // reverse secure()
$storeDir = "mwstore://{$this->name}/{$shortCont}";
$status->fatal( 'backend-fail-delete', "{$storeDir}/.htaccess" );
- return $status;
}
+ $this->untrapWarnings();
}
return $status;
}
@@ -563,14 +549,14 @@ class FSFileBackend extends FileBackendStore {
*/
protected function doCleanInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
$dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
- wfSuppressWarnings();
+ $this->trapWarnings();
if ( is_dir( $dir ) ) {
rmdir( $dir ); // remove directory if empty
}
- wfRestoreWarnings();
+ $this->untrapWarnings();
return $status;
}
@@ -612,7 +598,7 @@ class FSFileBackend extends FileBackendStore {
* @return bool|null
*/
protected function doDirectoryExists( $fullCont, $dirRel, array $params ) {
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
$dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
@@ -628,7 +614,7 @@ class FSFileBackend extends FileBackendStore {
* @return Array|null
*/
public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) {
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
$dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
$exists = is_dir( $dir );
@@ -644,10 +630,10 @@ class FSFileBackend extends FileBackendStore {
/**
* @see FileBackendStore::getFileListInternal()
- * @return array|FSFileBackendFileList|null
+ * @return Array|FSFileBackendFileList|null
*/
public function getFileListInternal( $fullCont, $dirRel, array $params ) {
- list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
$dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
$exists = is_dir( $dir );
@@ -662,44 +648,58 @@ class FSFileBackend extends FileBackendStore {
}
/**
- * @see FileBackendStore::getLocalReference()
- * @return FSFile|null
+ * @see FileBackendStore::doGetLocalReferenceMulti()
+ * @return Array
*/
- public function getLocalReference( array $params ) {
- $source = $this->resolveToFSPath( $params['src'] );
- if ( $source === null ) {
- return null;
+ protected function doGetLocalReferenceMulti( array $params ) {
+ $fsFiles = array(); // (path => FSFile)
+
+ foreach ( $params['srcs'] as $src ) {
+ $source = $this->resolveToFSPath( $src );
+ if ( $source === null || !is_file( $source ) ) {
+ $fsFiles[$src] = null; // invalid path or file does not exist
+ } else {
+ $fsFiles[$src] = new FSFile( $source );
+ }
}
- return new FSFile( $source );
+
+ return $fsFiles;
}
/**
- * @see FileBackendStore::getLocalCopy()
- * @return null|TempFSFile
+ * @see FileBackendStore::doGetLocalCopyMulti()
+ * @return Array
*/
- public function getLocalCopy( array $params ) {
- $source = $this->resolveToFSPath( $params['src'] );
- if ( $source === null ) {
- return null;
- }
-
- // Create a new temporary file with the same extension...
- $ext = FileBackend::extensionFromPath( $params['src'] );
- $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
- if ( !$tmpFile ) {
- return null;
- }
- $tmpPath = $tmpFile->getPath();
+ protected function doGetLocalCopyMulti( array $params ) {
+ $tmpFiles = array(); // (path => TempFSFile)
- // Copy the source file over the temp file
- $ok = copy( $source, $tmpPath );
- if ( !$ok ) {
- return null;
+ foreach ( $params['srcs'] as $src ) {
+ $source = $this->resolveToFSPath( $src );
+ if ( $source === null ) {
+ $tmpFiles[$src] = null; // invalid path
+ } else {
+ // Create a new temporary file with the same extension...
+ $ext = FileBackend::extensionFromPath( $src );
+ $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
+ if ( !$tmpFile ) {
+ $tmpFiles[$src] = null;
+ } else {
+ $tmpPath = $tmpFile->getPath();
+ // Copy the source file over the temp file
+ $this->trapWarnings();
+ $ok = copy( $source, $tmpPath );
+ $this->untrapWarnings();
+ if ( !$ok ) {
+ $tmpFiles[$src] = null;
+ } else {
+ $this->chmod( $tmpPath );
+ $tmpFiles[$src] = $tmpFile;
+ }
+ }
+ }
}
- $this->chmod( $tmpPath );
-
- return $tmpFile;
+ return $tmpFiles;
}
/**
@@ -747,13 +747,13 @@ class FSFileBackend extends FileBackendStore {
/**
* Chmod a file, suppressing the warnings
*
- * @param $path string Absolute file system path
+ * @param string $path Absolute file system path
* @return bool Success
*/
protected function chmod( $path ) {
- wfSuppressWarnings();
+ $this->trapWarnings();
$ok = chmod( $path, $this->fileMode );
- wfRestoreWarnings();
+ $this->untrapWarnings();
return $ok;
}
@@ -779,7 +779,7 @@ class FSFileBackend extends FileBackendStore {
/**
* Clean up directory separators for the given OS
*
- * @param $path string FS path
+ * @param string $path FS path
* @return string
*/
protected function cleanPathSlashes( $path ) {
@@ -789,12 +789,11 @@ class FSFileBackend extends FileBackendStore {
/**
* Listen for E_WARNING errors and track whether any happen
*
- * @return bool
+ * @return void
*/
protected function trapWarnings() {
$this->hadWarningErrors[] = false; // push to stack
set_error_handler( array( $this, 'handleWarning' ), E_WARNING );
- return false; // invoke normal PHP error handler
}
/**
@@ -808,9 +807,12 @@ class FSFileBackend extends FileBackendStore {
}
/**
+ * @param $errno integer
+ * @param $errstr string
* @return bool
*/
- private function handleWarning() {
+ private function handleWarning( $errno, $errstr ) {
+ wfDebugLog( 'FSFileBackend', $errstr ); // more detailed error logging
$this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true;
return true; // suppress from PHP handler
}
@@ -855,16 +857,19 @@ abstract class FSFileBackendList implements Iterator {
protected $params = array();
/**
- * @param $dir string file system directory
+ * @param string $dir file system directory
* @param $params array
*/
public function __construct( $dir, array $params ) {
- $dir = realpath( $dir ); // normalize
- $this->suffixStart = strlen( $dir ) + 1; // size of "path/to/dir/"
+ $path = realpath( $dir ); // normalize
+ if( $path === false ) {
+ $path = $dir;
+ }
+ $this->suffixStart = strlen( $path ) + 1; // size of "path/to/dir/"
$this->params = $params;
try {
- $this->iter = $this->initIterator( $dir );
+ $this->iter = $this->initIterator( $path );
} catch ( UnexpectedValueException $e ) {
$this->iter = null; // bad permissions? deleted?
}
@@ -873,7 +878,7 @@ abstract class FSFileBackendList implements Iterator {
/**
* Return an appropriate iterator object to wrap
*
- * @param $dir string file system directory
+ * @param string $dir file system directory
* @return Iterator
*/
protected function initIterator( $dir ) {
@@ -956,8 +961,12 @@ abstract class FSFileBackendList implements Iterator {
* @param $path string
* @return string
*/
- protected function getRelPath( $path ) {
- return strtr( substr( realpath( $path ), $this->suffixStart ), '\\', '/' );
+ protected function getRelPath( $dir ) {
+ $path = realpath( $dir );
+ if( $path === false ) {
+ $path = $dir;
+ }
+ return strtr( substr( $path, $this->suffixStart ), '\\', '/' );
}
}
diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php
index 76c761b0..f40b8c16 100644
--- a/includes/filebackend/FileBackend.php
+++ b/includes/filebackend/FileBackend.php
@@ -37,14 +37,18 @@
* Outside callers can assume that all backends will have these functions.
*
* All "storage paths" are of the format "mwstore://<backend>/<container>/<path>".
- * The "<path>" portion is a relative path that uses UNIX file system (FS)
- * notation, though any particular backend may not actually be using a local
- * filesystem. Therefore, the relative paths are only virtual.
+ * The "backend" portion is unique name for MediaWiki to refer to a backend, while
+ * the "container" portion is a top-level directory of the backend. The "path" portion
+ * is a relative path that uses UNIX file system (FS) notation, though any particular
+ * backend may not actually be using a local filesystem. Therefore, the relative paths
+ * are only virtual.
*
* Backend contents are stored under wiki-specific container names by default.
- * For legacy reasons, this has no effect for the FS backend class, and per-wiki
- * segregation must be done by setting the container paths appropriately.
+ * Global (qualified) backends are achieved by configuring the "wiki ID" to a constant.
+ * For legacy reasons, the FSFileBackend class allows manually setting the paths of
+ * containers to ones that do not respect the "wiki ID".
*
+ * In key/value stores, the container is the only hierarchy (the rest is emulated).
* FS-based backends are somewhat more restrictive due to the existence of real
* directory files; a regular file cannot have the same name as a directory. Other
* backends with virtual directories may not have this limitation. Callers should
@@ -75,9 +79,13 @@ abstract class FileBackend {
* $config includes:
* - name : The unique name of this backend.
* This should consist of alphanumberic, '-', and '_' characters.
- * This name should not be changed after use.
- * - wikiId : Prefix to container names that is unique to this wiki.
+ * This name should not be changed after use (e.g. with journaling).
+ * Note that the name is *not* used in actual container names.
+ * - wikiId : Prefix to container names that is unique to this backend.
+ * If not provided, this defaults to the current wiki ID.
* It should only consist of alphanumberic, '-', and '_' characters.
+ * This ID is what avoids collisions if multiple logical backends
+ * use the same storage system, so this should be set carefully.
* - lockManager : Registered name of a file lock manager to use.
* - fileJournal : File journal configuration; see FileJournal::factory().
* Journals simply log changes to files stored in the backend.
@@ -100,7 +108,7 @@ abstract class FileBackend {
: wfWikiID(); // e.g. "my_wiki-en_"
$this->lockManager = ( $config['lockManager'] instanceof LockManager )
? $config['lockManager']
- : LockManagerGroup::singleton()->get( $config['lockManager'] );
+ : LockManagerGroup::singleton( $this->wikiId )->get( $config['lockManager'] );
$this->fileJournal = isset( $config['fileJournal'] )
? ( ( $config['fileJournal'] instanceof FileJournal )
? $config['fileJournal']
@@ -129,7 +137,8 @@ abstract class FileBackend {
}
/**
- * Get the wiki identifier used for this backend (possibly empty)
+ * Get the wiki identifier used for this backend (possibly empty).
+ * Note that this might *not* be in the same format as wfWikiID().
*
* @return string
* @since 1.20
@@ -171,6 +180,7 @@ abstract class FileBackend {
* - copy
* - move
* - delete
+ * - describe (since 1.21)
* - null
*
* a) Create a new file in storage with the contents of a string
@@ -181,7 +191,8 @@ abstract class FileBackend {
* 'content' => <string of new file contents>,
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
- * 'disposition' => <Content-Disposition header value>
+ * 'disposition' => <Content-Disposition header value>,
+ * 'headers' => <HTTP header name/value map> # since 1.21
* );
* @endcode
*
@@ -193,7 +204,8 @@ abstract class FileBackend {
* 'dst' => <storage path>,
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
- * 'disposition' => <Content-Disposition header value>
+ * 'disposition' => <Content-Disposition header value>,
+ * 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
*
@@ -205,6 +217,7 @@ abstract class FileBackend {
* 'dst' => <storage path>,
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
+ * 'ignoreMissingSource' => <boolean>, # since 1.21
* 'disposition' => <Content-Disposition header value>
* )
* @endcode
@@ -217,6 +230,7 @@ abstract class FileBackend {
* 'dst' => <storage path>,
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
+ * 'ignoreMissingSource' => <boolean>, # since 1.21
* 'disposition' => <Content-Disposition header value>
* )
* @endcode
@@ -230,7 +244,17 @@ abstract class FileBackend {
* )
* @endcode
*
- * f) Do nothing (no-op)
+ * f) Update metadata for a file within storage
+ * @code
+ * array(
+ * 'op' => 'describe',
+ * 'src' => <storage path>,
+ * 'disposition' => <Content-Disposition header value>,
+ * 'headers' => <HTTP header name/value map>
+ * )
+ * @endcode
+ *
+ * g) Do nothing (no-op)
* @code
* array(
* 'op' => 'null',
@@ -244,28 +268,32 @@ abstract class FileBackend {
* - overwriteSame : An error will not be given if a file already
* exists at the destination that has the same
* contents as the new contents to be written there.
- * - disposition : When supplied, the backend will add a Content-Disposition
+ * - disposition : If supplied, the backend will return a Content-Disposition
* header when GETs/HEADs of the destination file are made.
- * Backends that don't support file metadata will ignore this.
- * See http://tools.ietf.org/html/rfc6266 (since 1.20).
+ * Backends that don't support metadata ignore this.
+ * See http://tools.ietf.org/html/rfc6266. (since 1.20)
+ * - headers : If supplied, the backend will return these headers when
+ * GETs/HEADs of the destination file are made. Header values
+ * should be smaller than 256 bytes, often options or numbers.
+ * Existing headers will remain, but these will replace any
+ * conflicting previous headers, and headers will be removed
+ * if they are set to an empty string.
+ * Backends that don't support metadata ignore this. (since 1.21)
*
* $opts is an associative of boolean flags, including:
* - force : Operation precondition errors no longer trigger an abort.
* Any remaining operations are still attempted. Unexpected
- * failures may still cause remaning operations to be aborted.
+ * failures may still cause remaining operations to be aborted.
* - nonLocking : No locks are acquired for the operations.
* This can increase performance for non-critical writes.
* This has no effect unless the 'force' flag is set.
- * - allowStale : Don't require the latest available data.
- * This can increase performance for non-critical writes.
- * This has no effect unless the 'force' flag is set.
* - nonJournaled : Don't log this operation batch in the file journal.
* This limits the ability of recovery scripts.
* - parallelize : Try to do operations in parallel when possible.
- * - bypassReadOnly : Allow writes in read-only mode (since 1.20).
+ * - bypassReadOnly : Allow writes in read-only mode. (since 1.20)
* - preserveCache : Don't clear the process cache before checking files.
* This should only be used if all entries in the process
- * cache were added after the files were already locked (since 1.20).
+ * cache were added after the files were already locked. (since 1.20)
*
* @remarks Remarks on locking:
* File system paths given to operations should refer to files that are
@@ -282,8 +310,8 @@ abstract class FileBackend {
* - a) unexpected operation errors occurred (network partitions, disk full...)
* - b) significant operation errors occurred and 'force' was not set
*
- * @param $ops Array List of operations to execute in order
- * @param $opts Array Batch operation options
+ * @param array $ops List of operations to execute in order
+ * @param array $opts Batch operation options
* @return Status
*/
final public function doOperations( array $ops, array $opts = array() ) {
@@ -292,18 +320,8 @@ abstract class FileBackend {
}
if ( empty( $opts['force'] ) ) { // sanity
unset( $opts['nonLocking'] );
- unset( $opts['allowStale'] );
- }
- $opts['concurrency'] = 1; // off
- if ( $this->parallelize === 'implicit' ) {
- if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
- $opts['concurrency'] = $this->concurrency;
- }
- } elseif ( $this->parallelize === 'explicit' ) {
- if ( !empty( $opts['parallelize'] ) ) {
- $opts['concurrency'] = $this->concurrency;
- }
}
+ $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doOperationsInternal( $ops, $opts );
}
@@ -319,8 +337,8 @@ abstract class FileBackend {
*
* @see FileBackend::doOperations()
*
- * @param $op Array Operation
- * @param $opts Array Operation options
+ * @param array $op Operation
+ * @param array $opts Operation options
* @return Status
*/
final public function doOperation( array $op, array $opts = array() ) {
@@ -333,8 +351,8 @@ abstract class FileBackend {
*
* @see FileBackend::doOperation()
*
- * @param $params Array Operation parameters
- * @param $opts Array Operation options
+ * @param array $params Operation parameters
+ * @param array $opts Operation options
* @return Status
*/
final public function create( array $params, array $opts = array() ) {
@@ -347,8 +365,8 @@ abstract class FileBackend {
*
* @see FileBackend::doOperation()
*
- * @param $params Array Operation parameters
- * @param $opts Array Operation options
+ * @param array $params Operation parameters
+ * @param array $opts Operation options
* @return Status
*/
final public function store( array $params, array $opts = array() ) {
@@ -361,8 +379,8 @@ abstract class FileBackend {
*
* @see FileBackend::doOperation()
*
- * @param $params Array Operation parameters
- * @param $opts Array Operation options
+ * @param array $params Operation parameters
+ * @param array $opts Operation options
* @return Status
*/
final public function copy( array $params, array $opts = array() ) {
@@ -375,8 +393,8 @@ abstract class FileBackend {
*
* @see FileBackend::doOperation()
*
- * @param $params Array Operation parameters
- * @param $opts Array Operation options
+ * @param array $params Operation parameters
+ * @param array $opts Operation options
* @return Status
*/
final public function move( array $params, array $opts = array() ) {
@@ -389,8 +407,8 @@ abstract class FileBackend {
*
* @see FileBackend::doOperation()
*
- * @param $params Array Operation parameters
- * @param $opts Array Operation options
+ * @param array $params Operation parameters
+ * @param array $opts Operation options
* @return Status
*/
final public function delete( array $params, array $opts = array() ) {
@@ -398,6 +416,21 @@ abstract class FileBackend {
}
/**
+ * Performs a single describe operation.
+ * This sets $params['op'] to 'describe' and passes it to doOperation().
+ *
+ * @see FileBackend::doOperation()
+ *
+ * @param array $params Operation parameters
+ * @param array $opts Operation options
+ * @return Status
+ * @since 1.21
+ */
+ final public function describe( array $params, array $opts = array() ) {
+ return $this->doOperation( array( 'op' => 'describe' ) + $params, $opts );
+ }
+
+ /**
* Perform a set of independent file operations on some files.
*
* This does no locking, nor journaling, and possibly no stat calls.
@@ -410,6 +443,7 @@ abstract class FileBackend {
* - copy
* - move
* - delete
+ * - describe (since 1.21)
* - null
*
* a) Create a new file in storage with the contents of a string
@@ -418,36 +452,44 @@ abstract class FileBackend {
* 'op' => 'create',
* 'dst' => <storage path>,
* 'content' => <string of new file contents>,
- * 'disposition' => <Content-Disposition header value>
+ * 'disposition' => <Content-Disposition header value>,
+ * 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
+ *
* b) Copy a file system file into storage
* @code
* array(
* 'op' => 'store',
* 'src' => <file system path>,
* 'dst' => <storage path>,
- * 'disposition' => <Content-Disposition header value>
+ * 'disposition' => <Content-Disposition header value>,
+ * 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
+ *
* c) Copy a file within storage
* @code
* array(
* 'op' => 'copy',
* 'src' => <storage path>,
* 'dst' => <storage path>,
+ * 'ignoreMissingSource' => <boolean>, # since 1.21
* 'disposition' => <Content-Disposition header value>
* )
* @endcode
+ *
* d) Move a file within storage
* @code
* array(
* 'op' => 'move',
* 'src' => <storage path>,
* 'dst' => <storage path>,
+ * 'ignoreMissingSource' => <boolean>, # since 1.21
* 'disposition' => <Content-Disposition header value>
* )
* @endcode
+ *
* e) Delete a file within storage
* @code
* array(
@@ -456,7 +498,18 @@ abstract class FileBackend {
* 'ignoreMissingSource' => <boolean>
* )
* @endcode
- * f) Do nothing (no-op)
+ *
+ * f) Update metadata for a file within storage
+ * @code
+ * array(
+ * 'op' => 'describe',
+ * 'src' => <storage path>,
+ * 'disposition' => <Content-Disposition header value>,
+ * 'headers' => <HTTP header name/value map>
+ * )
+ * @endcode
+ *
+ * g) Do nothing (no-op)
* @code
* array(
* 'op' => 'null',
@@ -470,6 +523,13 @@ abstract class FileBackend {
* header when GETs/HEADs of the destination file are made.
* Backends that don't support file metadata will ignore this.
* See http://tools.ietf.org/html/rfc6266 (since 1.20).
+ * - headers : If supplied with a header name/value map, the backend will
+ * reply with these headers when GETs/HEADs of the destination
+ * file are made. Header values should be smaller than 256 bytes.
+ * Existing headers will remain, but these will replace any
+ * conflicting previous headers, and headers will be removed
+ * if they are set to an empty string.
+ * Backends that don't support metadata ignore this. (since 1.21)
*
* $opts is an associative of boolean flags, including:
* - bypassReadOnly : Allow writes in read-only mode (since 1.20)
@@ -480,8 +540,8 @@ abstract class FileBackend {
* will reflect each operation attempted for the given files. The status will be
* considered "OK" as long as no fatal errors occurred.
*
- * @param $ops Array Set of operations to execute
- * @param $opts Array Batch operation options
+ * @param array $ops Set of operations to execute
+ * @param array $opts Batch operation options
* @return Status
* @since 1.20
*/
@@ -492,6 +552,7 @@ abstract class FileBackend {
foreach ( $ops as &$op ) {
$op['overwrite'] = true; // avoids RTTs in key/value stores
}
+ $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doQuickOperationsInternal( $ops );
}
@@ -507,7 +568,7 @@ abstract class FileBackend {
*
* @see FileBackend::doQuickOperations()
*
- * @param $op Array Operation
+ * @param array $op Operation
* @return Status
* @since 1.20
*/
@@ -521,7 +582,7 @@ abstract class FileBackend {
*
* @see FileBackend::doQuickOperation()
*
- * @param $params Array Operation parameters
+ * @param array $params Operation parameters
* @return Status
* @since 1.20
*/
@@ -535,7 +596,7 @@ abstract class FileBackend {
*
* @see FileBackend::doQuickOperation()
*
- * @param $params Array Operation parameters
+ * @param array $params Operation parameters
* @return Status
* @since 1.20
*/
@@ -549,7 +610,7 @@ abstract class FileBackend {
*
* @see FileBackend::doQuickOperation()
*
- * @param $params Array Operation parameters
+ * @param array $params Operation parameters
* @return Status
* @since 1.20
*/
@@ -563,7 +624,7 @@ abstract class FileBackend {
*
* @see FileBackend::doQuickOperation()
*
- * @param $params Array Operation parameters
+ * @param array $params Operation parameters
* @return Status
* @since 1.20
*/
@@ -577,7 +638,7 @@ abstract class FileBackend {
*
* @see FileBackend::doQuickOperation()
*
- * @param $params Array Operation parameters
+ * @param array $params Operation parameters
* @return Status
* @since 1.20
*/
@@ -586,15 +647,30 @@ abstract class FileBackend {
}
/**
+ * Performs a single quick describe operation.
+ * This sets $params['op'] to 'describe' and passes it to doQuickOperation().
+ *
+ * @see FileBackend::doQuickOperation()
+ *
+ * @param array $params Operation parameters
+ * @return Status
+ * @since 1.21
+ */
+ final public function quickDescribe( array $params ) {
+ return $this->doQuickOperation( array( 'op' => 'describe' ) + $params );
+ }
+
+ /**
* Concatenate a list of storage files into a single file system file.
* The target path should refer to a file that is already locked or
* otherwise safe from modification from other processes. Normally,
* the file will be a new temp file, which should be adequate.
*
- * @param $params Array Operation parameters
+ * @param array $params Operation parameters
* $params include:
- * - srcs : ordered source storage paths (e.g. chunk1, chunk2, ...)
- * - dst : file system path to 0-byte temp file
+ * - srcs : ordered source storage paths (e.g. chunk1, chunk2, ...)
+ * - dst : file system path to 0-byte temp file
+ * - parallelize : try to do operations in parallel when possible
* @return Status
*/
abstract public function concatenate( array $params );
@@ -608,7 +684,7 @@ abstract class FileBackend {
* except they are only applied *if* the directory/container had to be created.
* These flags should always be set for directories that have private files.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - dir : storage directory
* - noAccess : try to deny file access (since 1.20)
@@ -620,6 +696,7 @@ abstract class FileBackend {
if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
}
+ $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doPrepare( $params );
}
@@ -635,7 +712,7 @@ abstract class FileBackend {
* access to the storage user representing end-users in web requests.
* This is not guaranteed to actually do anything.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - dir : storage directory
* - noAccess : try to deny file access
@@ -647,6 +724,7 @@ abstract class FileBackend {
if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
}
+ $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doSecure( $params );
}
@@ -662,7 +740,7 @@ abstract class FileBackend {
* access to the storage user representing end-users in web requests.
* This essentially can undo the result of secure() calls.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - dir : storage directory
* - access : try to allow file access
@@ -675,6 +753,7 @@ abstract class FileBackend {
if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
}
+ $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doPublish( $params );
}
@@ -688,7 +767,7 @@ abstract class FileBackend {
* Backends using key/value stores may do nothing unless the directory
* is that of an empty container, in which case it will be deleted.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - dir : storage directory
* - recursive : recursively delete empty subdirectories first (since 1.20)
@@ -699,6 +778,7 @@ abstract class FileBackend {
if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) {
return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
}
+ $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doClean( $params );
}
@@ -708,10 +788,25 @@ abstract class FileBackend {
abstract protected function doClean( array $params );
/**
+ * Enter file operation scope.
+ * This just makes PHP ignore user aborts/disconnects until the return
+ * value leaves scope. This returns null and does nothing in CLI mode.
+ *
+ * @return ScopedCallback|null
+ */
+ final protected function getScopedPHPBehaviorForOps() {
+ if ( php_sapi_name() != 'cli' ) { // http://bugs.php.net/bug.php?id=47540
+ $old = ignore_user_abort( true ); // avoid half-finished operations
+ return new ScopedCallback( function() use ( $old ) { ignore_user_abort( $old ); } );
+ }
+ return null;
+ }
+
+ /**
* Check if a file exists at a storage path in the backend.
* This returns false if only a directory exists at the path.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
@@ -722,7 +817,7 @@ abstract class FileBackend {
/**
* Get the last-modified timestamp of the file at a storage path.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
@@ -734,18 +829,40 @@ abstract class FileBackend {
* Get the contents of a file at a storage path in the backend.
* This should be avoided for potentially large files.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
* @return string|bool Returns false on failure
*/
- abstract public function getFileContents( array $params );
+ final public function getFileContents( array $params ) {
+ $contents = $this->getFileContentsMulti(
+ array( 'srcs' => array( $params['src'] ) ) + $params );
+
+ return $contents[$params['src']];
+ }
+
+ /**
+ * Like getFileContents() except it takes an array of storage paths
+ * and returns a map of storage paths to strings (or null on failure).
+ * The map keys (paths) are in the same order as the provided list of paths.
+ *
+ * @see FileBackend::getFileContents()
+ *
+ * @param array $params
+ * $params include:
+ * - srcs : list of source storage paths
+ * - latest : use the latest available data
+ * - parallelize : try to do operations in parallel when possible
+ * @return Array Map of (path name => string or false on failure)
+ * @since 1.20
+ */
+ abstract public function getFileContentsMulti( array $params );
/**
* Get the size (bytes) of a file at a storage path in the backend.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
@@ -761,7 +878,7 @@ abstract class FileBackend {
* - size : the file size (bytes)
* Additional values may be included for internal use only.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
@@ -772,7 +889,7 @@ abstract class FileBackend {
/**
* Get a SHA-1 hash of the file at a storage path in the backend.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
@@ -782,13 +899,13 @@ abstract class FileBackend {
/**
* Get the properties of the file at a storage path in the backend.
- * Returns FSFile::placeholderProps() on failure.
+ * This gives the result of FSFile::getProps() on a local copy of the file.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
- * @return Array
+ * @return Array Returns FSFile::placeholderProps() on failure
*/
abstract public function getFileProps( array $params );
@@ -799,7 +916,7 @@ abstract class FileBackend {
* will be sent if streaming began, while none will be sent otherwise.
* Implementations should flush the output buffer before sending data.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - headers : list of additional HTTP headers to send on success
@@ -821,26 +938,89 @@ abstract class FileBackend {
* In that later case, there are copies of the file that must stay in sync.
* Additionally, further calls to this function may return the same file.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
* @return FSFile|null Returns null on failure
*/
- abstract public function getLocalReference( array $params );
+ final public function getLocalReference( array $params ) {
+ $fsFiles = $this->getLocalReferenceMulti(
+ array( 'srcs' => array( $params['src'] ) ) + $params );
+
+ return $fsFiles[$params['src']];
+ }
+
+ /**
+ * Like getLocalReference() except it takes an array of storage paths
+ * and returns a map of storage paths to FSFile objects (or null on failure).
+ * The map keys (paths) are in the same order as the provided list of paths.
+ *
+ * @see FileBackend::getLocalReference()
+ *
+ * @param array $params
+ * $params include:
+ * - srcs : list of source storage paths
+ * - latest : use the latest available data
+ * - parallelize : try to do operations in parallel when possible
+ * @return Array Map of (path name => FSFile or null on failure)
+ * @since 1.20
+ */
+ abstract public function getLocalReferenceMulti( array $params );
/**
* Get a local copy on disk of the file at a storage path in the backend.
* The temporary copy will have the same file extension as the source.
* Temporary files may be purged when the file object falls out of scope.
*
- * @param $params Array
+ * @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
* @return TempFSFile|null Returns null on failure
*/
- abstract public function getLocalCopy( array $params );
+ final public function getLocalCopy( array $params ) {
+ $tmpFiles = $this->getLocalCopyMulti(
+ array( 'srcs' => array( $params['src'] ) ) + $params );
+
+ return $tmpFiles[$params['src']];
+ }
+
+ /**
+ * Like getLocalCopy() except it takes an array of storage paths and
+ * returns a map of storage paths to TempFSFile objects (or null on failure).
+ * The map keys (paths) are in the same order as the provided list of paths.
+ *
+ * @see FileBackend::getLocalCopy()
+ *
+ * @param array $params
+ * $params include:
+ * - srcs : list of source storage paths
+ * - latest : use the latest available data
+ * - parallelize : try to do operations in parallel when possible
+ * @return Array Map of (path name => TempFSFile or null on failure)
+ * @since 1.20
+ */
+ abstract public function getLocalCopyMulti( array $params );
+
+ /**
+ * Return an HTTP URL to a given file that requires no authentication to use.
+ * The URL may be pre-authenticated (via some token in the URL) and temporary.
+ * This will return null if the backend cannot make an HTTP URL for the file.
+ *
+ * This is useful for key/value stores when using scripts that seek around
+ * large files and those scripts (and the backend) support HTTP Range headers.
+ * Otherwise, one would need to use getLocalReference(), which involves loading
+ * the entire file on to local disk.
+ *
+ * @param array $params
+ * $params include:
+ * - src : source storage path
+ * - ttl : lifetime (seconds) if pre-authenticated; default is 1 day
+ * @return string|null
+ * @since 1.21
+ */
+ abstract public function getFileHttpUrl( array $params );
/**
* Check if a directory exists at a given storage path.
@@ -930,7 +1110,7 @@ abstract class FileBackend {
* Preload persistent file stat and property cache into in-process cache.
* This should be used when stat calls will be made on a known list of a many files.
*
- * @param $paths Array Storage paths
+ * @param array $paths Storage paths
* @return void
*/
public function preloadCache( array $paths ) {}
@@ -939,7 +1119,7 @@ abstract class FileBackend {
* Invalidate any in-process file stat and property cache.
* If $paths is given, then only the cache for those files will be cleared.
*
- * @param $paths Array Storage paths (optional)
+ * @param array $paths Storage paths (optional)
* @return void
*/
public function clearCache( array $paths = null ) {}
@@ -950,7 +1130,7 @@ abstract class FileBackend {
*
* Callers should consider using getScopedFileLocks() instead.
*
- * @param $paths Array Storage paths
+ * @param array $paths Storage paths
* @param $type integer LockManager::LOCK_* constant
* @return Status
*/
@@ -961,7 +1141,7 @@ abstract class FileBackend {
/**
* Unlock the files at the given storage paths in the backend.
*
- * @param $paths Array Storage paths
+ * @param array $paths Storage paths
* @param $type integer LockManager::LOCK_* constant
* @return Status
*/
@@ -977,7 +1157,7 @@ abstract class FileBackend {
* Once the return value goes out scope, the locks will be released and
* the status updated. Unlock fatals will not change the status "OK" value.
*
- * @param $paths Array Storage paths
+ * @param array $paths Storage paths
* @param $type integer LockManager::LOCK_* constant
* @param $status Status Status to update on lock/unlock
* @return ScopedLock|null Returns null on failure
@@ -997,7 +1177,7 @@ abstract class FileBackend {
*
* @see FileBackend::doOperations()
*
- * @param $ops Array List of file operations to FileBackend::doOperations()
+ * @param array $ops List of file operations to FileBackend::doOperations()
* @param $status Status Status to update on lock/unlock
* @return Array List of ScopedFileLocks or null values
* @since 1.20
@@ -1016,6 +1196,17 @@ abstract class FileBackend {
}
/**
+ * Get the storage path for the given container for this backend
+ *
+ * @param string $container Container name
+ * @return string Storage path
+ * @since 1.21
+ */
+ final public function getContainerStoragePath( $container ) {
+ return $this->getRootStoragePath() . "/{$container}";
+ }
+
+ /**
* Get the file journal object for this backend
*
* @return FileJournal
@@ -1088,7 +1279,7 @@ abstract class FileBackend {
*/
final public static function parentStoragePath( $storagePath ) {
$storagePath = dirname( $storagePath );
- list( $b, $cont, $rel ) = self::splitStoragePath( $storagePath );
+ list( , , $rel ) = self::splitStoragePath( $storagePath );
return ( $rel === null ) ? null : $storagePath;
}
@@ -1117,8 +1308,9 @@ abstract class FileBackend {
/**
* Build a Content-Disposition header value per RFC 6266.
*
- * @param $type string One of (attachment, inline)
- * @param $filename string Suggested file name (should not contain slashes)
+ * @param string $type One of (attachment, inline)
+ * @param string $filename Suggested file name (should not contain slashes)
+ * @throws MWException
* @return string
* @since 1.20
*/
@@ -1145,7 +1337,7 @@ abstract class FileBackend {
*
* This uses the same traversal protection as Title::secureAndSplit().
*
- * @param $path string Storage path relative to a container
+ * @param string $path Storage path relative to a container
* @return string|null
*/
final protected static function normalizeContainerPath( $path ) {
diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php
index 8bbc96d0..d790a996 100644
--- a/includes/filebackend/FileBackendGroup.php
+++ b/includes/filebackend/FileBackendGroup.php
@@ -87,6 +87,9 @@ class FileBackendGroup {
$thumbDir = isset( $info['thumbDir'] )
? $info['thumbDir']
: "{$directory}/thumb";
+ $transcodedDir = isset( $info['transcodedDir'] )
+ ? $info['transcodedDir']
+ : "{$directory}/transcoded";
$fileMode = isset( $info['fileMode'] )
? $info['fileMode']
: 0644;
@@ -98,6 +101,7 @@ class FileBackendGroup {
'containerPaths' => array(
"{$repoName}-public" => "{$directory}",
"{$repoName}-thumb" => $thumbDir,
+ "{$repoName}-transcoded" => $transcodedDir,
"{$repoName}-deleted" => $deletedDir,
"{$repoName}-temp" => "{$directory}/temp"
),
@@ -122,7 +126,9 @@ class FileBackendGroup {
throw new MWException( "Cannot register a backend with no name." );
}
$name = $config['name'];
- if ( !isset( $config['class'] ) ) {
+ if ( isset( $this->backends[$name] ) ) {
+ throw new MWException( "Backend with name `{$name}` already registered." );
+ } elseif ( !isset( $config['class'] ) ) {
throw new MWException( "Cannot register backend `{$name}` with no class." );
}
$class = $config['class'];
@@ -178,7 +184,7 @@ class FileBackendGroup {
* @return FileBackend|null Backend or null on failure
*/
public function backendFromPath( $storagePath ) {
- list( $backend, $c, $p ) = FileBackend::splitStoragePath( $storagePath );
+ list( $backend, , ) = FileBackend::splitStoragePath( $storagePath );
if ( $backend !== null && isset( $this->backends[$backend] ) ) {
return $this->get( $backend );
}
diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php
index 4be03231..939315d1 100644
--- a/includes/filebackend/FileBackendMultiWrite.php
+++ b/includes/filebackend/FileBackendMultiWrite.php
@@ -62,7 +62,7 @@ class FileBackendMultiWrite extends FileBackend {
* Additional $config params include:
* - backends : Array of backend config and multi-backend settings.
* Each value is the config used in the constructor of a
- * FileBackendStore class, but with these additional settings:
+ * FileBackendStore class, but with these additional settings:
* - class : The name of the backend class
* - isMultiMaster : This must be set for one backend.
* - template: : If given a backend name, this will use
@@ -179,10 +179,11 @@ class FileBackendMultiWrite extends FileBackend {
// Actually attempt the operation batch on the master backend...
$masterStatus = $mbe->doOperations( $realOps, $opts );
$status->merge( $masterStatus );
- // Propagate the operations to the clone backends if there were no fatal errors.
- // If $ops only had one operation, this might avoid backend inconsistencies.
- // This also avoids inconsistency for expected errors (like "file already exists").
- if ( !count( $masterStatus->getErrorsArray() ) ) {
+ // Propagate the operations to the clone backends if there were no unexpected errors
+ // and if there were either no expected errors or if the 'force' option was used.
+ // However, if nothing succeeded at all, then don't replicate any of the operations.
+ // If $ops only had one operation, this might avoid backend sync inconsistencies.
+ if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
foreach ( $this->backends as $index => $backend ) {
if ( $index !== $this->masterIndex ) { // not done already
$realOps = $this->substOpBatchPaths( $ops, $backend );
@@ -203,7 +204,7 @@ class FileBackendMultiWrite extends FileBackend {
/**
* Check that a set of files are consistent across all internal backends
*
- * @param $paths Array List of storage paths
+ * @param array $paths List of storage paths
* @return Status
*/
public function consistencyCheck( array $paths ) {
@@ -269,7 +270,7 @@ class FileBackendMultiWrite extends FileBackend {
/**
* Check that a set of file paths are usable across all internal backends
*
- * @param $paths Array List of storage paths
+ * @param array $paths List of storage paths
* @return Status
*/
public function accessibilityCheck( array $paths ) {
@@ -294,7 +295,7 @@ class FileBackendMultiWrite extends FileBackend {
* Check that a set of files are consistent across all internal backends
* and re-synchronize those files againt the "multi master" if needed.
*
- * @param $paths Array List of storage paths
+ * @param array $paths List of storage paths
* @return Status
*/
public function resyncFiles( array $paths ) {
@@ -302,8 +303,8 @@ class FileBackendMultiWrite extends FileBackend {
$mBackend = $this->backends[$this->masterIndex];
foreach ( $paths as $path ) {
- $mPath = $this->substPaths( $path, $mBackend );
- $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) );
+ $mPath = $this->substPaths( $path, $mBackend );
+ $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) );
$mExist = $mBackend->fileExists( array( 'src' => $mPath ) );
// Check if the master backend is available...
if ( $mExist === null ) {
@@ -335,14 +336,20 @@ class FileBackendMultiWrite extends FileBackend {
/**
* Get a list of file storage paths to read or write for a list of operations
*
- * @param $ops Array Same format as doOperations()
+ * @param array $ops Same format as doOperations()
* @return Array List of storage paths to files (does not include directories)
*/
protected function fileStoragePathsForOps( array $ops ) {
$paths = array();
foreach ( $ops as $op ) {
if ( isset( $op['src'] ) ) {
- $paths[] = $op['src'];
+ // For things like copy/move/delete with "ignoreMissingSource" and there
+ // is no source file, nothing should happen and there should be no errors.
+ if ( empty( $op['ignoreMissingSource'] )
+ || $this->fileExists( array( 'src' => $op['src'] ) ) )
+ {
+ $paths[] = $op['src'];
+ }
}
if ( isset( $op['srcs'] ) ) {
$paths = array_merge( $paths, $op['srcs'] );
@@ -351,14 +358,14 @@ class FileBackendMultiWrite extends FileBackend {
$paths[] = $op['dst'];
}
}
- return array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) );
+ return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) );
}
/**
* Substitute the backend name in storage path parameters
* for a set of operations with that of a given internal backend.
*
- * @param $ops Array List of file operation arrays
+ * @param array $ops List of file operation arrays
* @param $backend FileBackendStore
* @return Array
*/
@@ -379,7 +386,7 @@ class FileBackendMultiWrite extends FileBackend {
/**
* Same as substOpBatchPaths() but for a single operation
*
- * @param $ops array File operation array
+ * @param array $ops File operation array
* @param $backend FileBackendStore
* @return Array
*/
@@ -391,7 +398,7 @@ class FileBackendMultiWrite extends FileBackend {
/**
* Substitute the backend of storage paths with an internal backend's name
*
- * @param $paths Array|string List of paths or single string path
+ * @param array|string $paths List of paths or single string path
* @param $backend FileBackendStore
* @return Array|string
*/
@@ -406,7 +413,7 @@ class FileBackendMultiWrite extends FileBackend {
/**
* Substitute the backend of internal storage paths with the proxy backend's name
*
- * @param $paths Array|string List of paths or single string path
+ * @param array|string $paths List of paths or single string path
* @return Array|string
*/
protected function unsubstPaths( $paths ) {
@@ -446,11 +453,11 @@ class FileBackendMultiWrite extends FileBackend {
}
/**
- * @param $path string Storage path
+ * @param string $path Storage path
* @return bool Path container should have dir changes pushed to all backends
*/
protected function replicateContainerDirChanges( $path ) {
- list( $b, $shortCont, $r ) = self::splitStoragePath( $path );
+ list( , $shortCont, ) = self::splitStoragePath( $path );
return !in_array( $shortCont, $this->noPushDirConts );
}
@@ -535,6 +542,7 @@ class FileBackendMultiWrite extends FileBackend {
/**
* @see FileBackend::fileExists()
* @param $params array
+ * @return bool|null
*/
public function fileExists( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
@@ -572,13 +580,19 @@ class FileBackendMultiWrite extends FileBackend {
}
/**
- * @see FileBackend::getFileContents()
+ * @see FileBackend::getFileContentsMulti()
* @param $params array
* @return bool|string
*/
- public function getFileContents( array $params ) {
+ public function getFileContentsMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
- return $this->backends[$this->masterIndex]->getFileContents( $realParams );
+ $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams );
+
+ $contents = array(); // (path => FSFile) mapping using the proxy backend's name
+ foreach ( $contentsM as $path => $data ) {
+ $contents[$this->unsubstPaths( $path )] = $data;
+ }
+ return $contents;
}
/**
@@ -612,23 +626,44 @@ class FileBackendMultiWrite extends FileBackend {
}
/**
- * @see FileBackend::getLocalReference()
+ * @see FileBackend::getLocalReferenceMulti()
* @param $params array
* @return FSFile|null
*/
- public function getLocalReference( array $params ) {
+ public function getLocalReferenceMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
- return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
+ $fsFilesM = $this->backends[$this->masterIndex]->getLocalReferenceMulti( $realParams );
+
+ $fsFiles = array(); // (path => FSFile) mapping using the proxy backend's name
+ foreach ( $fsFilesM as $path => $fsFile ) {
+ $fsFiles[$this->unsubstPaths( $path )] = $fsFile;
+ }
+ return $fsFiles;
}
/**
- * @see FileBackend::getLocalCopy()
+ * @see FileBackend::getLocalCopyMulti()
* @param $params array
* @return null|TempFSFile
*/
- public function getLocalCopy( array $params ) {
+ public function getLocalCopyMulti( array $params ) {
+ $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+ $tempFilesM = $this->backends[$this->masterIndex]->getLocalCopyMulti( $realParams );
+
+ $tempFiles = array(); // (path => TempFSFile) mapping using the proxy backend's name
+ foreach ( $tempFilesM as $path => $tempFile ) {
+ $tempFiles[$this->unsubstPaths( $path )] = $tempFile;
+ }
+ return $tempFiles;
+ }
+
+ /**
+ * @see FileBackend::getFileHttpUrl()
+ * @return string|null
+ */
+ public function getFileHttpUrl( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
- return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
+ return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams );
}
/**
diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php
index 083dfea9..3f1d1857 100644
--- a/includes/filebackend/FileBackendStore.php
+++ b/includes/filebackend/FileBackendStore.php
@@ -48,6 +48,8 @@ abstract class FileBackendStore extends FileBackend {
protected $maxFileSize = 4294967296; // integer bytes (4GiB)
+ const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
+
/**
* @see FileBackend::__construct()
*
@@ -55,8 +57,8 @@ abstract class FileBackendStore extends FileBackend {
*/
public function __construct( array $config ) {
parent::__construct( $config );
- $this->memCache = new EmptyBagOStuff(); // disabled by default
- $this->cheapCache = new ProcessCacheLRU( 300 );
+ $this->memCache = new EmptyBagOStuff(); // disabled by default
+ $this->cheapCache = new ProcessCacheLRU( 300 );
$this->expensiveCache = new ProcessCacheLRU( 5 );
}
@@ -72,8 +74,9 @@ abstract class FileBackendStore extends FileBackend {
}
/**
- * Check if a file can be created at a given storage path.
- * FS backends should check if the parent directory exists and the file is writable.
+ * Check if a file can be created or changed at a given storage path.
+ * FS backends should check if the parent directory exists, files can be
+ * written under it, and that any file already there is writable.
* Backends using key/value stores should check if the container exists.
*
* @param $storagePath string
@@ -83,18 +86,21 @@ abstract class FileBackendStore extends FileBackend {
/**
* Create a file in the backend with the given contents.
+ * This will overwrite any file that exists at the destination.
* Do not call this function from places outside FileBackend and FileOp.
*
* $params include:
- * - content : the raw file contents
- * - dst : destination storage path
- * - overwrite : overwrite any file that exists at the destination
- * - disposition : Content-Disposition header value for the destination
- * - async : Status will be returned immediately if supported.
- * If the status is OK, then its value field will be
- * set to a FileBackendStoreOpHandle object.
+ * - content : the raw file contents
+ * - dst : destination storage path
+ * - disposition : Content-Disposition header value for the destination
+ * - headers : HTTP header name/value map
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ * - dstExists : Whether a file exists at the destination (optimization).
+ * Callers can use "false" if no existing file is being changed.
*
- * @param $params Array
+ * @param array $params
* @return Status
*/
final public function createInternal( array $params ) {
@@ -106,7 +112,7 @@ abstract class FileBackendStore extends FileBackend {
} else {
$status = $this->doCreateInternal( $params );
$this->clearCache( array( $params['dst'] ) );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
@@ -117,23 +123,27 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::createInternal()
+ * @return Status
*/
abstract protected function doCreateInternal( array $params );
/**
* Store a file into the backend from a file on disk.
+ * This will overwrite any file that exists at the destination.
* Do not call this function from places outside FileBackend and FileOp.
*
* $params include:
- * - src : source path on disk
- * - dst : destination storage path
- * - overwrite : overwrite any file that exists at the destination
- * - disposition : Content-Disposition header value for the destination
- * - async : Status will be returned immediately if supported.
- * If the status is OK, then its value field will be
- * set to a FileBackendStoreOpHandle object.
+ * - src : source path on disk
+ * - dst : destination storage path
+ * - disposition : Content-Disposition header value for the destination
+ * - headers : HTTP header name/value map
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ * - dstExists : Whether a file exists at the destination (optimization).
+ * Callers can use "false" if no existing file is being changed.
*
- * @param $params Array
+ * @param array $params
* @return Status
*/
final public function storeInternal( array $params ) {
@@ -145,7 +155,7 @@ abstract class FileBackendStore extends FileBackend {
} else {
$status = $this->doStoreInternal( $params );
$this->clearCache( array( $params['dst'] ) );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
@@ -156,23 +166,27 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::storeInternal()
+ * @return Status
*/
abstract protected function doStoreInternal( array $params );
/**
* Copy a file from one storage path to another in the backend.
+ * This will overwrite any file that exists at the destination.
* Do not call this function from places outside FileBackend and FileOp.
*
* $params include:
- * - src : source storage path
- * - dst : destination storage path
- * - overwrite : overwrite any file that exists at the destination
- * - disposition : Content-Disposition header value for the destination
- * - async : Status will be returned immediately if supported.
- * If the status is OK, then its value field will be
- * set to a FileBackendStoreOpHandle object.
+ * - src : source storage path
+ * - dst : destination storage path
+ * - ignoreMissingSource : do nothing if the source file does not exist
+ * - disposition : Content-Disposition header value for the destination
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ * - dstExists : Whether a file exists at the destination (optimization).
+ * Callers can use "false" if no existing file is being changed.
*
- * @param $params Array
+ * @param array $params
* @return Status
*/
final public function copyInternal( array $params ) {
@@ -180,7 +194,7 @@ abstract class FileBackendStore extends FileBackend {
wfProfileIn( __METHOD__ . '-' . $this->name );
$status = $this->doCopyInternal( $params );
$this->clearCache( array( $params['dst'] ) );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
wfProfileOut( __METHOD__ . '-' . $this->name );
@@ -190,6 +204,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::copyInternal()
+ * @return Status
*/
abstract protected function doCopyInternal( array $params );
@@ -204,7 +219,7 @@ abstract class FileBackendStore extends FileBackend {
* If the status is OK, then its value field will be
* set to a FileBackendStoreOpHandle object.
*
- * @param $params Array
+ * @param array $params
* @return Status
*/
final public function deleteInternal( array $params ) {
@@ -220,23 +235,27 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::deleteInternal()
+ * @return Status
*/
abstract protected function doDeleteInternal( array $params );
/**
* Move a file from one storage path to another in the backend.
+ * This will overwrite any file that exists at the destination.
* Do not call this function from places outside FileBackend and FileOp.
*
* $params include:
- * - src : source storage path
- * - dst : destination storage path
- * - overwrite : overwrite any file that exists at the destination
- * - disposition : Content-Disposition header value for the destination
- * - async : Status will be returned immediately if supported.
- * If the status is OK, then its value field will be
- * set to a FileBackendStoreOpHandle object.
+ * - src : source storage path
+ * - dst : destination storage path
+ * - ignoreMissingSource : do nothing if the source file does not exist
+ * - disposition : Content-Disposition header value for the destination
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ * - dstExists : Whether a file exists at the destination (optimization).
+ * Callers can use "false" if no existing file is being changed.
*
- * @param $params Array
+ * @param array $params
* @return Status
*/
final public function moveInternal( array $params ) {
@@ -245,7 +264,7 @@ abstract class FileBackendStore extends FileBackend {
$status = $this->doMoveInternal( $params );
$this->clearCache( array( $params['src'], $params['dst'] ) );
$this->deleteFileCache( $params['src'] ); // persistent cache
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
wfProfileOut( __METHOD__ . '-' . $this->name );
@@ -270,10 +289,44 @@ abstract class FileBackendStore extends FileBackend {
}
/**
+ * Alter metadata for a file at the storage path.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * $params include:
+ * - src : source storage path
+ * - disposition : Content-Disposition header value for the destination
+ * - headers : HTTP header name/value map
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ *
+ * @param array $params
+ * @return Status
+ */
+ final public function describeInternal( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = $this->doDescribeInternal( $params );
+ $this->clearCache( array( $params['src'] ) );
+ $this->deleteFileCache( $params['src'] ); // persistent cache
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::describeInternal()
+ * @return Status
+ */
+ protected function doDescribeInternal( array $params ) {
+ return Status::newGood();
+ }
+
+ /**
* No-op file operation that does nothing.
* Do not call this function from places outside FileBackend and FileOp.
*
- * @param $params Array
+ * @param array $params
* @return Status
*/
final public function nullInternal( array $params ) {
@@ -314,31 +367,41 @@ abstract class FileBackendStore extends FileBackend {
protected function doConcatenate( array $params ) {
$status = Status::newGood();
$tmpPath = $params['dst']; // convenience
+ unset( $params['latest'] ); // sanity
// Check that the specified temp file is valid...
wfSuppressWarnings();
- $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
+ $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
wfRestoreWarnings();
if ( !$ok ) { // not present or not empty
$status->fatal( 'backend-fail-opentemp', $tmpPath );
return $status;
}
- // Build up the temp file using the source chunks (in order)...
+ // Get local FS versions of the chunks needed for the concatenation...
+ $fsFiles = $this->getLocalReferenceMulti( $params );
+ foreach ( $fsFiles as $path => &$fsFile ) {
+ if ( !$fsFile ) { // chunk failed to download?
+ $fsFile = $this->getLocalReference( array( 'src' => $path ) );
+ if ( !$fsFile ) { // retry failed?
+ $status->fatal( 'backend-fail-read', $path );
+ return $status;
+ }
+ }
+ }
+ unset( $fsFile ); // unset reference so we can reuse $fsFile
+
+ // Get a handle for the destination temp file
$tmpHandle = fopen( $tmpPath, 'ab' );
if ( $tmpHandle === false ) {
$status->fatal( 'backend-fail-opentemp', $tmpPath );
return $status;
}
- foreach ( $params['srcs'] as $virtualSource ) {
- // Get a local FS version of the chunk
- $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
- if ( !$tmpFile ) {
- $status->fatal( 'backend-fail-read', $virtualSource );
- return $status;
- }
+
+ // Build up the temp file using the source chunks (in order)...
+ foreach ( $fsFiles as $virtualSource => $fsFile ) {
// Get a handle to the local FS version
- $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
+ $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
if ( $sourceHandle === false ) {
fclose( $tmpHandle );
$status->fatal( 'backend-fail-read', $virtualSource );
@@ -384,7 +447,7 @@ abstract class FileBackendStore extends FileBackend {
$status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
} else { // directory is on several shards
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
- list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
$status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
}
@@ -424,7 +487,7 @@ abstract class FileBackendStore extends FileBackend {
$status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
} else { // directory is on several shards
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
- list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
$status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
}
@@ -464,7 +527,7 @@ abstract class FileBackendStore extends FileBackend {
$status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
} else { // directory is on several shards
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
- list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
$status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
}
@@ -500,6 +563,7 @@ abstract class FileBackendStore extends FileBackend {
$subDir = $params['dir'] . "/{$subDirRel}"; // full path
$status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
}
+ unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
}
}
@@ -525,7 +589,7 @@ abstract class FileBackendStore extends FileBackend {
$this->deleteContainerCache( $fullCont ); // purge cache
} else { // directory is on several shards
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
- list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
$status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
$this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
@@ -596,17 +660,25 @@ abstract class FileBackendStore extends FileBackend {
wfProfileIn( __METHOD__ );
wfProfileIn( __METHOD__ . '-' . $this->name );
$latest = !empty( $params['latest'] ); // use latest data?
- if ( !$this->cheapCache->has( $path, 'stat' ) ) {
+ if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
$this->primeFileCache( array( $path ) ); // check persistent cache
}
- if ( $this->cheapCache->has( $path, 'stat' ) ) {
+ if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
$stat = $this->cheapCache->get( $path, 'stat' );
// If we want the latest data, check that this cached
// value was in fact fetched with the latest available data.
- if ( !$latest || $stat['latest'] ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
- return $stat;
+ if ( is_array( $stat ) ) {
+ if ( !$latest || $stat['latest'] ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $stat;
+ }
+ } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) {
+ if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return false;
+ }
}
}
wfProfileIn( __METHOD__ . '-miss' );
@@ -614,7 +686,7 @@ abstract class FileBackendStore extends FileBackend {
$stat = $this->doGetFileStat( $params );
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
wfProfileOut( __METHOD__ . '-miss' );
- if ( is_array( $stat ) ) { // don't cache negatives
+ if ( is_array( $stat ) ) { // file exists
$stat['latest'] = $latest;
$this->cheapCache->set( $path, 'stat', $stat );
$this->setFileCache( $path, $stat ); // update persistent cache
@@ -622,8 +694,11 @@ abstract class FileBackendStore extends FileBackend {
$this->cheapCache->set( $path, 'sha1',
array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
}
- } else {
+ } elseif ( $stat === false ) { // file does not exist
+ $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
wfDebug( __METHOD__ . ": File $path does not exist.\n" );
+ } else { // an error occurred
+ wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
}
wfProfileOut( __METHOD__ . '-' . $this->name );
wfProfileOut( __METHOD__ );
@@ -636,24 +711,33 @@ abstract class FileBackendStore extends FileBackend {
abstract protected function doGetFileStat( array $params );
/**
- * @see FileBackend::getFileContents()
- * @return bool|string
+ * @see FileBackend::getFileContentsMulti()
+ * @return Array
*/
- public function getFileContents( array $params ) {
+ public function getFileContentsMulti( array $params ) {
wfProfileIn( __METHOD__ );
wfProfileIn( __METHOD__ . '-' . $this->name );
- $tmpFile = $this->getLocalReference( $params );
- if ( !$tmpFile ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
- return false;
- }
- wfSuppressWarnings();
- $data = file_get_contents( $tmpFile->getPath() );
- wfRestoreWarnings();
+
+ $params = $this->setConcurrencyFlags( $params );
+ $contents = $this->doGetFileContentsMulti( $params );
+
wfProfileOut( __METHOD__ . '-' . $this->name );
wfProfileOut( __METHOD__ );
- return $data;
+ return $contents;
+ }
+
+ /**
+ * @see FileBackendStore::getFileContentsMulti()
+ * @return Array
+ */
+ protected function doGetFileContentsMulti( array $params ) {
+ $contents = array();
+ foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
+ wfSuppressWarnings();
+ $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
+ wfRestoreWarnings();
+ }
+ return $contents;
}
/**
@@ -668,7 +752,7 @@ abstract class FileBackendStore extends FileBackend {
wfProfileIn( __METHOD__ );
wfProfileIn( __METHOD__ . '-' . $this->name );
$latest = !empty( $params['latest'] ); // use latest data?
- if ( $this->cheapCache->has( $path, 'sha1' ) ) {
+ if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
$stat = $this->cheapCache->get( $path, 'sha1' );
// If we want the latest data, check that this cached
// value was in fact fetched with the latest available data.
@@ -683,10 +767,7 @@ abstract class FileBackendStore extends FileBackend {
$hash = $this->doGetFileSha1Base36( $params );
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
wfProfileOut( __METHOD__ . '-miss' );
- if ( $hash ) { // don't cache negatives
- $this->cheapCache->set( $path, 'sha1',
- array( 'hash' => $hash, 'latest' => $latest ) );
- }
+ $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
wfProfileOut( __METHOD__ . '-' . $this->name );
wfProfileOut( __METHOD__ );
return $hash;
@@ -720,35 +801,82 @@ abstract class FileBackendStore extends FileBackend {
}
/**
- * @see FileBackend::getLocalReference()
- * @return TempFSFile|null
+ * @see FileBackend::getLocalReferenceMulti()
+ * @return Array
*/
- public function getLocalReference( array $params ) {
- $path = self::normalizeStoragePath( $params['src'] );
- if ( $path === null ) {
- return null; // invalid storage path
- }
+ final public function getLocalReferenceMulti( array $params ) {
wfProfileIn( __METHOD__ );
wfProfileIn( __METHOD__ . '-' . $this->name );
+
+ $params = $this->setConcurrencyFlags( $params );
+
+ $fsFiles = array(); // (path => FSFile)
$latest = !empty( $params['latest'] ); // use latest data?
- if ( $this->expensiveCache->has( $path, 'localRef' ) ) {
- $val = $this->expensiveCache->get( $path, 'localRef' );
- // If we want the latest data, check that this cached
- // value was in fact fetched with the latest available data.
- if ( !$latest || $val['latest'] ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
- return $val['object'];
+ // Reuse any files already in process cache...
+ foreach ( $params['srcs'] as $src ) {
+ $path = self::normalizeStoragePath( $src );
+ if ( $path === null ) {
+ $fsFiles[$src] = null; // invalid storage path
+ } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
+ $val = $this->expensiveCache->get( $path, 'localRef' );
+ // If we want the latest data, check that this cached
+ // value was in fact fetched with the latest available data.
+ if ( !$latest || $val['latest'] ) {
+ $fsFiles[$src] = $val['object'];
+ }
}
}
- $tmpFile = $this->getLocalCopy( $params );
- if ( $tmpFile ) { // don't cache negatives
- $this->expensiveCache->set( $path, 'localRef',
- array( 'object' => $tmpFile, 'latest' => $latest ) );
+ // Fetch local references of any remaning files...
+ $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
+ foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
+ $fsFiles[$path] = $fsFile;
+ if ( $fsFile ) { // update the process cache...
+ $this->expensiveCache->set( $path, 'localRef',
+ array( 'object' => $fsFile, 'latest' => $latest ) );
+ }
}
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $fsFiles;
+ }
+
+ /**
+ * @see FileBackendStore::getLocalReferenceMulti()
+ * @return Array
+ */
+ protected function doGetLocalReferenceMulti( array $params ) {
+ return $this->doGetLocalCopyMulti( $params );
+ }
+
+ /**
+ * @see FileBackend::getLocalCopyMulti()
+ * @return Array
+ */
+ final public function getLocalCopyMulti( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+
+ $params = $this->setConcurrencyFlags( $params );
+ $tmpFiles = $this->doGetLocalCopyMulti( $params );
+
wfProfileOut( __METHOD__ . '-' . $this->name );
wfProfileOut( __METHOD__ );
- return $tmpFile;
+ return $tmpFiles;
+ }
+
+ /**
+ * @see FileBackendStore::getLocalCopyMulti()
+ * @return Array
+ */
+ abstract protected function doGetLocalCopyMulti( array $params );
+
+ /**
+ * @see FileBackend::getFileHttpUrl()
+ * @return string|null
+ */
+ public function getFileHttpUrl( array $params ) {
+ return null; // not supported
}
/**
@@ -776,6 +904,14 @@ abstract class FileBackendStore extends FileBackend {
$status = $this->doStreamFile( $params );
wfProfileOut( __METHOD__ . '-send-' . $this->name );
wfProfileOut( __METHOD__ . '-send' );
+ if ( !$status->isOK() ) {
+ // Per bug 41113, nasty things can happen if bad cache entries get
+ // stuck in cache. It's also possible that this error can come up
+ // with simple race conditions. Clear out the stat cache to be safe.
+ $this->clearCache( array( $params['src'] ) );
+ $this->deleteFileCache( $params['src'] );
+ trigger_error( "Bad stat cache or race condition for file {$params['src']}." );
+ }
} else {
$status->fatal( 'backend-fail-stream', $params['src'] );
}
@@ -815,7 +951,7 @@ abstract class FileBackendStore extends FileBackend {
return $this->doDirectoryExists( $fullCont, $dir, $params );
} else { // directory is on several shards
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
- list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
$res = false; // response
foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
$exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
@@ -833,9 +969,9 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::directoryExists()
*
- * @param $container string Resolved container name
- * @param $dir string Resolved path relative to container
- * @param $params Array
+ * @param string $container Resolved container name
+ * @param string $dir Resolved path relative to container
+ * @param array $params
* @return bool|null
*/
abstract protected function doDirectoryExists( $container, $dir, array $params );
@@ -855,7 +991,7 @@ abstract class FileBackendStore extends FileBackend {
} else {
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
// File listing spans multiple containers/shards
- list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
return new FileBackendStoreShardDirIterator( $this,
$fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
}
@@ -866,9 +1002,9 @@ abstract class FileBackendStore extends FileBackend {
*
* @see FileBackendStore::getDirectoryList()
*
- * @param $container string Resolved container name
- * @param $dir string Resolved path relative to container
- * @param $params Array
+ * @param string $container Resolved container name
+ * @param string $dir Resolved path relative to container
+ * @param array $params
* @return Traversable|Array|null Returns null on failure
*/
abstract public function getDirectoryListInternal( $container, $dir, array $params );
@@ -888,7 +1024,7 @@ abstract class FileBackendStore extends FileBackend {
} else {
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
// File listing spans multiple containers/shards
- list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
return new FileBackendStoreShardFileIterator( $this,
$fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
}
@@ -899,9 +1035,9 @@ abstract class FileBackendStore extends FileBackend {
*
* @see FileBackendStore::getFileList()
*
- * @param $container string Resolved container name
- * @param $dir string Resolved path relative to container
- * @param $params Array
+ * @param string $container Resolved container name
+ * @param string $dir Resolved path relative to container
+ * @param array $params
* @return Traversable|Array|null Returns null on failure
*/
abstract public function getFileListInternal( $container, $dir, array $params );
@@ -913,18 +1049,19 @@ abstract class FileBackendStore extends FileBackend {
* The result must have the same number of items as the input.
* An exception is thrown if an unsupported operation is requested.
*
- * @param $ops Array Same format as doOperations()
+ * @param array $ops Same format as doOperations()
* @return Array List of FileOp objects
* @throws MWException
*/
final public function getOperationsInternal( array $ops ) {
$supportedOps = array(
- 'store' => 'StoreFileOp',
- 'copy' => 'CopyFileOp',
- 'move' => 'MoveFileOp',
- 'delete' => 'DeleteFileOp',
- 'create' => 'CreateFileOp',
- 'null' => 'NullFileOp'
+ 'store' => 'StoreFileOp',
+ 'copy' => 'CopyFileOp',
+ 'move' => 'MoveFileOp',
+ 'delete' => 'DeleteFileOp',
+ 'create' => 'CreateFileOp',
+ 'describe' => 'DescribeFileOp',
+ 'null' => 'NullFileOp'
);
$performOps = array(); // array of FileOp objects
@@ -949,8 +1086,9 @@ abstract class FileBackendStore extends FileBackend {
* Get a list of storage paths to lock for a list of operations
* Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
* each corresponding to a list of storage paths to be locked.
+ * All returned paths are normalized.
*
- * @param $performOps Array List of FileOp objects
+ * @param array $performOps List of FileOp objects
* @return Array ('sh' => list of paths, 'ex' => list of paths)
*/
final public function getPathsToLockForOpsInternal( array $performOps ) {
@@ -989,6 +1127,9 @@ abstract class FileBackendStore extends FileBackend {
wfProfileIn( __METHOD__ . '-' . $this->name );
$status = Status::newGood();
+ // Fix up custom header name/value pairs...
+ $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
+
// Build up a list of FileOps...
$performOps = $this->getOperationsInternal( $ops );
@@ -1016,6 +1157,7 @@ abstract class FileBackendStore extends FileBackend {
$this->primeContainerCache( $performOps );
// Actually attempt the operation batch...
+ $opts = $this->setConcurrencyFlags( $opts );
$subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
// Merge errors into status fields
@@ -1037,6 +1179,12 @@ abstract class FileBackendStore extends FileBackend {
wfProfileIn( __METHOD__ . '-' . $this->name );
$status = Status::newGood();
+ // Fix up custom header name/value pairs...
+ $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
+
+ // Clear any file cache entries
+ $this->clearCache();
+
$supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
$async = ( $this->parallelize === 'implicit' );
$maxConcurrency = $this->concurrency; // throttle
@@ -1092,7 +1240,7 @@ abstract class FileBackendStore extends FileBackend {
* The resulting Status object fields will correspond
* to the order in which the handles where given.
*
- * @param $handles Array List of FileBackendStoreOpHandle objects
+ * @param array $handles List of FileBackendStoreOpHandle objects
* @return Array Map of Status objects
* @throws MWException
*/
@@ -1117,6 +1265,8 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::executeOpHandlesInternal()
+ * @param array $fileOpHandles
+ * @throws MWException
* @return Array List of corresponding Status objects
*/
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
@@ -1127,12 +1277,32 @@ abstract class FileBackendStore extends FileBackend {
}
/**
+ * Strip long HTTP headers from a file operation
+ *
+ * @param array $op Same format as doOperation()
+ * @return Array
+ */
+ protected function stripInvalidHeadersFromOp( array $op ) {
+ if ( isset( $op['headers'] ) ) {
+ foreach ( $op['headers'] as $name => $value ) {
+ if ( strlen( $name ) > 255 || strlen( $value ) > 255 ) {
+ trigger_error( "Header '$name: $value' is too long." );
+ unset( $op['headers'][$name] );
+ } elseif ( !strlen( $value ) ) {
+ $op['headers'][$name] = ''; // null/false => ""
+ }
+ }
+ }
+ return $op;
+ }
+
+ /**
* @see FileBackend::preloadCache()
*/
final public function preloadCache( array $paths ) {
$fullConts = array(); // full container names
foreach ( $paths as $path ) {
- list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
+ list( $fullCont, , ) = $this->resolveStoragePath( $path );
$fullConts[] = $fullCont;
}
// Load from the persistent file and container caches
@@ -1165,7 +1335,7 @@ abstract class FileBackendStore extends FileBackend {
*
* @see FileBackend::clearCache()
*
- * @param $paths Array Storage paths (optional)
+ * @param array $paths Storage paths (optional)
* @return void
*/
protected function doClearCache( array $paths = null ) {}
@@ -1254,8 +1424,8 @@ abstract class FileBackendStore extends FileBackend {
* Get the container name shard suffix for a given path.
* Any empty suffix means the container is not sharded.
*
- * @param $container string Container name
- * @param $relPath string Storage path relative to the container
+ * @param string $container Container name
+ * @param string $relPath Storage path relative to the container
* @return string|null Returns null if shard could not be determined
*/
final protected function getContainerShard( $container, $relPath ) {
@@ -1291,11 +1461,11 @@ abstract class FileBackendStore extends FileBackend {
* Container dirs like "a", where the container shards on "x/xy",
* can reside on several shards. Such paths are tricky to handle.
*
- * @param $storagePath string Storage path
+ * @param string $storagePath Storage path
* @return bool
*/
final public function isSingleShardPathInternal( $storagePath ) {
- list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
+ list( , , $shard ) = $this->resolveStoragePath( $storagePath );
return ( $shard !== null );
}
@@ -1371,8 +1541,8 @@ abstract class FileBackendStore extends FileBackend {
* getting absolute paths (e.g. FS based backends). Note that the relative path
* may be the empty string (e.g. the path is simply to the container).
*
- * @param $container string Container name
- * @param $relStoragePath string Storage path relative to the container
+ * @param string $container Container name
+ * @param string $relStoragePath Storage path relative to the container
* @return string|null Path or null if not valid
*/
protected function resolveContainerPath( $container, $relStoragePath ) {
@@ -1382,7 +1552,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get the cache key for a container
*
- * @param $container string Resolved container name
+ * @param string $container Resolved container name
* @return string
*/
private function containerCacheKey( $container ) {
@@ -1392,7 +1562,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Set the cached info for a container
*
- * @param $container string Resolved container name
+ * @param string $container Resolved container name
* @param $val mixed Information to cache
*/
final protected function setContainerCache( $container, $val ) {
@@ -1403,7 +1573,7 @@ abstract class FileBackendStore extends FileBackend {
* Delete the cached info for a container.
* The cache key is salted for a while to prevent race conditions.
*
- * @param $container string Resolved container name
+ * @param string $container Resolved container name
*/
final protected function deleteContainerCache( $container ) {
if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
@@ -1414,6 +1584,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Do a batch lookup from cache for container stats for all containers
* used in a list of container names, storage paths, or FileOp objects.
+ * This loads the persistent cache values into the process cache.
*
* @param $items Array
* @return void
@@ -1437,7 +1608,7 @@ abstract class FileBackendStore extends FileBackend {
}
// Get all the corresponding cache keys for paths...
foreach ( $paths as $path ) {
- list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
+ list( $fullCont, , ) = $this->resolveStoragePath( $path );
if ( $fullCont !== null ) { // valid path for this backend
$contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
}
@@ -1462,7 +1633,7 @@ abstract class FileBackendStore extends FileBackend {
* resolved container names and their corresponding cached info.
* Only containers that actually exist should appear in the map.
*
- * @param $containerInfo Array Map of resolved container names to cached info
+ * @param array $containerInfo Map of resolved container names to cached info
* @return void
*/
protected function doPrimeContainerCache( array $containerInfo ) {}
@@ -1470,7 +1641,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get the cache key for a file path
*
- * @param $path string Storage path
+ * @param string $path Normalized storage path
* @return string
*/
private function fileCacheKey( $path ) {
@@ -1482,20 +1653,30 @@ abstract class FileBackendStore extends FileBackend {
* Negatives (404s) are not cached. By not caching negatives, we can skip cache
* salting for the case when a file is created at a path were there was none before.
*
- * @param $path string Storage path
+ * @param string $path Storage path
* @param $val mixed Information to cache
*/
final protected function setFileCache( $path, $val ) {
+ $path = FileBackend::normalizeStoragePath( $path );
+ if ( $path === null ) {
+ return; // invalid storage path
+ }
$this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
}
/**
* Delete the cached stat info for a file path.
* The cache key is salted for a while to prevent race conditions.
+ * Since negatives (404s) are not cached, this does not need to be called when
+ * a file is created at a path were there was none before.
*
- * @param $path string Storage path
+ * @param string $path Storage path
*/
final protected function deleteFileCache( $path ) {
+ $path = FileBackend::normalizeStoragePath( $path );
+ if ( $path === null ) {
+ return; // invalid storage path
+ }
if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
trigger_error( "Unable to delete stat cache for file $path." );
}
@@ -1504,8 +1685,9 @@ abstract class FileBackendStore extends FileBackend {
/**
* Do a batch lookup from cache for file stats for all paths
* used in a list of storage paths or FileOp objects.
+ * This loads the persistent cache values into the process cache.
*
- * @param $items Array List of storage paths or FileOps
+ * @param array $items List of storage paths or FileOps
* @return void
*/
final protected function primeFileCache( array $items ) {
@@ -1520,12 +1702,14 @@ abstract class FileBackendStore extends FileBackend {
$paths = array_merge( $paths, $item->storagePathsRead() );
$paths = array_merge( $paths, $item->storagePathsChanged() );
} elseif ( self::isStoragePath( $item ) ) {
- $paths[] = $item;
+ $paths[] = FileBackend::normalizeStoragePath( $item );
}
}
+ // Get rid of any paths that failed normalization...
+ $paths = array_filter( $paths, 'strlen' ); // remove nulls
// Get all the corresponding cache keys for paths...
foreach ( $paths as $path ) {
- list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
+ list( , $rel, ) = $this->resolveStoragePath( $path );
if ( $rel !== null ) { // valid path for this backend
$pathNames[$this->fileCacheKey( $path )] = $path;
}
@@ -1546,6 +1730,26 @@ abstract class FileBackendStore extends FileBackend {
wfProfileOut( __METHOD__ . '-' . $this->name );
wfProfileOut( __METHOD__ );
}
+
+ /**
+ * Set the 'concurrency' option from a list of operation options
+ *
+ * @param array $opts Map of operation options
+ * @return Array
+ */
+ final protected function setConcurrencyFlags( array $opts ) {
+ $opts['concurrency'] = 1; // off
+ if ( $this->parallelize === 'implicit' ) {
+ if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
+ $opts['concurrency'] = $this->concurrency;
+ }
+ } elseif ( $this->parallelize === 'explicit' ) {
+ if ( !empty( $opts['parallelize'] ) ) {
+ $opts['concurrency'] = $this->concurrency;
+ }
+ }
+ return $opts;
+ }
}
/**
@@ -1602,10 +1806,10 @@ abstract class FileBackendStoreShardListIterator implements Iterator {
/**
* @param $backend FileBackendStore
- * @param $container string Full storage container name
- * @param $dir string Storage directory relative to container
- * @param $suffixes Array List of container shard suffixes
- * @param $params Array
+ * @param string $container Full storage container name
+ * @param string $dir Storage directory relative to container
+ * @param array $suffixes List of container shard suffixes
+ * @param array $params
*/
public function __construct(
FileBackendStore $backend, $container, $dir, array $suffixes, array $params
@@ -1731,9 +1935,9 @@ abstract class FileBackendStoreShardListIterator implements Iterator {
/**
* Get the list for a given container shard
*
- * @param $container string Resolved container name
- * @param $dir string Resolved path relative to container
- * @param $params Array
+ * @param string $container Resolved container name
+ * @param string $dir Resolved path relative to container
+ * @param array $params
* @return Traversable|Array|null
*/
abstract protected function listFromShard( $container, $dir, array $params );
diff --git a/includes/filebackend/FileOp.php b/includes/filebackend/FileOp.php
index 7c43c489..bb0ab578 100644
--- a/includes/filebackend/FileOp.php
+++ b/includes/filebackend/FileOp.php
@@ -42,11 +42,12 @@ abstract class FileOp {
protected $state = self::STATE_NEW; // integer
protected $failed = false; // boolean
protected $async = false; // boolean
- protected $useLatest = true; // boolean
protected $batchId; // string
+ protected $doOperation = true; // boolean; operation is not a no-op
protected $sourceSha1; // string
protected $destSameAsSource; // boolean
+ protected $destExists; // boolean
/* Object life-cycle */
const STATE_NEW = 1;
@@ -65,37 +66,61 @@ abstract class FileOp {
list( $required, $optional ) = $this->allowedParams();
foreach ( $required as $name ) {
if ( isset( $params[$name] ) ) {
- $this->params[$name] = $params[$name];
+ $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] );
} else {
throw new MWException( "File operation missing parameter '$name'." );
}
}
foreach ( $optional as $name ) {
if ( isset( $params[$name] ) ) {
- $this->params[$name] = $params[$name];
+ $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] );
}
}
$this->params = $params;
}
/**
- * Set the batch UUID this operation belongs to
+ * Normalize $item or anything in $item that is a valid storage path
*
- * @param $batchId string
- * @return void
+ * @param $item string|array
+ * @return string|Array
+ */
+ protected function normalizeAnyStoragePaths( $item ) {
+ if ( is_array( $item ) ) {
+ $res = array();
+ foreach ( $item as $k => $v ) {
+ $k = self::normalizeIfValidStoragePath( $k );
+ $v = self::normalizeIfValidStoragePath( $v );
+ $res[$k] = $v;
+ }
+ return $res;
+ } else {
+ return self::normalizeIfValidStoragePath( $item );
+ }
+ }
+
+ /**
+ * Normalize a string if it is a valid storage path
+ *
+ * @param $path string
+ * @return string
*/
- final public function setBatchId( $batchId ) {
- $this->batchId = $batchId;
+ protected static function normalizeIfValidStoragePath( $path ) {
+ if ( FileBackend::isStoragePath( $path ) ) {
+ $res = FileBackend::normalizeStoragePath( $path );
+ return ( $res !== null ) ? $res : $path;
+ }
+ return $path;
}
/**
- * Whether to allow stale data for file reads and stat checks
+ * Set the batch UUID this operation belongs to
*
- * @param $allowStale bool
+ * @param $batchId string
* @return void
*/
- final public function allowStaleReads( $allowStale ) {
- $this->useLatest = !$allowStale;
+ final public function setBatchId( $batchId ) {
+ $this->batchId = $batchId;
}
/**
@@ -138,11 +163,11 @@ abstract class FileOp {
/**
* Update a dependency tracking array to account for this operation
*
- * @param $deps Array Prior path reads/writes; format of FileOp::newPredicates()
+ * @param array $deps Prior path reads/writes; format of FileOp::newPredicates()
* @return Array
*/
final public function applyDependencies( array $deps ) {
- $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 );
+ $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 );
$deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 );
return $deps;
}
@@ -150,7 +175,7 @@ abstract class FileOp {
/**
* Check if this operation changes files listed in $paths
*
- * @param $paths Array Prior path reads/writes; format of FileOp::newPredicates()
+ * @param array $paths Prior path reads/writes; format of FileOp::newPredicates()
* @return boolean
*/
final public function dependsOn( array $deps ) {
@@ -170,16 +195,19 @@ abstract class FileOp {
/**
* Get the file journal entries for this file operation
*
- * @param $oPredicates Array Pre-op info about files (format of FileOp::newPredicates)
- * @param $nPredicates Array Post-op info about files (format of FileOp::newPredicates)
+ * @param array $oPredicates Pre-op info about files (format of FileOp::newPredicates)
+ * @param array $nPredicates Post-op info about files (format of FileOp::newPredicates)
* @return Array
*/
final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
+ if ( !$this->doOperation ) {
+ return array(); // this is a no-op
+ }
$nullEntries = array();
$updateEntries = array();
$deleteEntries = array();
$pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
- foreach ( $pathsUsed as $path ) {
+ foreach ( array_unique( $pathsUsed ) as $path ) {
$nullEntries[] = array( // assertion for recovery
'op' => 'null',
'path' => $path,
@@ -205,7 +233,9 @@ abstract class FileOp {
}
/**
- * Check preconditions of the operation without writing anything
+ * Check preconditions of the operation without writing anything.
+ * This must update $predicates for each path that the op can change
+ * except when a failing status object is returned.
*
* @param $predicates Array
* @return Status
@@ -241,10 +271,14 @@ abstract class FileOp {
return Status::newFatal( 'fileop-fail-attempt-precheck' );
}
$this->state = self::STATE_ATTEMPTED;
- $status = $this->doAttempt();
- if ( !$status->isOK() ) {
- $this->failed = true;
- $this->logFailure( 'attempt' );
+ if ( $this->doOperation ) {
+ $status = $this->doAttempt();
+ if ( !$status->isOK() ) {
+ $this->failed = true;
+ $this->logFailure( 'attempt' );
+ }
+ } else { // no-op
+ $status = Status::newGood();
}
return $status;
}
@@ -292,15 +326,7 @@ abstract class FileOp {
*
* @return Array
*/
- final public function storagePathsRead() {
- return array_map( 'FileBackend::normalizeStoragePath', $this->doStoragePathsRead() );
- }
-
- /**
- * @see FileOp::storagePathsRead()
- * @return Array
- */
- protected function doStoragePathsRead() {
+ public function storagePathsRead() {
return array();
}
@@ -309,21 +335,13 @@ abstract class FileOp {
*
* @return Array
*/
- final public function storagePathsChanged() {
- return array_map( 'FileBackend::normalizeStoragePath', $this->doStoragePathsChanged() );
- }
-
- /**
- * @see FileOp::storagePathsChanged()
- * @return Array
- */
- protected function doStoragePathsChanged() {
+ public function storagePathsChanged() {
return array();
}
/**
* Check for errors with regards to the destination file already existing.
- * This also updates the destSameAsSource and sourceSha1 member variables.
+ * Also set the destExists, destSameAsSource and sourceSha1 member variables.
* A bad status will be returned if there is no chance it can be overwritten.
*
* @param $predicates Array
@@ -337,7 +355,8 @@ abstract class FileOp {
$this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
}
$this->destSameAsSource = false;
- if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
+ $this->destExists = $this->fileExists( $this->params['dst'], $predicates );
+ if ( $this->destExists ) {
if ( $this->getParam( 'overwrite' ) ) {
return $status; // OK
} elseif ( $this->getParam( 'overwriteSame' ) ) {
@@ -373,7 +392,7 @@ abstract class FileOp {
/**
* Check if a file will exist in storage when this operation is attempted
*
- * @param $source string Storage path
+ * @param string $source Storage path
* @param $predicates Array
* @return bool
*/
@@ -381,7 +400,7 @@ abstract class FileOp {
if ( isset( $predicates['exists'][$source] ) ) {
return $predicates['exists'][$source]; // previous op assures this
} else {
- $params = array( 'src' => $source, 'latest' => $this->useLatest );
+ $params = array( 'src' => $source, 'latest' => true );
return $this->backend->fileExists( $params );
}
}
@@ -389,15 +408,17 @@ abstract class FileOp {
/**
* Get the SHA-1 of a file in storage when this operation is attempted
*
- * @param $source string Storage path
+ * @param string $source Storage path
* @param $predicates Array
* @return string|bool False on failure
*/
final protected function fileSha1( $source, array $predicates ) {
if ( isset( $predicates['sha1'][$source] ) ) {
return $predicates['sha1'][$source]; // previous op assures this
+ } elseif ( isset( $predicates['exists'][$source] ) && !$predicates['exists'][$source] ) {
+ return false; // previous op assures this
} else {
- $params = array( 'src' => $source, 'latest' => $this->useLatest );
+ $params = array( 'src' => $source, 'latest' => true );
return $this->backend->getFileSha1Base36( $params );
}
}
@@ -430,42 +451,32 @@ abstract class FileOp {
}
/**
- * Store a file into the backend from a file on the file system.
+ * Create a file in the backend with the given content.
* Parameters for this operation are outlined in FileBackend::doOperations().
*/
-class StoreFileOp extends FileOp {
- /**
- * @return array
- */
+class CreateFileOp extends FileOp {
protected function allowedParams() {
- return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'disposition' ) );
+ return array( array( 'content', 'dst' ),
+ array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) );
}
- /**
- * @param $predicates array
- * @return Status
- */
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
- // Check if the source file exists on the file system
- if ( !is_file( $this->params['src'] ) ) {
- $status->fatal( 'backend-fail-notexists', $this->params['src'] );
- return $status;
- // Check if the source file is too big
- } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
+ // Check if the source data is too big
+ if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
$status->fatal( 'backend-fail-maxsize',
$this->params['dst'], $this->backend->maxFileSizeInternal() );
- $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
+ $status->fatal( 'backend-fail-create', $this->params['dst'] );
return $status;
- // Check if a file can be placed at the destination
+ // Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
- $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
+ $status->fatal( 'backend-fail-create', $this->params['dst'] );
return $status;
}
// Check if destination file exists
$status->merge( $this->precheckDestExistence( $predicates ) );
+ $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache()
if ( $status->isOK() ) {
// Update file existence predicates
$predicates['exists'][$this->params['dst']] = true;
@@ -478,57 +489,66 @@ class StoreFileOp extends FileOp {
* @return Status
*/
protected function doAttempt() {
- // Store the file at the destination
if ( !$this->destSameAsSource ) {
- return $this->backend->storeInternal( $this->setFlags( $this->params ) );
+ // Create the file at the destination
+ return $this->backend->createInternal( $this->setFlags( $this->params ) );
}
return Status::newGood();
}
/**
- * @return bool|string
+ * @return bool|String
*/
protected function getSourceSha1Base36() {
- wfSuppressWarnings();
- $hash = sha1_file( $this->params['src'] );
- wfRestoreWarnings();
- if ( $hash !== false ) {
- $hash = wfBaseConvert( $hash, 16, 36, 31 );
- }
- return $hash;
+ return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
}
- protected function doStoragePathsChanged() {
+ /**
+ * @return array
+ */
+ public function storagePathsChanged() {
return array( $this->params['dst'] );
}
}
/**
- * Create a file in the backend with the given content.
+ * Store a file into the backend from a file on the file system.
* Parameters for this operation are outlined in FileBackend::doOperations().
*/
-class CreateFileOp extends FileOp {
+class StoreFileOp extends FileOp {
+ /**
+ * @return array
+ */
protected function allowedParams() {
- return array( array( 'content', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'disposition' ) );
+ return array( array( 'src', 'dst' ),
+ array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) );
}
+ /**
+ * @param $predicates array
+ * @return Status
+ */
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
- // Check if the source data is too big
- if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
+ // Check if the source file exists on the file system
+ if ( !is_file( $this->params['src'] ) ) {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ // Check if the source file is too big
+ } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
$status->fatal( 'backend-fail-maxsize',
$this->params['dst'], $this->backend->maxFileSizeInternal() );
- $status->fatal( 'backend-fail-create', $this->params['dst'] );
+ $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
return $status;
- // Check if a file can be placed at the destination
+ // Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
- $status->fatal( 'backend-fail-create', $this->params['dst'] );
+ $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
return $status;
}
// Check if destination file exists
$status->merge( $this->precheckDestExistence( $predicates ) );
+ $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache()
if ( $status->isOK() ) {
// Update file existence predicates
$predicates['exists'][$this->params['dst']] = true;
@@ -541,24 +561,27 @@ class CreateFileOp extends FileOp {
* @return Status
*/
protected function doAttempt() {
+ // Store the file at the destination
if ( !$this->destSameAsSource ) {
- // Create the file at the destination
- return $this->backend->createInternal( $this->setFlags( $this->params ) );
+ return $this->backend->storeInternal( $this->setFlags( $this->params ) );
}
return Status::newGood();
}
/**
- * @return bool|String
+ * @return bool|string
*/
protected function getSourceSha1Base36() {
- return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
+ wfSuppressWarnings();
+ $hash = sha1_file( $this->params['src'] );
+ wfRestoreWarnings();
+ if ( $hash !== false ) {
+ $hash = wfBaseConvert( $hash, 16, 36, 31 );
+ }
+ return $hash;
}
- /**
- * @return array
- */
- protected function doStoragePathsChanged() {
+ public function storagePathsChanged() {
return array( $this->params['dst'] );
}
}
@@ -573,7 +596,7 @@ class CopyFileOp extends FileOp {
*/
protected function allowedParams() {
return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'disposition' ) );
+ array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
}
/**
@@ -584,9 +607,17 @@ class CopyFileOp extends FileOp {
$status = Status::newGood();
// Check if the source file exists
if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
- $status->fatal( 'backend-fail-notexists', $this->params['src'] );
- return $status;
- // Check if a file can be placed at the destination
+ if ( $this->getParam( 'ignoreMissingSource' ) ) {
+ $this->doOperation = false; // no-op
+ // Update file existence predicates (cache 404s)
+ $predicates['exists'][$this->params['src']] = false;
+ $predicates['sha1'][$this->params['src']] = false;
+ return $status; // nothing to do
+ } else {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ }
+ // Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
$status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
@@ -594,6 +625,7 @@ class CopyFileOp extends FileOp {
}
// Check if destination file exists
$status->merge( $this->precheckDestExistence( $predicates ) );
+ $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache()
if ( $status->isOK() ) {
// Update file existence predicates
$predicates['exists'][$this->params['dst']] = true;
@@ -619,14 +651,14 @@ class CopyFileOp extends FileOp {
/**
* @return array
*/
- protected function doStoragePathsRead() {
+ public function storagePathsRead() {
return array( $this->params['src'] );
}
/**
* @return array
*/
- protected function doStoragePathsChanged() {
+ public function storagePathsChanged() {
return array( $this->params['dst'] );
}
}
@@ -641,7 +673,7 @@ class MoveFileOp extends FileOp {
*/
protected function allowedParams() {
return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'disposition' ) );
+ array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
}
/**
@@ -652,9 +684,17 @@ class MoveFileOp extends FileOp {
$status = Status::newGood();
// Check if the source file exists
if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
- $status->fatal( 'backend-fail-notexists', $this->params['src'] );
- return $status;
- // Check if a file can be placed at the destination
+ if ( $this->getParam( 'ignoreMissingSource' ) ) {
+ $this->doOperation = false; // no-op
+ // Update file existence predicates (cache 404s)
+ $predicates['exists'][$this->params['src']] = false;
+ $predicates['sha1'][$this->params['src']] = false;
+ return $status; // nothing to do
+ } else {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ }
+ // Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
$status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
@@ -662,6 +702,7 @@ class MoveFileOp extends FileOp {
}
// Check if destination file exists
$status->merge( $this->precheckDestExistence( $predicates ) );
+ $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache()
if ( $status->isOK() ) {
// Update file existence predicates
$predicates['exists'][$this->params['src']] = false;
@@ -693,14 +734,14 @@ class MoveFileOp extends FileOp {
/**
* @return array
*/
- protected function doStoragePathsRead() {
+ public function storagePathsRead() {
return array( $this->params['src'] );
}
/**
* @return array
*/
- protected function doStoragePathsChanged() {
+ public function storagePathsChanged() {
return array( $this->params['src'], $this->params['dst'] );
}
}
@@ -717,21 +758,29 @@ class DeleteFileOp extends FileOp {
return array( array( 'src' ), array( 'ignoreMissingSource' ) );
}
- protected $needsDelete = true;
-
/**
- * @param array $predicates
+ * @param $predicates array
* @return Status
*/
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source file exists
if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
- if ( !$this->getParam( 'ignoreMissingSource' ) ) {
+ if ( $this->getParam( 'ignoreMissingSource' ) ) {
+ $this->doOperation = false; // no-op
+ // Update file existence predicates (cache 404s)
+ $predicates['exists'][$this->params['src']] = false;
+ $predicates['sha1'][$this->params['src']] = false;
+ return $status; // nothing to do
+ } else {
$status->fatal( 'backend-fail-notexists', $this->params['src'] );
return $status;
}
- $this->needsDelete = false;
+ // Check if a file can be placed/changed at the source
+ } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) {
+ $status->fatal( 'backend-fail-usable', $this->params['src'] );
+ $status->fatal( 'backend-fail-delete', $this->params['src'] );
+ return $status;
}
// Update file existence predicates
$predicates['exists'][$this->params['src']] = false;
@@ -743,17 +792,66 @@ class DeleteFileOp extends FileOp {
* @return Status
*/
protected function doAttempt() {
- if ( $this->needsDelete ) {
- // Delete the source file
- return $this->backend->deleteInternal( $this->setFlags( $this->params ) );
+ // Delete the source file
+ return $this->backend->deleteInternal( $this->setFlags( $this->params ) );
+ }
+
+ /**
+ * @return array
+ */
+ public function storagePathsChanged() {
+ return array( $this->params['src'] );
+ }
+}
+
+/**
+ * Change metadata for a file at the given storage path in the backend.
+ * Parameters for this operation are outlined in FileBackend::doOperations().
+ */
+class DescribeFileOp extends FileOp {
+ /**
+ * @return array
+ */
+ protected function allowedParams() {
+ return array( array( 'src' ), array( 'disposition', 'headers' ) );
+ }
+
+ /**
+ * @param $predicates array
+ * @return Status
+ */
+ protected function doPrecheck( array &$predicates ) {
+ $status = Status::newGood();
+ // Check if the source file exists
+ if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ // Check if a file can be placed/changed at the source
+ } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) {
+ $status->fatal( 'backend-fail-usable', $this->params['src'] );
+ $status->fatal( 'backend-fail-describe', $this->params['src'] );
+ return $status;
}
- return Status::newGood();
+ // Update file existence predicates
+ $predicates['exists'][$this->params['src']] =
+ $this->fileExists( $this->params['src'], $predicates );
+ $predicates['sha1'][$this->params['src']] =
+ $this->fileSha1( $this->params['src'], $predicates );
+ return $status; // safe to call attempt()
+ }
+
+ /**
+ * @return Status
+ */
+ protected function doAttempt() {
+ // Update the source file's metadata
+ return $this->backend->describeInternal( $this->setFlags( $this->params ) );
}
/**
* @return array
*/
- protected function doStoragePathsChanged() {
+ public function storagePathsChanged() {
return array( $this->params['src'] );
}
}
diff --git a/includes/filebackend/FileOpBatch.php b/includes/filebackend/FileOpBatch.php
index 33558725..fc51d78a 100644
--- a/includes/filebackend/FileOpBatch.php
+++ b/includes/filebackend/FileOpBatch.php
@@ -42,9 +42,6 @@ class FileOpBatch {
* $opts is an array of options, including:
* - force : Errors that would normally cause a rollback do not.
* The remaining operations are still attempted if any fail.
- * - allowStale : Don't require the latest available data.
- * This can increase performance for non-critical writes.
- * This has no effect unless the 'force' flag is set.
* - nonJournaled : Don't log this operation batch in the file journal.
* - concurrency : Try to do this many operations in parallel when possible.
*
@@ -52,8 +49,8 @@ class FileOpBatch {
* - a) unexpected operation errors occurred (network partitions, disk full...)
* - b) significant operation errors occurred and 'force' was not set
*
- * @param $performOps Array List of FileOp operations
- * @param $opts Array Batch operation options
+ * @param array $performOps List of FileOp operations
+ * @param array $opts Batch operation options
* @param $journal FileJournal Journal to log operations to
* @return Status
*/
@@ -69,7 +66,6 @@ class FileOpBatch {
}
$batchId = $journal->getTimestampedUUID();
- $allowStale = !empty( $opts['allowStale'] );
$ignoreErrors = !empty( $opts['force'] );
$journaled = empty( $opts['nonJournaled'] );
$maxConcurrency = isset( $opts['concurrency'] ) ? $opts['concurrency'] : 1;
@@ -84,7 +80,6 @@ class FileOpBatch {
foreach ( $performOps as $index => $fileOp ) {
$backendName = $fileOp->getBackend()->getName();
$fileOp->setBatchId( $batchId ); // transaction ID
- $fileOp->allowStaleReads( $allowStale ); // consistency level
// Decide if this op can be done concurrently within this sub-batch
// or if a new concurrent sub-batch must be started after this one...
if ( $fileOp->dependsOn( $curBatchDeps )
@@ -136,49 +131,13 @@ class FileOpBatch {
}
// Attempt each operation (in parallel if allowed and possible)...
- if ( count( $pPerformOps ) < count( $performOps ) ) {
- self::runBatchParallel( $pPerformOps, $status );
- } else {
- self::runBatchSeries( $performOps, $status );
- }
+ self::runParallelBatches( $pPerformOps, $status );
wfProfileOut( __METHOD__ );
return $status;
}
/**
- * Attempt a list of file operations in series.
- * This will abort remaining ops on failure.
- *
- * @param $performOps Array
- * @param $status Status
- * @return bool Success
- */
- protected static function runBatchSeries( array $performOps, Status $status ) {
- foreach ( $performOps as $index => $fileOp ) {
- if ( $fileOp->failed() ) {
- continue; // nothing to do
- }
- $subStatus = $fileOp->attempt();
- $status->merge( $subStatus );
- if ( $subStatus->isOK() ) {
- $status->success[$index] = true;
- ++$status->successCount;
- } else {
- $status->success[$index] = false;
- ++$status->failCount;
- // We can't continue (even with $ignoreErrors) as $predicates is wrong.
- // Log the remaining ops as failed for recovery...
- for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
- $performOps[$i]->logFailure( 'attempt_aborted' );
- }
- return false; // bail out
- }
- }
- return true;
- }
-
- /**
* Attempt a list of file operations sub-batches in series.
*
* The operations *in* each sub-batch will be done in parallel.
@@ -190,8 +149,8 @@ class FileOpBatch {
* @param $status Status
* @return bool Success
*/
- protected static function runBatchParallel( array $pPerformOps, Status $status ) {
- $aborted = false;
+ protected static function runParallelBatches( array $pPerformOps, Status $status ) {
+ $aborted = false; // set to true on unexpected errors
foreach ( $pPerformOps as $performOpsBatch ) {
if ( $aborted ) { // check batch op abort flag...
// We can't continue (even with $ignoreErrors) as $predicates is wrong.
@@ -205,11 +164,16 @@ class FileOpBatch {
$opHandles = array();
// Get the backend; all sub-batch ops belong to a single backend
$backend = reset( $performOpsBatch )->getBackend();
- // If attemptAsync() returns synchronously, it was either an
- // error Status or the backend just doesn't support async ops.
+ // Get the operation handles or actually do it if there is just one.
+ // If attemptAsync() returns a Status, it was either due to an error
+ // or the backend does not support async ops and did it synchronously.
foreach ( $performOpsBatch as $i => $fileOp ) {
if ( !$fileOp->failed() ) { // failed => already has Status
- $subStatus = $fileOp->attemptAsync();
+ // If the batch is just one operation, it's faster to avoid
+ // pipelining as that can involve creating new TCP connections.
+ $subStatus = ( count( $performOpsBatch ) > 1 )
+ ? $fileOp->attemptAsync()
+ : $fileOp->attempt();
if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
$opHandles[$i] = $subStatus->value; // deferred
} else {
diff --git a/includes/filebackend/README b/includes/filebackend/README
new file mode 100644
index 00000000..6ab54810
--- /dev/null
+++ b/includes/filebackend/README
@@ -0,0 +1,208 @@
+/*!
+\ingroup FileBackend
+\page file_backend_design File backend design
+
+Some notes on the FileBackend architecture.
+
+\section intro Introduction
+
+To abstract away the differences among different types of storage media,
+MediaWiki is providing an interface known as FileBackend. Any MediaWiki
+interaction with stored files should thus use a FileBackend object.
+
+Different types of backing storage media are supported (ranging from local
+file system to distributed object stores). The types include:
+
+* FSFileBackend (used for mounted file systems)
+* SwiftFileBackend (used for Swift or Ceph Rados+RGW object stores)
+* FileBackendMultiWrite (useful for transitioning from one backend to another)
+
+Configuration documentation for each type of backend is to be found in their
+__construct() inline documentation.
+
+
+\section setup Setup
+
+File backends are registered in LocalSettings.php via the global variable
+$wgFileBackends. To access one of those defined backends, one would use
+FileBackendStore::get( <name> ) which will bring back a FileBackend object
+handle. Such handles are reused for any subsequent get() call (via singleton).
+The FileBackends objects are caching request calls such as file stats,
+SHA1 requests or TCP connection handles.
+
+\par Note:
+Some backends may require additional PHP extensions to be enabled or can rely on a
+MediaWiki extension. This is often the case when a FileBackend subclass makes use of an
+upstream client API for communicating with the backing store.
+
+
+\section fileoperations File operations
+
+The MediaWiki FileBackend API supports various operations on either files or
+directories. See FileBackend.php for full documentation for each function.
+
+
+\subsection reading Reading
+
+The following basic operations are supported for reading from a backend:
+
+On files:
+* state a file for basic information (timestamp, size)
+* read a file into a string or several files into a map of path names to strings
+* download a file or set of files to a temporary file (on a mounted file system)
+* get the SHA1 hash of a file
+* get various properties of a file (stat information, content time, mime information, ...)
+
+On directories:
+* get a list of files directly under a directory
+* get a recursive list of files under a directory
+* get a list of directories directly under a directory
+* get a recursive list of directories under a directory
+
+\par Note:
+Backend handles should return directory listings as iterators, all though in some cases
+they may just be simple arrays (which can still be iterated over). Iterators allow for
+callers to traverse a large number of file listings without consuming excessive RAM in
+the process. Either the memory consumed is flatly bounded (if the iterator does paging)
+or it is proportional to the depth of the portion of the directory tree being traversed
+(if the iterator works via recursion).
+
+
+\subsection writing Writing
+
+The following basic operations are supported for writing or changing in the backend:
+
+On files:
+* store (copying a mounted file system file into storage)
+* create (creating a file within storage from a string)
+* copy (within storage)
+* move (within storage)
+* delete (within storage)
+* lock/unlock (lock or unlock a file in storage)
+
+The following operations are supported for writing directories in the backend:
+* prepare (create parent container and directories for a path)
+* secure (try to lock-down access to a container)
+* publish (try to reverse the effects of secure)
+* clean (remove empty containers or directories)
+
+
+\subsection invokingoperation Invoking an operation
+
+Generally, callers should use doOperations() or doQuickOperations() when doing
+batches of changes, rather than making a suite of single operation calls. This
+makes the system tolerate high latency much better by pipelining operations
+when possible.
+
+doOperations() should be used for working on important original data, i.e. when
+consistency is important. The former will only pipeline operations that do not
+depend on each other. It is best if the operations that do not depend on each
+other occur in consecutive groups. This function can also log file changes to
+a journal (see FileJournal), which can be used to sync two backend instances.
+One might use this function for user uploads of file for example.
+
+doQuickOperations() is more geared toward ephemeral items that can be easily
+regenerated from original data. It will always pipeline without checking for
+dependencies within the operation batch. One might use this function for
+creating and purging generated thumbnails of original files for example.
+
+
+\section consistency Consistency
+
+Not all backing stores are sequentially consistent by default. Various FileBackend
+functions offer a "latest" option that can be passed in to assure (or try to assure)
+that the latest version of the file is read. Some backing stores are consistent by
+default, but callers should always assume that without this option, stale data may
+be read. This is actually true for stores that have eventual consistency.
+
+Note that file listing functions have no "latest" flag, and thus some systems may
+return stale data. Thus callers should avoid assuming that listings contain changes
+made my the current client or any other client from a very short time ago. For example,
+creating a file under a directory and then immediately doing a file listing operation
+on that directory may result in a listing that does not include that file.
+
+
+\section locking Locking
+
+Locking is effective if and only if a proper lock manager is registered and is
+actually being used by the backend. Lock managers can be registered in LocalSettings.php
+using the $wgLockManagers global configuration variable.
+
+For object stores, locking is not generally useful for avoiding partially
+written or read objects, since most stores use Multi Version Concurrency
+Control (MVCC) to avoid this. However, locking can be important when:
+* One or more operations must be done without objects changing in the meantime.
+* It can also be useful when a file read is used to determine a file write or DB change.
+ For example, doOperations() first checks that there will be no "file already exists"
+ or "file does not exist" type errors before attempting an operation batch. This works
+ by stating the files first, and is only safe if the files are locked in the meantime.
+
+When locking, callers should use the latest available file data for reads.
+Also, one should always lock the file *before* reading it, not after. If stale data is
+used to determine a write, there will be some data corruption, even when reads of the
+original file finally start returning the updated data without needing the "latest"
+option (eventual consistency). The "scoped" lock functions are preferable since
+there is not the problem of forgetting to unlock due to early returns or exceptions.
+
+Since acquiring locks can fail, and lock managers can be non-blocking, callers should:
+* Acquire all required locks up font
+* Be prepared for the case where locks fail to be acquired
+* Possible retry acquiring certain locks
+
+MVCC is also a useful pattern to use on top of the backend interface, because operations
+are not atomic, even with doOperations(), so doing complex batch file changes or changing
+files and updating a database row can result in partially written "transactions". Thus one
+should avoid changing files once they have been stored, except perhaps with ephemeral data
+that are tolerant of some degree of inconsistency.
+
+Callers can use their own locking (e.g. SELECT FOR UPDATE) if it is more convenient, but
+note that all callers that change any of the files should then go through functions that
+acquire these locks. For example, if a caller just directly uses the file backend store()
+function, it will ignore any custom "FOR UPDATE" locks, which can cause problems.
+
+\section objectstore Object stores
+
+Support for object stores (like Amazon S3/Swift) drive much of the API and design
+decisions of FileBackend, but using any POSIX compliant file systems works fine.
+The system essentially stores "files" in "containers". For a mounted file system
+as a backing store, "files" will just be files under directories. For an object store
+as a backing store, the "files" will be objects stored in actual containers.
+
+
+\section file_obj_diffs File system and Object store differences
+
+An advantage of object stores is the reduced Round-Trip Times. This is
+achieved by avoiding the need to create each parent directory before placing a
+file somewhere. It gets worse the deeper the directory hierarchy is. Another
+advantage of object stores is that object listings tend to use databases, which
+scale better than the linked list directories that file sytems sometimes use.
+File systems like btrfs and xfs use tree structures, which scale better.
+For both object stores and file systems, using "/" in filenames will allow for the
+intuitive use of directory functions. For example, creating a file in Swift
+called "container/a/b/file1" will mean that:
+- a "directory listing" of "container/a" will contain "b",
+- and a "file listing" of "b" will contain "file1"
+
+This means that switching from an object store to a file system and vise versa
+using the FileBackend interface will generally be harmless. However, one must be
+aware of some important differences:
+
+* In a file system, you cannot have a file and a directory within the same path
+ whereas it is possible in an object stores. Calling code should avoid any layouts
+ which allow files and directories at the same path.
+* Some file systems have file name length restrictions or overall path length
+ restrictions that others do not. The same goes with object stores which might
+ have a maximum object length or a limitation regarding the number of files
+ under a container or volume.
+* Latency varies among systems, certain access patterns may not be tolerable for
+ certain backends but may hold up for others. Some backend subclasses use
+ MediaWiki's object caching for serving stat requests, which can greatly
+ reduce latency. Making sure that the backend has pipelining (see the
+ "parallelize" and "concurrency" settings) enabled can also mask latency in
+ batch operation scenarios.
+* File systems may implement directories as linked-lists or other structures
+ with poor scalability, so calling code should use layouts that shard the data.
+ Instead of storing files like "container/file.txt", one can store files like
+ "container/<x>/<y>/file.txt". It is best if "sharding" optional or configurable.
+
+*/
diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php
index b6f0aa60..0f3d97a3 100644
--- a/includes/filebackend/SwiftFileBackend.php
+++ b/includes/filebackend/SwiftFileBackend.php
@@ -40,11 +40,16 @@ class SwiftFileBackend extends FileBackendStore {
/** @var CF_Authentication */
protected $auth; // Swift authentication handler
protected $authTTL; // integer seconds
+ protected $swiftTempUrlKey; // string; shared secret value for making temp urls
protected $swiftAnonUser; // string; username to handle unauthenticated requests
protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
+ // Rados Gateway specific options
+ protected $rgwS3AccessKey; // string; S3 access key
+ protected $rgwS3SecretKey; // string; S3 authentication key
+
/** @var CF_Connection */
protected $conn; // Swift connection handle
protected $sessionStarted = 0; // integer UNIX timestamp
@@ -66,6 +71,8 @@ class SwiftFileBackend extends FileBackendStore {
* - swiftUser : Swift user used by MediaWiki (account:username)
* - swiftKey : Swift authentication key for the above user
* - swiftAuthTTL : Swift authentication TTL (seconds)
+ * - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
+ * Do not set this until it has been set in the backend.
* - swiftAnonUser : Swift user used for end-user requests (account:username).
* If set, then views of public containers are assumed to go
* through this user. If not set, then public containers are
@@ -84,6 +91,16 @@ class SwiftFileBackend extends FileBackendStore {
* - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
* If those are not available, then the main cache will be used.
* This is probably insecure in shared hosting environments.
+ * - rgwS3AccessKey : Ragos Gateway S3 "access key" value on the account.
+ * Do not set this until it has been set in the backend.
+ * This is used for generating expiring pre-authenticated URLs.
+ * Only use this when using rgw and to work around
+ * http://tracker.newdream.net/issues/3454.
+ * - rgwS3SecretKey : Ragos Gateway S3 "secret key" value on the account.
+ * Do not set this until it has been set in the backend.
+ * This is used for generating expiring pre-authenticated URLs.
+ * Only use this when using rgw and to work around
+ * http://tracker.newdream.net/issues/3454.
*/
public function __construct( array $config ) {
parent::__construct( $config );
@@ -104,6 +121,9 @@ class SwiftFileBackend extends FileBackendStore {
$this->swiftAnonUser = isset( $config['swiftAnonUser'] )
? $config['swiftAnonUser']
: '';
+ $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
+ ? $config['swiftTempUrlKey']
+ : '';
$this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
? $config['shardViaHashLevels']
: '';
@@ -116,13 +136,19 @@ class SwiftFileBackend extends FileBackendStore {
$this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
? $config['swiftCDNPurgable']
: true;
+ $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
+ ? $config['rgwS3AccessKey']
+ : '';
+ $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
+ ? $config['rgwS3SecretKey']
+ : '';
// Cache container information to mask latency
$this->memCache = wfGetMainCache();
// Process cache for container info
$this->connContainerCache = new ProcessCacheLRU( 300 );
// Cache auth token information to avoid RTTs
if ( !empty( $config['cacheAuthInfo'] ) ) {
- if ( php_sapi_name() === 'cli' ) {
+ if ( PHP_SAPI === 'cli' ) {
$this->srvCache = wfGetMainCache(); // preferrably memcached
} else {
try { // look for APC, XCache, WinCache, ect...
@@ -168,14 +194,14 @@ class SwiftFileBackend extends FileBackendStore {
}
/**
- * @param $disposition string Content-Disposition header value
+ * @param string $disposition Content-Disposition header value
* @return string Truncated Content-Disposition header value to meet Swift limits
*/
protected function truncDisp( $disposition ) {
$res = '';
foreach ( explode( ';', $disposition ) as $part ) {
$part = trim( $part );
- $new = ( $res === '' ) ? $part : "{$res};{$part}";
+ $new = ( $res === '' ) ? $part : "{$res};{$part}";
if ( strlen( $new ) <= 255 ) {
$res = $new;
} else {
@@ -201,12 +227,6 @@ class SwiftFileBackend extends FileBackendStore {
// (a) Check the destination container and object
try {
$dContObj = $this->getContainer( $dstCont );
- if ( empty( $params['overwrite'] ) &&
- $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
- {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-create', $params['dst'] );
return $status;
@@ -223,8 +243,7 @@ class SwiftFileBackend extends FileBackendStore {
// Create a fresh CF_Object with no fields preloaded.
// We don't want to preserve headers, metadata, and such.
$obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
- // Note: metadata keys stored as [Upper case char][[Lower case char]...]
- $obj->metadata = array( 'Sha1base36' => $sha1Hash );
+ $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
// Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
// The MD5 here will be checked within Swift against its own MD5.
$obj->set_etag( md5( $params['content'] ) );
@@ -237,17 +256,17 @@ class SwiftFileBackend extends FileBackendStore {
if ( isset( $params['disposition'] ) ) {
$obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
}
+ // Set any other custom headers if requested
+ if ( isset( $params['headers'] ) ) {
+ $obj->headers += $params['headers'];
+ }
if ( !empty( $params['async'] ) ) { // deferred
$op = $obj->write_async( $params['content'] );
$status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $status->value->affectedObjects[] = $obj;
- }
+ $status->value->affectedObjects[] = $obj;
} else { // actually write the object in Swift
$obj->write( $params['content'] );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $this->purgeCDNCache( array( $obj ) );
- }
+ $this->purgeCDNCache( array( $obj ) );
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
@@ -287,12 +306,6 @@ class SwiftFileBackend extends FileBackendStore {
// (a) Check the destination container and object
try {
$dContObj = $this->getContainer( $dstCont );
- if ( empty( $params['overwrite'] ) &&
- $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
- {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
return $status;
@@ -302,7 +315,9 @@ class SwiftFileBackend extends FileBackendStore {
}
// (b) Get a SHA-1 hash of the object
+ wfSuppressWarnings();
$sha1Hash = sha1_file( $params['src'] );
+ wfRestoreWarnings();
if ( $sha1Hash === false ) { // source doesn't exist?
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
return $status;
@@ -314,8 +329,7 @@ class SwiftFileBackend extends FileBackendStore {
// Create a fresh CF_Object with no fields preloaded.
// We don't want to preserve headers, metadata, and such.
$obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
- // Note: metadata keys stored as [Upper case char][[Lower case char]...]
- $obj->metadata = array( 'Sha1base36' => $sha1Hash );
+ $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
// The MD5 here will be checked within Swift against its own MD5.
$obj->set_etag( md5_file( $params['src'] ) );
// Use the same content type as StreamFile for security
@@ -327,6 +341,10 @@ class SwiftFileBackend extends FileBackendStore {
if ( isset( $params['disposition'] ) ) {
$obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
}
+ // Set any other custom headers if requested
+ if ( isset( $params['headers'] ) ) {
+ $obj->headers += $params['headers'];
+ }
if ( !empty( $params['async'] ) ) { // deferred
wfSuppressWarnings();
$fp = fopen( $params['src'], 'rb' );
@@ -337,15 +355,11 @@ class SwiftFileBackend extends FileBackendStore {
$op = $obj->write_async( $fp, filesize( $params['src'] ), true );
$status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
$status->value->resourcesToClose[] = $fp;
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $status->value->affectedObjects[] = $obj;
- }
+ $status->value->affectedObjects[] = $obj;
}
} else { // actually write the object in Swift
$obj->load_from_filename( $params['src'], true ); // calls $obj->write()
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $this->purgeCDNCache( array( $obj ) );
- }
+ $this->purgeCDNCache( array( $obj ) );
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
@@ -396,14 +410,10 @@ class SwiftFileBackend extends FileBackendStore {
try {
$sContObj = $this->getContainer( $srcCont );
$dContObj = $this->getContainer( $dstCont );
- if ( empty( $params['overwrite'] ) &&
- $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
- {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
} catch ( NoSuchContainerException $e ) {
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
+ $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ }
return $status;
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
@@ -420,19 +430,17 @@ class SwiftFileBackend extends FileBackendStore {
if ( !empty( $params['async'] ) ) { // deferred
$op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
$status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $status->value->affectedObjects[] = $dstObj;
- }
+ $status->value->affectedObjects[] = $dstObj;
} else { // actually write the object in Swift
$sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $this->purgeCDNCache( array( $dstObj ) );
- }
+ $this->purgeCDNCache( array( $dstObj ) );
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
} catch ( NoSuchObjectException $e ) { // source object does not exist
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ }
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
}
@@ -474,14 +482,10 @@ class SwiftFileBackend extends FileBackendStore {
try {
$sContObj = $this->getContainer( $srcCont );
$dContObj = $this->getContainer( $dstCont );
- if ( empty( $params['overwrite'] ) &&
- $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
- {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
} catch ( NoSuchContainerException $e ) {
- $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+ if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
+ $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+ }
return $status;
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
@@ -500,20 +504,18 @@ class SwiftFileBackend extends FileBackendStore {
$op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
$status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
$status->value->affectedObjects[] = $srcObj;
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $status->value->affectedObjects[] = $dstObj;
- }
+ $status->value->affectedObjects[] = $dstObj;
} else { // actually write the object in Swift
$sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
$this->purgeCDNCache( array( $srcObj ) );
- if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
- $this->purgeCDNCache( array( $dstObj ) );
- }
+ $this->purgeCDNCache( array( $dstObj ) );
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
} catch ( NoSuchObjectException $e ) { // source object does not exist
- $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+ }
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
}
@@ -559,7 +561,9 @@ class SwiftFileBackend extends FileBackendStore {
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
} catch ( NoSuchContainerException $e ) {
- $status->fatal( 'backend-fail-delete', $params['src'] );
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-delete', $params['src'] );
+ }
} catch ( NoSuchObjectException $e ) {
if ( empty( $params['ignoreMissingSource'] ) ) {
$status->fatal( 'backend-fail-delete', $params['src'] );
@@ -587,6 +591,47 @@ class SwiftFileBackend extends FileBackendStore {
}
/**
+ * @see FileBackendStore::doDescribeInternal()
+ * @return Status
+ */
+ protected function doDescribeInternal( array $params ) {
+ $status = Status::newGood();
+
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
+ if ( $srcRel === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['src'] );
+ return $status;
+ }
+
+ $hdrs = isset( $params['headers'] ) ? $params['headers'] : array();
+ // Set the Content-Disposition header if requested
+ if ( isset( $params['disposition'] ) ) {
+ $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
+ }
+
+ try {
+ $sContObj = $this->getContainer( $srcCont );
+ // Get the latest version of the current metadata
+ $srcObj = $sContObj->get_object( $srcRel,
+ $this->headersFromParams( array( 'latest' => true ) ) );
+ // Merge in the metadata updates...
+ $srcObj->headers = $hdrs + $srcObj->headers;
+ $srcObj->sync_metadata(); // save to Swift
+ $this->purgeCDNCache( array( $srcObj ) );
+ } catch ( CDNNotEnabledException $e ) {
+ // CDN not enabled; nothing to see here
+ } catch ( NoSuchContainerException $e ) {
+ $status->fatal( 'backend-fail-describe', $params['src'] );
+ } catch ( NoSuchObjectException $e ) {
+ $status->fatal( 'backend-fail-describe', $params['src'] );
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $this->handleException( $e, $status, __METHOD__, $params );
+ }
+
+ return $status;
+ }
+
+ /**
* @see FileBackendStore::doPrepareInternal()
* @return Status
*/
@@ -595,7 +640,7 @@ class SwiftFileBackend extends FileBackendStore {
// (a) Check if container already exists
try {
- $contObj = $this->getContainer( $fullCont );
+ $this->getContainer( $fullCont );
// NoSuchContainerException not thrown: container must exist
return $status; // already exists
} catch ( NoSuchContainerException $e ) {
@@ -761,7 +806,7 @@ class SwiftFileBackend extends FileBackendStore {
// Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
'size' => (int)$srcObj->content_length,
- 'sha1' => $srcObj->metadata['Sha1base36']
+ 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' )
);
} catch ( NoSuchContainerException $e ) {
} catch ( NoSuchObjectException $e ) {
@@ -777,60 +822,106 @@ class SwiftFileBackend extends FileBackendStore {
* Fill in any missing object metadata and save it to Swift
*
* @param $obj CF_Object
- * @param $path string Storage path to object
+ * @param string $path Storage path to object
* @return bool Success
* @throws Exception cloudfiles exceptions
*/
protected function addMissingMetadata( CF_Object $obj, $path ) {
- if ( isset( $obj->metadata['Sha1base36'] ) ) {
+ if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
return true; // nothing to do
}
wfProfileIn( __METHOD__ );
+ trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
$status = Status::newGood();
$scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
if ( $status->isOK() ) {
- # Do not stat the file in getLocalCopy() to avoid infinite loops
- $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
+ $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
if ( $tmpFile ) {
$hash = $tmpFile->getSha1Base36();
if ( $hash !== false ) {
- $obj->metadata['Sha1base36'] = $hash;
+ $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
$obj->sync_metadata(); // save to Swift
wfProfileOut( __METHOD__ );
return true; // success
}
}
}
- $obj->metadata['Sha1base36'] = false;
+ trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
+ $obj->setMetadataValues( array( 'Sha1base36' => false ) );
wfProfileOut( __METHOD__ );
return false; // failed
}
/**
- * @see FileBackend::getFileContents()
- * @return bool|null|string
+ * @see FileBackendStore::doGetFileContentsMulti()
+ * @return Array
*/
- public function getFileContents( array $params ) {
- list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
- if ( $srcRel === null ) {
- return false; // invalid storage path
- }
-
- if ( !$this->fileExists( $params ) ) {
- return null;
- }
+ protected function doGetFileContentsMulti( array $params ) {
+ $contents = array();
+
+ $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
+ // Blindly create tmp files and stream to them, catching any exception if the file does
+ // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
+ foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
+ $cfOps = array(); // (path => CF_Async_Op)
+
+ foreach ( $pathBatch as $path ) { // each path in this concurrent batch
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ if ( $srcRel === null ) {
+ $contents[$path] = false;
+ continue;
+ }
+ $data = false;
+ try {
+ $sContObj = $this->getContainer( $srcCont );
+ $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
+ // Create a new temporary memory file...
+ $handle = fopen( 'php://temp', 'wb' );
+ if ( $handle ) {
+ $headers = $this->headersFromParams( $params );
+ if ( count( $pathBatch ) > 1 ) {
+ $cfOps[$path] = $obj->stream_async( $handle, $headers );
+ $cfOps[$path]->_file_handle = $handle; // close this later
+ } else {
+ $obj->stream( $handle, $headers );
+ rewind( $handle ); // start from the beginning
+ $data = stream_get_contents( $handle );
+ fclose( $handle );
+ }
+ } else {
+ $data = false;
+ }
+ } catch ( NoSuchContainerException $e ) {
+ $data = false;
+ } catch ( NoSuchObjectException $e ) {
+ $data = false;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $data = false;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
+ }
+ $contents[$path] = $data;
+ }
- $data = false;
- try {
- $sContObj = $this->getContainer( $srcCont );
- $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- $data = $obj->read( $this->headersFromParams( $params ) );
- } catch ( NoSuchContainerException $e ) {
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__, $params );
+ $batch = new CF_Async_Op_Batch( $cfOps );
+ $cfOps = $batch->execute();
+ foreach ( $cfOps as $path => $cfOp ) {
+ try {
+ $cfOp->getLastResponse();
+ rewind( $cfOp->_file_handle ); // start from the beginning
+ $contents[$path] = stream_get_contents( $cfOp->_file_handle );
+ } catch ( NoSuchContainerException $e ) {
+ $contents[$path] = false;
+ } catch ( NoSuchObjectException $e ) {
+ $contents[$path] = false;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $contents[$path] = false;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
+ }
+ fclose( $cfOp->_file_handle ); // close open handle
+ }
}
- return $data;
+ return $contents;
}
/**
@@ -871,11 +962,11 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Do not call this function outside of SwiftFileBackendFileList
*
- * @param $fullCont string Resolved container name
- * @param $dir string Resolved storage directory with no trailing slash
- * @param $after string|null Storage path of file to list items after
+ * @param string $fullCont Resolved container name
+ * @param string $dir Resolved storage directory with no trailing slash
+ * @param string|null $after Storage path of file to list items after
* @param $limit integer Max number of items to list
- * @param $params Array Includes flag for 'topOnly'
+ * @param array $params Includes flag for 'topOnly'
* @return Array List of relative paths of dirs directly under $dir
*/
public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
@@ -903,7 +994,7 @@ class SwiftFileBackend extends FileBackendStore {
$objects = $container->list_objects( $limit, $after, $prefix );
foreach ( $objects as $object ) { // files
$objectDir = $this->getParentDir( $object ); // directory of object
- if ( $objectDir !== false ) { // file has a parent dir
+ if ( $objectDir !== false && $objectDir !== $dir ) {
// Swift stores paths in UTF-8, using binary sorting.
// See function "create_container_table" in common/db.py.
// If a directory is not "greater" than the last one,
@@ -944,11 +1035,11 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Do not call this function outside of SwiftFileBackendFileList
*
- * @param $fullCont string Resolved container name
- * @param $dir string Resolved storage directory with no trailing slash
- * @param $after string|null Storage path of file to list items after
+ * @param string $fullCont Resolved container name
+ * @param string $dir Resolved storage directory with no trailing slash
+ * @param string|null $after Storage path of file to list items after
* @param $limit integer Max number of items to list
- * @param $params Array Includes flag for 'topOnly'
+ * @param array $params Includes flag for 'topOnly'
* @return Array List of relative paths of files under $dir
*/
public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
@@ -1038,44 +1129,125 @@ class SwiftFileBackend extends FileBackendStore {
}
/**
- * @see FileBackendStore::getLocalCopy()
+ * @see FileBackendStore::doGetLocalCopyMulti()
* @return null|TempFSFile
*/
- public function getLocalCopy( array $params ) {
- list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
- if ( $srcRel === null ) {
- return null;
- }
+ protected function doGetLocalCopyMulti( array $params ) {
+ $tmpFiles = array();
+
+ $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
+ // Blindly create tmp files and stream to them, catching any exception if the file does
+ // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
+ foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
+ $cfOps = array(); // (path => CF_Async_Op)
+
+ foreach ( $pathBatch as $path ) { // each path in this concurrent batch
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ if ( $srcRel === null ) {
+ $tmpFiles[$path] = null;
+ continue;
+ }
+ $tmpFile = null;
+ try {
+ $sContObj = $this->getContainer( $srcCont );
+ $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
+ // Get source file extension
+ $ext = FileBackend::extensionFromPath( $path );
+ // Create a new temporary file...
+ $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
+ if ( $tmpFile ) {
+ $handle = fopen( $tmpFile->getPath(), 'wb' );
+ if ( $handle ) {
+ $headers = $this->headersFromParams( $params );
+ if ( count( $pathBatch ) > 1 ) {
+ $cfOps[$path] = $obj->stream_async( $handle, $headers );
+ $cfOps[$path]->_file_handle = $handle; // close this later
+ } else {
+ $obj->stream( $handle, $headers );
+ fclose( $handle );
+ }
+ } else {
+ $tmpFile = null;
+ }
+ }
+ } catch ( NoSuchContainerException $e ) {
+ $tmpFile = null;
+ } catch ( NoSuchObjectException $e ) {
+ $tmpFile = null;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $tmpFile = null;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
+ }
+ $tmpFiles[$path] = $tmpFile;
+ }
- // Blindly create a tmp file and stream to it, catching any exception if the file does
- // not exist. Also, doing a stat here will cause infinite loops when filling metadata.
- $tmpFile = null;
- try {
- $sContObj = $this->getContainer( $srcCont );
- $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- // Get source file extension
- $ext = FileBackend::extensionFromPath( $srcRel );
- // Create a new temporary file...
- $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
- if ( $tmpFile ) {
- $handle = fopen( $tmpFile->getPath(), 'wb' );
- if ( $handle ) {
- $obj->stream( $handle, $this->headersFromParams( $params ) );
- fclose( $handle );
- } else {
- $tmpFile = null; // couldn't open temp file
+ $batch = new CF_Async_Op_Batch( $cfOps );
+ $cfOps = $batch->execute();
+ foreach ( $cfOps as $path => $cfOp ) {
+ try {
+ $cfOp->getLastResponse();
+ } catch ( NoSuchContainerException $e ) {
+ $tmpFiles[$path] = null;
+ } catch ( NoSuchObjectException $e ) {
+ $tmpFiles[$path] = null;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $tmpFiles[$path] = null;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
}
+ fclose( $cfOp->_file_handle ); // close open handle
}
- } catch ( NoSuchContainerException $e ) {
- $tmpFile = null;
- } catch ( NoSuchObjectException $e ) {
- $tmpFile = null;
- } catch ( CloudFilesException $e ) { // some other exception?
- $tmpFile = null;
- $this->handleException( $e, null, __METHOD__, $params );
}
- return $tmpFile;
+ return $tmpFiles;
+ }
+
+ /**
+ * @see FileBackendStore::getFileHttpUrl()
+ * @return string|null
+ */
+ public function getFileHttpUrl( array $params ) {
+ if ( $this->swiftTempUrlKey != '' ||
+ ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
+ {
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
+ if ( $srcRel === null ) {
+ return null; // invalid path
+ }
+ try {
+ $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
+ $sContObj = $this->getContainer( $srcCont );
+ $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
+ if ( $this->swiftTempUrlKey != '' ) {
+ return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" );
+ } else { // give S3 API URL for rgw
+ $expires = time() + $ttl;
+ // Path for signature starts with the bucket
+ $spath = '/' . rawurlencode( $srcCont ) . '/' .
+ str_replace( '%2F', '/', rawurlencode( $srcRel ) );
+ // Calculate the hash
+ $signature = base64_encode( hash_hmac(
+ 'sha1',
+ "GET\n\n\n{$expires}\n{$spath}",
+ $this->rgwS3SecretKey,
+ true // raw
+ ) );
+ // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
+ // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
+ return wfAppendQuery(
+ str_replace( '/swift/v1', '', // S3 API is the rgw default
+ $sContObj->cfs_http->getStorageUrl() . $spath ),
+ array(
+ 'Signature' => $signature,
+ 'Expires' => $expires,
+ 'AWSAccessKeyId' => $this->rgwS3AccessKey )
+ );
+ }
+ } catch ( NoSuchContainerException $e ) {
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $this->handleException( $e, null, __METHOD__, $params );
+ }
+ }
+ return null;
}
/**
@@ -1091,7 +1263,7 @@ class SwiftFileBackend extends FileBackendStore {
* on a FileBackend params array, e.g. that of getLocalCopy().
* $params is currently only checked for a 'latest' flag.
*
- * @param $params Array
+ * @param array $params
* @return Array
*/
protected function headersFromParams( array $params ) {
@@ -1118,8 +1290,8 @@ class SwiftFileBackend extends FileBackendStore {
$cfOps = $batch->execute();
foreach ( $cfOps as $index => $cfOp ) {
$status = Status::newGood();
+ $function = '_getResponse' . $fileOpHandles[$index]->call;
try { // catch exceptions; update status
- $function = '_getResponse' . $fileOpHandles[$index]->call;
$this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
$this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
} catch ( CloudFilesException $e ) { // some other exception?
@@ -1137,12 +1309,12 @@ class SwiftFileBackend extends FileBackendStore {
*
* $readGrps is a list of the possible criteria for a request to have
* access to read a container. Each item is one of the following formats:
- * - account:user : Grants access if the request is by the given user
- * - .r:<regex> : Grants access if the request is from a referrer host that
- * matches the expression and the request is not for a listing.
- * Setting this to '*' effectively makes a container public.
- * - .rlistings:<regex> : Grants access if the request is from a referrer host that
- * matches the expression and the request for a listing.
+ * - account:user : Grants access if the request is by the given user
+ * - ".r:<regex>" : Grants access if the request is from a referrer host that
+ * matches the expression and the request is not for a listing.
+ * Setting this to '*' effectively makes a container public.
+ * -".rlistings:<regex>" : Grants access if the request is from a referrer host that
+ * matches the expression and the request for a listing.
*
* $writeGrps is a list of the possible criteria for a request to have
* access to write to a container. Each item is of the following format:
@@ -1154,8 +1326,8 @@ class SwiftFileBackend extends FileBackendStore {
* (lists are truncated to 10000 item with no way to page), and is just a performance risk.
*
* @param $contObj CF_Container Swift container
- * @param $readGrps Array List of read access routes
- * @param $writeGrps Array List of write access routes
+ * @param array $readGrps List of read access routes
+ * @param array $writeGrps List of write access routes
* @return Status
*/
protected function setContainerAccess(
@@ -1178,7 +1350,7 @@ class SwiftFileBackend extends FileBackendStore {
* Purge the CDN cache of affected objects if CDN caching is enabled.
* This is for Rackspace/Akamai CDNs.
*
- * @param $objects Array List of CF_Object items
+ * @param array $objects List of CF_Object items
* @return void
*/
public function purgeCDNCache( array $objects ) {
@@ -1199,8 +1371,9 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Get an authenticated connection handle to the Swift proxy
*
- * @return CF_Connection|bool False on failure
* @throws CloudFilesException
+ * @throws CloudFilesException|Exception
+ * @return CF_Connection|bool False on failure
*/
protected function getConnection() {
if ( $this->connException instanceof CloudFilesException ) {
@@ -1251,6 +1424,7 @@ class SwiftFileBackend extends FileBackendStore {
protected function closeConnection() {
if ( $this->conn ) {
$this->conn->close(); // close active cURL handles in CF_Http object
+ $this->conn = null;
$this->sessionStarted = 0;
$this->connContainerCache->clear();
}
@@ -1267,18 +1441,11 @@ class SwiftFileBackend extends FileBackendStore {
}
/**
- * @see FileBackendStore::doClearCache()
- */
- protected function doClearCache( array $paths = null ) {
- $this->connContainerCache->clear(); // clear container object cache
- }
-
- /**
* Get a Swift container object, possibly from process cache.
* Use $reCache if the file count or byte count is needed.
*
- * @param $container string Container name
- * @param $bypassCache bool Bypass all caches and load from Swift
+ * @param string $container Container name
+ * @param bool $bypassCache Bypass all caches and load from Swift
* @return CF_Container
* @throws CloudFilesException
*/
@@ -1305,7 +1472,7 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Create a Swift container
*
- * @param $container string Container name
+ * @param string $container Container name
* @return CF_Container
* @throws CloudFilesException
*/
@@ -1319,7 +1486,7 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Delete a Swift container
*
- * @param $container string Container name
+ * @param string $container Container name
* @return void
* @throws CloudFilesException
*/
@@ -1353,7 +1520,7 @@ class SwiftFileBackend extends FileBackendStore {
* @param $e Exception
* @param $status Status|null
* @param $func string
- * @param $params Array
+ * @param array $params
* @return void
*/
protected function handleException( Exception $e, $status, $func, array $params ) {
@@ -1420,9 +1587,9 @@ abstract class SwiftFileBackendList implements Iterator {
/**
* @param $backend SwiftFileBackend
- * @param $fullCont string Resolved container name
- * @param $dir string Resolved directory relative to container
- * @param $params Array
+ * @param string $fullCont Resolved container name
+ * @param string $dir Resolved directory relative to container
+ * @param array $params
*/
public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
$this->backend = $backend;
@@ -1491,11 +1658,11 @@ abstract class SwiftFileBackendList implements Iterator {
/**
* Get the given list portion (page)
*
- * @param $container string Resolved container name
- * @param $dir string Resolved path relative to container
+ * @param string $container Resolved container name
+ * @param string $dir Resolved path relative to container
* @param $after string|null
* @param $limit integer
- * @param $params Array
+ * @param array $params
* @return Traversable|Array|null Returns null on failure
*/
abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php
index 5032bf68..11e125c1 100644
--- a/includes/filebackend/TempFSFile.php
+++ b/includes/filebackend/TempFSFile.php
@@ -82,30 +82,37 @@ class TempFSFile extends FSFile {
* Clean up the temporary file only after an object goes out of scope
*
* @param $object Object
- * @return void
+ * @return TempFSFile This object
*/
public function bind( $object ) {
if ( is_object( $object ) ) {
+ if ( !isset( $object->tempFSFileReferences ) ) {
+ // Init first since $object might use __get() and return only a copy variable
+ $object->tempFSFileReferences = array();
+ }
$object->tempFSFileReferences[] = $this;
}
+ return $this;
}
/**
* Set flag to not clean up after the temporary file
*
- * @return void
+ * @return TempFSFile This object
*/
public function preserve() {
$this->canDelete = false;
+ return $this;
}
/**
* Set flag clean up after the temporary file
*
- * @return void
+ * @return TempFSFile This object
*/
public function autocollect() {
$this->canDelete = true;
+ return $this;
}
/**
diff --git a/includes/filebackend/filejournal/DBFileJournal.php b/includes/filebackend/filejournal/DBFileJournal.php
index f6268c25..73f29a95 100644
--- a/includes/filebackend/filejournal/DBFileJournal.php
+++ b/includes/filebackend/filejournal/DBFileJournal.php
@@ -75,6 +75,9 @@ class DBFileJournal extends FileJournal {
try {
$dbw->insert( 'filejournal', $data, __METHOD__ );
+ if ( mt_rand( 0, 99 ) == 0 ) {
+ $this->purgeOldLogs(); // occasionally delete old logs
+ }
} catch ( DBError $e ) {
$status->fatal( 'filejournal-fail-dbquery', $this->backend );
return $status;
@@ -84,6 +87,35 @@ class DBFileJournal extends FileJournal {
}
/**
+ * @see FileJournal::doGetCurrentPosition()
+ * @return integer|false
+ */
+ protected function doGetCurrentPosition() {
+ $dbw = $this->getMasterDB();
+
+ return $dbw->selectField( 'filejournal', 'MAX(fj_id)',
+ array( 'fj_backend' => $this->backend ),
+ __METHOD__
+ );
+ }
+
+ /**
+ * @see FileJournal::doGetPositionAtTime()
+ * @param $time integer|string timestamp
+ * @return integer|false
+ */
+ protected function doGetPositionAtTime( $time ) {
+ $dbw = $this->getMasterDB();
+
+ $encTimestamp = $dbw->addQuotes( $dbw->timestamp( $time ) );
+ return $dbw->selectField( 'filejournal', 'fj_id',
+ array( 'fj_backend' => $this->backend, "fj_timestamp <= $encTimestamp" ),
+ __METHOD__,
+ array( 'ORDER BY' => 'fj_timestamp DESC' )
+ );
+ }
+
+ /**
* @see FileJournal::doGetChangeEntries()
* @return Array
* @throws DBError
diff --git a/includes/filebackend/filejournal/FileJournal.php b/includes/filebackend/filejournal/FileJournal.php
index ce029bbe..a1b7a459 100644
--- a/includes/filebackend/filejournal/FileJournal.php
+++ b/includes/filebackend/filejournal/FileJournal.php
@@ -54,7 +54,7 @@ abstract class FileJournal {
* Create an appropriate FileJournal object from config
*
* @param $config Array
- * @param $backend string A registered file backend name
+ * @param string $backend A registered file backend name
* @throws MWException
* @return FileJournal
*/
@@ -85,13 +85,13 @@ abstract class FileJournal {
/**
* Log changes made by a batch file operation.
* $entries is an array of log entries, each of which contains:
- * op : Basic operation name (create, store, copy, delete)
+ * op : Basic operation name (create, update, delete)
* path : The storage path of the file
* newSha1 : The final base 36 SHA-1 of the file
* Note that 'false' should be used as the SHA-1 for non-existing files.
*
- * @param $entries Array List of file operations (each an array of parameters)
- * @param $batchId string UUID string that identifies the operation batch
+ * @param array $entries List of file operations (each an array of parameters)
+ * @param string $batchId UUID string that identifies the operation batch
* @return Status
*/
final public function logChangeBatch( array $entries, $batchId ) {
@@ -104,13 +104,45 @@ abstract class FileJournal {
/**
* @see FileJournal::logChangeBatch()
*
- * @param $entries Array List of file operations (each an array of parameters)
- * @param $batchId string UUID string that identifies the operation batch
+ * @param array $entries List of file operations (each an array of parameters)
+ * @param string $batchId UUID string that identifies the operation batch
* @return Status
*/
abstract protected function doLogChangeBatch( array $entries, $batchId );
/**
+ * Get the position ID of the latest journal entry
+ *
+ * @return integer|false
+ */
+ final public function getCurrentPosition() {
+ return $this->doGetCurrentPosition();
+ }
+
+ /**
+ * @see FileJournal::getCurrentPosition()
+ * @return integer|false
+ */
+ abstract protected function doGetCurrentPosition();
+
+ /**
+ * Get the position ID of the latest journal entry at some point in time
+ *
+ * @param $time integer|string timestamp
+ * @return integer|false
+ */
+ final public function getPositionAtTime( $time ) {
+ return $this->doGetPositionAtTime( $time );
+ }
+
+ /**
+ * @see FileJournal::getPositionAtTime()
+ * @param $time integer|string timestamp
+ * @return integer|false
+ */
+ abstract protected function doGetPositionAtTime( $time );
+
+ /**
* Get an array of file change log entries.
* A starting change ID and/or limit can be specified.
*
@@ -169,7 +201,7 @@ abstract class FileJournal {
*/
class NullFileJournal extends FileJournal {
/**
- * @see FileJournal::logChangeBatch()
+ * @see FileJournal::doLogChangeBatch()
* @param $entries array
* @param $batchId string
* @return Status
@@ -179,6 +211,23 @@ class NullFileJournal extends FileJournal {
}
/**
+ * @see FileJournal::doGetCurrentPosition()
+ * @return integer|false
+ */
+ protected function doGetCurrentPosition() {
+ return false;
+ }
+
+ /**
+ * @see FileJournal::doGetPositionAtTime()
+ * @param $time integer|string timestamp
+ * @return integer|false
+ */
+ protected function doGetPositionAtTime( $time ) {
+ return false;
+ }
+
+ /**
* @see FileJournal::doGetChangeEntries()
* @return Array
*/
@@ -187,7 +236,7 @@ class NullFileJournal extends FileJournal {
}
/**
- * @see FileJournal::purgeOldLogs()
+ * @see FileJournal::doPurgeOldLogs()
* @return Status
*/
protected function doPurgeOldLogs() {
diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php
index a8fe258b..f02387dc 100644
--- a/includes/filebackend/lockmanager/DBLockManager.php
+++ b/includes/filebackend/lockmanager/DBLockManager.php
@@ -22,10 +22,9 @@
*/
/**
- * Version of LockManager based on using DB table locks.
+ * Version of LockManager based on using named/row DB locks.
+ *
* This is meant for multi-wiki systems that may share files.
- * All locks are blocking, so it might be useful to set a small
- * lock-wait timeout via server config to curtail deadlocks.
*
* All lock requests for a resource, identified by a hash string, will map
* to one bucket. Each bucket maps to one or several peer DBs, each on their
@@ -37,7 +36,7 @@
* @ingroup LockManager
* @since 1.19
*/
-class DBLockManager extends QuorumLockManager {
+abstract class DBLockManager extends QuorumLockManager {
/** @var Array Map of DB names to server config */
protected $dbServers; // (DB name => server config array)
/** @var BagOStuff */
@@ -67,11 +66,12 @@ class DBLockManager extends QuorumLockManager {
* each having an odd-numbered list of DB names (peers) as values.
* Any DB named 'localDBMaster' will automatically use the DB master
* settings for this wiki (without the need for a dbServers entry).
+ * Only use 'localDBMaster' if the domain is a valid wiki ID.
* - lockExpiry : Lock timeout (seconds) for dropped connections. [optional]
* This tells the DB server how long to wait before assuming
* connection failure and releasing all the locks for a session.
*
- * @param Array $config
+ * @param array $config
*/
public function __construct( array $config ) {
parent::__construct( $config );
@@ -111,65 +111,6 @@ class DBLockManager extends QuorumLockManager {
}
/**
- * Get a connection to a lock DB and acquire locks on $paths.
- * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118.
- *
- * @see QuorumLockManager::getLocksOnServer()
- * @return Status
- */
- protected function getLocksOnServer( $lockSrv, array $paths, $type ) {
- $status = Status::newGood();
-
- if ( $type == self::LOCK_EX ) { // writer locks
- try {
- $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) );
- # Build up values for INSERT clause
- $data = array();
- foreach ( $keys as $key ) {
- $data[] = array( 'fle_key' => $key );
- }
- # Wait on any existing writers and block new ones if we get in
- $db = $this->getConnection( $lockSrv ); // checked in isServerUp()
- $db->insert( 'filelocks_exclusive', $data, __METHOD__ );
- } catch ( DBError $e ) {
- foreach ( $paths as $path ) {
- $status->fatal( 'lockmanager-fail-acquirelock', $path );
- }
- }
- }
-
- return $status;
- }
-
- /**
- * @see QuorumLockManager::freeLocksOnServer()
- * @return Status
- */
- protected function freeLocksOnServer( $lockSrv, array $paths, $type ) {
- return Status::newGood(); // not supported
- }
-
- /**
- * @see QuorumLockManager::releaseAllLocks()
- * @return Status
- */
- protected function releaseAllLocks() {
- $status = Status::newGood();
-
- foreach ( $this->conns as $lockDb => $db ) {
- if ( $db->trxLevel() ) { // in transaction
- try {
- $db->rollback( __METHOD__ ); // finish transaction and kill any rows
- } catch ( DBError $e ) {
- $status->fatal( 'lockmanager-fail-db-release', $lockDb );
- }
- }
- }
-
- return $status;
- }
-
- /**
* @see QuorumLockManager::isServerUp()
* @return bool
*/
@@ -197,8 +138,8 @@ class DBLockManager extends QuorumLockManager {
if ( !isset( $this->conns[$lockDb] ) ) {
$db = null;
if ( $lockDb === 'localDBMaster' ) {
- $lb = wfGetLBFactory()->newMainLB();
- $db = $lb->getConnection( DB_MASTER );
+ $lb = wfGetLBFactory()->getMainLB( $this->domain );
+ $db = $lb->getConnection( DB_MASTER, array(), $this->domain );
} elseif ( isset( $this->dbServers[$lockDb] ) ) {
$config = $this->dbServers[$lockDb];
$db = DatabaseBase::factory( $config['type'], $config );
@@ -274,14 +215,8 @@ class DBLockManager extends QuorumLockManager {
* Make sure remaining locks get cleared for sanity
*/
function __destruct() {
+ $this->releaseAllLocks();
foreach ( $this->conns as $db ) {
- if ( $db->trxLevel() ) { // in transaction
- try {
- $db->rollback( __METHOD__ ); // finish transaction and kill any rows
- } catch ( DBError $e ) {
- // oh well
- }
- }
$db->close();
}
}
@@ -321,27 +256,38 @@ class MySqlLockManager extends DBLockManager {
$status = Status::newGood();
$db = $this->getConnection( $lockSrv ); // checked in isServerUp()
- $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) );
+
+ $keys = array(); // list of hash keys for the paths
+ $data = array(); // list of rows to insert
+ $checkEXKeys = array(); // list of hash keys that this has no EX lock on
# Build up values for INSERT clause
- $data = array();
- foreach ( $keys as $key ) {
+ foreach ( $paths as $path ) {
+ $key = $this->sha1Base36Absolute( $path );
+ $keys[] = $key;
$data[] = array( 'fls_key' => $key, 'fls_session' => $this->session );
+ if ( !isset( $this->locksHeld[$path][self::LOCK_EX] ) ) {
+ $checkEXKeys[] = $key;
+ }
}
- # Block new writers...
+
+ # Block new writers (both EX and SH locks leave entries here)...
$db->insert( 'filelocks_shared', $data, __METHOD__, array( 'IGNORE' ) );
# Actually do the locking queries...
if ( $type == self::LOCK_SH ) { // reader locks
+ $blocked = false;
# Bail if there are any existing writers...
- $blocked = $db->selectField( 'filelocks_exclusive', '1',
- array( 'fle_key' => $keys ),
- __METHOD__
- );
- # Prospective writers that haven't yet updated filelocks_exclusive
- # will recheck filelocks_shared after doing so and bail due to our entry.
+ if ( count( $checkEXKeys ) ) {
+ $blocked = $db->selectField( 'filelocks_exclusive', '1',
+ array( 'fle_key' => $checkEXKeys ),
+ __METHOD__
+ );
+ }
+ # Other prospective writers that haven't yet updated filelocks_exclusive
+ # will recheck filelocks_shared after doing so and bail due to this entry.
} else { // writer locks
$encSession = $db->addQuotes( $this->session );
# Bail if there are any existing writers...
- # The may detect readers, but the safe check for them is below.
+ # This may detect readers, but the safe check for them is below.
# Note: if two writers come at the same time, both bail :)
$blocked = $db->selectField( 'filelocks_shared', '1',
array( 'fls_key' => $keys, "fls_session != $encSession" ),
@@ -371,4 +317,117 @@ class MySqlLockManager extends DBLockManager {
return $status;
}
+
+ /**
+ * @see QuorumLockManager::freeLocksOnServer()
+ * @return Status
+ */
+ protected function freeLocksOnServer( $lockSrv, array $paths, $type ) {
+ return Status::newGood(); // not supported
+ }
+
+ /**
+ * @see QuorumLockManager::releaseAllLocks()
+ * @return Status
+ */
+ protected function releaseAllLocks() {
+ $status = Status::newGood();
+
+ foreach ( $this->conns as $lockDb => $db ) {
+ if ( $db->trxLevel() ) { // in transaction
+ try {
+ $db->rollback( __METHOD__ ); // finish transaction and kill any rows
+ } catch ( DBError $e ) {
+ $status->fatal( 'lockmanager-fail-db-release', $lockDb );
+ }
+ }
+ }
+
+ return $status;
+ }
+}
+
+/**
+ * PostgreSQL version of DBLockManager that supports shared locks.
+ * All locks are non-blocking, which avoids deadlocks.
+ *
+ * @ingroup LockManager
+ */
+class PostgreSqlLockManager extends DBLockManager {
+ /** @var Array Mapping of lock types to the type actually used */
+ protected $lockTypeMap = array(
+ self::LOCK_SH => self::LOCK_SH,
+ self::LOCK_UW => self::LOCK_SH,
+ self::LOCK_EX => self::LOCK_EX
+ );
+
+ protected function getLocksOnServer( $lockSrv, array $paths, $type ) {
+ $status = Status::newGood();
+ if ( !count( $paths ) ) {
+ return $status; // nothing to lock
+ }
+
+ $db = $this->getConnection( $lockSrv ); // checked in isServerUp()
+ $bigints = array_unique( array_map(
+ function( $key ) { return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 ); },
+ array_map( array( $this, 'sha1Base16Absolute' ), $paths )
+ ) );
+
+ // Try to acquire all the locks...
+ $fields = array();
+ foreach ( $bigints as $bigint ) {
+ $fields[] = ( $type == self::LOCK_SH )
+ ? "pg_try_advisory_lock_shared({$db->addQuotes( $bigint )}) AS K$bigint"
+ : "pg_try_advisory_lock({$db->addQuotes( $bigint )}) AS K$bigint";
+ }
+ $res = $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ );
+ $row = (array)$res->fetchObject();
+
+ if ( in_array( 'f', $row ) ) {
+ // Release any acquired locks if some could not be acquired...
+ $fields = array();
+ foreach ( $row as $kbigint => $ok ) {
+ if ( $ok === 't' ) { // locked
+ $bigint = substr( $kbigint, 1 ); // strip off the "K"
+ $fields[] = ( $type == self::LOCK_SH )
+ ? "pg_advisory_unlock_shared({$db->addQuotes( $bigint )})"
+ : "pg_advisory_unlock({$db->addQuotes( $bigint )})";
+ }
+ }
+ if ( count( $fields ) ) {
+ $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ );
+ }
+ foreach ( $paths as $path ) {
+ $status->fatal( 'lockmanager-fail-acquirelock', $path );
+ }
+ }
+
+ return $status;
+ }
+
+ /**
+ * @see QuorumLockManager::freeLocksOnServer()
+ * @return Status
+ */
+ protected function freeLocksOnServer( $lockSrv, array $paths, $type ) {
+ return Status::newGood(); // not supported
+ }
+
+ /**
+ * @see QuorumLockManager::releaseAllLocks()
+ * @return Status
+ */
+ protected function releaseAllLocks() {
+ $status = Status::newGood();
+
+ foreach ( $this->conns as $lockDb => $db ) {
+ try {
+ $db->query( "SELECT pg_advisory_unlock_all()", __METHOD__ );
+ } catch ( DBError $e ) {
+ $status->fatal( 'lockmanager-fail-db-release', $lockDb );
+ }
+ }
+
+ return $status;
+ }
}
diff --git a/includes/filebackend/lockmanager/FSLockManager.php b/includes/filebackend/lockmanager/FSLockManager.php
index 9a6206fd..eacba704 100644
--- a/includes/filebackend/lockmanager/FSLockManager.php
+++ b/includes/filebackend/lockmanager/FSLockManager.php
@@ -43,7 +43,7 @@ class FSLockManager extends LockManager {
protected $lockDir; // global dir for all servers
- /** @var Array Map of (locked key => lock type => lock file handle) */
+ /** @var Array Map of (locked key => lock file handle) */
protected $handles = array();
/**
@@ -115,12 +115,16 @@ class FSLockManager extends LockManager {
} elseif ( isset( $this->locksHeld[$path][self::LOCK_EX] ) ) {
$this->locksHeld[$path][$type] = 1;
} else {
- wfSuppressWarnings();
- $handle = fopen( $this->getLockPath( $path ), 'a+' );
- wfRestoreWarnings();
- if ( !$handle ) { // lock dir missing?
- wfMkdirParents( $this->lockDir );
- $handle = fopen( $this->getLockPath( $path ), 'a+' ); // try again
+ if ( isset( $this->handles[$path] ) ) {
+ $handle = $this->handles[$path];
+ } else {
+ wfSuppressWarnings();
+ $handle = fopen( $this->getLockPath( $path ), 'a+' );
+ wfRestoreWarnings();
+ if ( !$handle ) { // lock dir missing?
+ wfMkdirParents( $this->lockDir );
+ $handle = fopen( $this->getLockPath( $path ), 'a+' ); // try again
+ }
}
if ( $handle ) {
// Either a shared or exclusive lock
@@ -128,7 +132,7 @@ class FSLockManager extends LockManager {
if ( flock( $handle, $lock | LOCK_NB ) ) {
// Record this lock as active
$this->locksHeld[$path][$type] = 1;
- $this->handles[$path][$type] = $handle;
+ $this->handles[$path] = $handle;
} else {
fclose( $handle );
$status->fatal( 'lockmanager-fail-acquirelock', $path );
@@ -160,24 +164,13 @@ class FSLockManager extends LockManager {
--$this->locksHeld[$path][$type];
if ( $this->locksHeld[$path][$type] <= 0 ) {
unset( $this->locksHeld[$path][$type] );
- // If a LOCK_SH comes in while we have a LOCK_EX, we don't
- // actually add a handler, so check for handler existence.
- if ( isset( $this->handles[$path][$type] ) ) {
- if ( $type === self::LOCK_EX
- && isset( $this->locksHeld[$path][self::LOCK_SH] )
- && !isset( $this->handles[$path][self::LOCK_SH] ) )
- {
- // EX lock came first: move this handle to the SH one
- $this->handles[$path][self::LOCK_SH] = $this->handles[$path][$type];
- } else {
- // Mark this handle to be unlocked and closed
- $handlesToClose[] = $this->handles[$path][$type];
- }
- unset( $this->handles[$path][$type] );
- }
}
if ( !count( $this->locksHeld[$path] ) ) {
unset( $this->locksHeld[$path] ); // no locks on this path
+ if ( isset( $this->handles[$path] ) ) {
+ $handlesToClose[] = $this->handles[$path];
+ unset( $this->handles[$path] );
+ }
}
// Unlock handles to release locks and delete
// any lock files that end up with no locks on them...
@@ -237,8 +230,7 @@ class FSLockManager extends LockManager {
* @return string
*/
protected function getLockPath( $path ) {
- $hash = self::sha1Base36( $path );
- return "{$this->lockDir}/{$hash}.lock";
+ return "{$this->lockDir}/{$this->sha1Base36Absolute( $path )}.lock";
}
/**
diff --git a/includes/filebackend/lockmanager/LSLockManager.php b/includes/filebackend/lockmanager/LSLockManager.php
index 89428182..97de8dca 100644
--- a/includes/filebackend/lockmanager/LSLockManager.php
+++ b/includes/filebackend/lockmanager/LSLockManager.php
@@ -66,7 +66,7 @@ class LSLockManager extends QuorumLockManager {
* each having an odd-numbered list of server names (peers) as values.
* - connTimeout : Lock server connection attempt timeout. [optional]
*
- * @param Array $config
+ * @param array $config
*/
public function __construct( array $config ) {
parent::__construct( $config );
@@ -94,7 +94,7 @@ class LSLockManager extends QuorumLockManager {
// Send out the command and get the response...
$type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX';
- $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) );
+ $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) );
$response = $this->sendCommand( $lockSrv, 'ACQUIRE', $type, $keys );
if ( $response !== 'ACQUIRED' ) {
@@ -115,7 +115,7 @@ class LSLockManager extends QuorumLockManager {
// Send out the command and get the response...
$type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX';
- $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) );
+ $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) );
$response = $this->sendCommand( $lockSrv, 'RELEASE', $type, $keys );
if ( $response !== 'RELEASED' ) {
@@ -169,7 +169,7 @@ class LSLockManager extends QuorumLockManager {
$authKey = $this->lockServers[$lockSrv]['authKey'];
// Build of the command as a flat string...
$values = implode( '|', $values );
- $key = sha1( $this->session . $action . $type . $values . $authKey );
+ $key = hash_hmac( 'sha1', "{$this->session}\n{$action}\n{$type}\n{$values}", $authKey );
// Send out the command...
if ( fwrite( $conn, "{$this->session}:$key:$action:$type:$values\n" ) === false ) {
return false;
diff --git a/includes/filebackend/lockmanager/LockManager.php b/includes/filebackend/lockmanager/LockManager.php
index 07853f87..0512a01b 100644
--- a/includes/filebackend/lockmanager/LockManager.php
+++ b/includes/filebackend/lockmanager/LockManager.php
@@ -53,6 +53,9 @@ abstract class LockManager {
/** @var Array Map of (resource path => lock type => count) */
protected $locksHeld = array();
+ protected $domain; // string; domain (usually wiki ID)
+ protected $lockTTL; // integer; maximum time locks can be held
+
/* Lock types; stronger locks have higher values */
const LOCK_SH = 1; // shared lock (for reads)
const LOCK_UW = 2; // shared lock (for reads used to write elsewhere)
@@ -61,14 +64,29 @@ abstract class LockManager {
/**
* Construct a new instance from configuration
*
+ * $config paramaters include:
+ * - domain : Domain (usually wiki ID) that all resources are relative to [optional]
+ * - lockTTL : Age (in seconds) at which resource locks should expire.
+ * This only applies if locks are not tied to a connection/process.
+ *
* @param $config Array
*/
- public function __construct( array $config ) {}
+ public function __construct( array $config ) {
+ $this->domain = isset( $config['domain'] ) ? $config['domain'] : wfWikiID();
+ if ( isset( $config['lockTTL'] ) ) {
+ $this->lockTTL = max( 1, $config['lockTTL'] );
+ } elseif ( PHP_SAPI === 'cli' ) {
+ $this->lockTTL = 2*3600;
+ } else {
+ $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode
+ $this->lockTTL = max( 5*60, 2*(int)$met );
+ }
+ }
/**
* Lock the resources at the given abstract paths
*
- * @param $paths Array List of resource names
+ * @param array $paths List of resource names
* @param $type integer LockManager::LOCK_* constant
* @return Status
*/
@@ -82,7 +100,7 @@ abstract class LockManager {
/**
* Unlock the resources at the given abstract paths
*
- * @param $paths Array List of storage paths
+ * @param array $paths List of storage paths
* @param $type integer LockManager::LOCK_* constant
* @return Status
*/
@@ -94,308 +112,46 @@ abstract class LockManager {
}
/**
- * Get the base 36 SHA-1 of a string, padded to 31 digits
+ * Get the base 36 SHA-1 of a string, padded to 31 digits.
+ * Before hashing, the path will be prefixed with the domain ID.
+ * This should be used interally for lock key or file names.
*
* @param $path string
* @return string
*/
- final protected static function sha1Base36( $path ) {
- return wfBaseConvert( sha1( $path ), 16, 36, 31 );
+ final protected function sha1Base36Absolute( $path ) {
+ return wfBaseConvert( sha1( "{$this->domain}:{$path}" ), 16, 36, 31 );
}
/**
- * Lock resources with the given keys and lock type
+ * Get the base 16 SHA-1 of a string, padded to 31 digits.
+ * Before hashing, the path will be prefixed with the domain ID.
+ * This should be used interally for lock key or file names.
*
- * @param $paths Array List of storage paths
- * @param $type integer LockManager::LOCK_* constant
+ * @param $path string
* @return string
*/
- abstract protected function doLock( array $paths, $type );
+ final protected function sha1Base16Absolute( $path ) {
+ return sha1( "{$this->domain}:{$path}" );
+ }
/**
- * Unlock resources with the given keys and lock type
+ * Lock resources with the given keys and lock type
*
- * @param $paths Array List of storage paths
+ * @param array $paths List of storage paths
* @param $type integer LockManager::LOCK_* constant
* @return string
*/
- abstract protected function doUnlock( array $paths, $type );
-}
-
-/**
- * Self-releasing locks
- *
- * LockManager helper class to handle scoped locks, which
- * release when an object is destroyed or goes out of scope.
- *
- * @ingroup LockManager
- * @since 1.19
- */
-class ScopedLock {
- /** @var LockManager */
- protected $manager;
- /** @var Status */
- protected $status;
- /** @var Array List of resource paths*/
- protected $paths;
-
- protected $type; // integer lock type
-
- /**
- * @param $manager LockManager
- * @param $paths Array List of storage paths
- * @param $type integer LockManager::LOCK_* constant
- * @param $status Status
- */
- protected function __construct(
- LockManager $manager, array $paths, $type, Status $status
- ) {
- $this->manager = $manager;
- $this->paths = $paths;
- $this->status = $status;
- $this->type = $type;
- }
+ abstract protected function doLock( array $paths, $type );
/**
- * Get a ScopedLock object representing a lock on resource paths.
- * Any locks are released once this object goes out of scope.
- * The status object is updated with any errors or warnings.
+ * Unlock resources with the given keys and lock type
*
- * @param $manager LockManager
- * @param $paths Array List of storage paths
+ * @param array $paths List of storage paths
* @param $type integer LockManager::LOCK_* constant
- * @param $status Status
- * @return ScopedLock|null Returns null on failure
- */
- public static function factory(
- LockManager $manager, array $paths, $type, Status $status
- ) {
- $lockStatus = $manager->lock( $paths, $type );
- $status->merge( $lockStatus );
- if ( $lockStatus->isOK() ) {
- return new self( $manager, $paths, $type, $status );
- }
- return null;
- }
-
- function __destruct() {
- $wasOk = $this->status->isOK();
- $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) );
- if ( $wasOk ) {
- // Make sure status is OK, despite any unlockFiles() fatals
- $this->status->setResult( true, $this->status->value );
- }
- }
-}
-
-/**
- * Version of LockManager that uses a quorum from peer servers for locks.
- * The resource space can also be sharded into separate peer groups.
- *
- * @ingroup LockManager
- * @since 1.20
- */
-abstract class QuorumLockManager extends LockManager {
- /** @var Array Map of bucket indexes to peer server lists */
- protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...))
-
- /**
- * @see LockManager::doLock()
- * @param $paths array
- * @param $type int
- * @return Status
- */
- final protected function doLock( array $paths, $type ) {
- $status = Status::newGood();
-
- $pathsToLock = array(); // (bucket => paths)
- // Get locks that need to be acquired (buckets => locks)...
- foreach ( $paths as $path ) {
- if ( isset( $this->locksHeld[$path][$type] ) ) {
- ++$this->locksHeld[$path][$type];
- } elseif ( isset( $this->locksHeld[$path][self::LOCK_EX] ) ) {
- $this->locksHeld[$path][$type] = 1;
- } else {
- $bucket = $this->getBucketFromKey( $path );
- $pathsToLock[$bucket][] = $path;
- }
- }
-
- $lockedPaths = array(); // files locked in this attempt
- // Attempt to acquire these locks...
- foreach ( $pathsToLock as $bucket => $paths ) {
- // Try to acquire the locks for this bucket
- $status->merge( $this->doLockingRequestBucket( $bucket, $paths, $type ) );
- if ( !$status->isOK() ) {
- $status->merge( $this->doUnlock( $lockedPaths, $type ) );
- return $status;
- }
- // Record these locks as active
- foreach ( $paths as $path ) {
- $this->locksHeld[$path][$type] = 1; // locked
- }
- // Keep track of what locks were made in this attempt
- $lockedPaths = array_merge( $lockedPaths, $paths );
- }
-
- return $status;
- }
-
- /**
- * @see LockManager::doUnlock()
- * @param $paths array
- * @param $type int
- * @return Status
- */
- final protected function doUnlock( array $paths, $type ) {
- $status = Status::newGood();
-
- $pathsToUnlock = array();
- foreach ( $paths as $path ) {
- if ( !isset( $this->locksHeld[$path][$type] ) ) {
- $status->warning( 'lockmanager-notlocked', $path );
- } else {
- --$this->locksHeld[$path][$type];
- // Reference count the locks held and release locks when zero
- if ( $this->locksHeld[$path][$type] <= 0 ) {
- unset( $this->locksHeld[$path][$type] );
- $bucket = $this->getBucketFromKey( $path );
- $pathsToUnlock[$bucket][] = $path;
- }
- if ( !count( $this->locksHeld[$path] ) ) {
- unset( $this->locksHeld[$path] ); // no SH or EX locks left for key
- }
- }
- }
-
- // Remove these specific locks if possible, or at least release
- // all locks once this process is currently not holding any locks.
- foreach ( $pathsToUnlock as $bucket => $paths ) {
- $status->merge( $this->doUnlockingRequestBucket( $bucket, $paths, $type ) );
- }
- if ( !count( $this->locksHeld ) ) {
- $status->merge( $this->releaseAllLocks() );
- }
-
- return $status;
- }
-
- /**
- * Attempt to acquire locks with the peers for a bucket.
- * This is all or nothing; if any key is locked then this totally fails.
- *
- * @param $bucket integer
- * @param $paths Array List of resource keys to lock
- * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
- * @return Status
- */
- final protected function doLockingRequestBucket( $bucket, array $paths, $type ) {
- $status = Status::newGood();
-
- $yesVotes = 0; // locks made on trustable servers
- $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers
- $quorum = floor( $votesLeft/2 + 1 ); // simple majority
- // Get votes for each peer, in order, until we have enough...
- foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) {
- if ( !$this->isServerUp( $lockSrv ) ) {
- --$votesLeft;
- $status->warning( 'lockmanager-fail-svr-acquire', $lockSrv );
- continue; // server down?
- }
- // Attempt to acquire the lock on this peer
- $status->merge( $this->getLocksOnServer( $lockSrv, $paths, $type ) );
- if ( !$status->isOK() ) {
- return $status; // vetoed; resource locked
- }
- ++$yesVotes; // success for this peer
- if ( $yesVotes >= $quorum ) {
- return $status; // lock obtained
- }
- --$votesLeft;
- $votesNeeded = $quorum - $yesVotes;
- if ( $votesNeeded > $votesLeft ) {
- break; // short-circuit
- }
- }
- // At this point, we must not have met the quorum
- $status->setResult( false );
-
- return $status;
- }
-
- /**
- * Attempt to release locks with the peers for a bucket
- *
- * @param $bucket integer
- * @param $paths Array List of resource keys to lock
- * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
- * @return Status
- */
- final protected function doUnlockingRequestBucket( $bucket, array $paths, $type ) {
- $status = Status::newGood();
-
- foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) {
- if ( !$this->isServerUp( $lockSrv ) ) {
- $status->fatal( 'lockmanager-fail-svr-release', $lockSrv );
- // Attempt to release the lock on this peer
- } else {
- $status->merge( $this->freeLocksOnServer( $lockSrv, $paths, $type ) );
- }
- }
-
- return $status;
- }
-
- /**
- * Get the bucket for resource path.
- * This should avoid throwing any exceptions.
- *
- * @param $path string
- * @return integer
- */
- protected function getBucketFromKey( $path ) {
- $prefix = substr( sha1( $path ), 0, 2 ); // first 2 hex chars (8 bits)
- return (int)base_convert( $prefix, 16, 10 ) % count( $this->srvsByBucket );
- }
-
- /**
- * Check if a lock server is up
- *
- * @param $lockSrv string
- * @return bool
- */
- abstract protected function isServerUp( $lockSrv );
-
- /**
- * Get a connection to a lock server and acquire locks on $paths
- *
- * @param $lockSrv string
- * @param $paths array
- * @param $type integer
- * @return Status
- */
- abstract protected function getLocksOnServer( $lockSrv, array $paths, $type );
-
- /**
- * Get a connection to a lock server and release locks on $paths.
- *
- * Subclasses must effectively implement this or releaseAllLocks().
- *
- * @param $lockSrv string
- * @param $paths array
- * @param $type integer
- * @return Status
- */
- abstract protected function freeLocksOnServer( $lockSrv, array $paths, $type );
-
- /**
- * Release all locks that this session is holding.
- *
- * Subclasses must effectively implement this or freeLocksOnServer().
- *
- * @return Status
+ * @return string
*/
- abstract protected function releaseAllLocks();
+ abstract protected function doUnlock( array $paths, $type );
}
/**
diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php
index 8c8c940a..ac0bd49b 100644
--- a/includes/filebackend/lockmanager/LockManagerGroup.php
+++ b/includes/filebackend/lockmanager/LockManagerGroup.php
@@ -29,33 +29,41 @@
* @since 1.19
*/
class LockManagerGroup {
- /**
- * @var LockManagerGroup
- */
- protected static $instance = null;
+ /** @var Array (domain => LockManager) */
+ protected static $instances = array();
+
+ protected $domain; // string; domain (usually wiki ID)
- /** @var Array of (name => ('class' =>, 'config' =>, 'instance' =>)) */
+ /** @var Array of (name => ('class' => ..., 'config' => ..., 'instance' => ...)) */
protected $managers = array();
- protected function __construct() {}
+ /**
+ * @param string $domain Domain (usually wiki ID)
+ */
+ protected function __construct( $domain ) {
+ $this->domain = $domain;
+ }
/**
+ * @param string $domain Domain (usually wiki ID)
* @return LockManagerGroup
*/
- public static function singleton() {
- if ( self::$instance == null ) {
- self::$instance = new self();
- self::$instance->initFromGlobals();
+ public static function singleton( $domain = false ) {
+ $domain = ( $domain === false ) ? wfWikiID() : $domain;
+ if ( !isset( self::$instances[$domain] ) ) {
+ self::$instances[$domain] = new self( $domain );
+ self::$instances[$domain]->initFromGlobals();
}
- return self::$instance;
+ return self::$instances[$domain];
}
/**
- * Destroy the singleton instance, so that a new one will be created next
- * time singleton() is called.
+ * Destroy the singleton instances
+ *
+ * @return void
*/
- public static function destroySingleton() {
- self::$instance = null;
+ public static function destroySingletons() {
+ self::$instances = array();
}
/**
@@ -78,6 +86,7 @@ class LockManagerGroup {
*/
protected function register( array $configs ) {
foreach ( $configs as $config ) {
+ $config['domain'] = $this->domain;
if ( !isset( $config['name'] ) ) {
throw new MWException( "Cannot register a lock manager with no name." );
}
@@ -116,6 +125,21 @@ class LockManagerGroup {
}
/**
+ * Get the config array for a lock manager object with a given name
+ *
+ * @param $name string
+ * @return Array
+ * @throws MWException
+ */
+ public function config( $name ) {
+ if ( !isset( $this->managers[$name] ) ) {
+ throw new MWException( "No lock manager defined with the name `$name`." );
+ }
+ $class = $this->managers[$name]['class'];
+ return array( 'class' => $class ) + $this->managers[$name]['config'];
+ }
+
+ /**
* Get the default lock manager configured for the site.
* Returns NullLockManager if no lock manager could be found.
*
diff --git a/includes/filebackend/lockmanager/MemcLockManager.php b/includes/filebackend/lockmanager/MemcLockManager.php
index 57c0463d..fafc588a 100644
--- a/includes/filebackend/lockmanager/MemcLockManager.php
+++ b/includes/filebackend/lockmanager/MemcLockManager.php
@@ -28,8 +28,8 @@
* This is meant for multi-wiki systems that may share files.
* All locks are non-blocking, which avoids deadlocks.
*
- * All lock requests for a resource, identified by a hash string, will map
- * to one bucket. Each bucket maps to one or several peer servers, each running memcached.
+ * All lock requests for a resource, identified by a hash string, will map to one
+ * bucket. Each bucket maps to one or several peer servers, each running memcached.
* A majority of peers must agree for a lock to be acquired.
*
* @ingroup LockManager
@@ -48,9 +48,7 @@ class MemcLockManager extends QuorumLockManager {
/** @var Array */
protected $serversUp = array(); // (server name => bool)
- protected $lockExpiry; // integer; maximum time locks can be held
- protected $session = ''; // string; random SHA-1 UUID
- protected $wikiId = ''; // string
+ protected $session = ''; // string; random UUID
/**
* Construct a new instance from configuration.
@@ -61,9 +59,9 @@ class MemcLockManager extends QuorumLockManager {
* each having an odd-numbered list of server names (peers) as values.
* - memcConfig : Configuration array for ObjectCache::newFromParams. [optional]
* If set, this must use one of the memcached classes.
- * - wikiId : Wiki ID string that all resources are relative to. [optional]
*
- * @param Array $config
+ * @param array $config
+ * @throws MWException
*/
public function __construct( array $config ) {
parent::__construct( $config );
@@ -87,11 +85,6 @@ class MemcLockManager extends QuorumLockManager {
}
}
- $this->wikiId = isset( $config['wikiId'] ) ? $config['wikiId'] : wfWikiID();
-
- $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode
- $this->lockExpiry = $met ? 2*(int)$met : 2*3600;
-
$this->session = wfRandomString( 32 );
}
@@ -110,7 +103,7 @@ class MemcLockManager extends QuorumLockManager {
foreach ( $paths as $path ) {
$status->fatal( 'lockmanager-fail-acquirelock', $path );
}
- return;
+ return $status;
}
// Fetch all the existing lock records...
@@ -121,8 +114,8 @@ class MemcLockManager extends QuorumLockManager {
foreach ( $paths as $path ) {
$locksKey = $this->recordKeyForPath( $path );
$locksHeld = isset( $lockRecords[$locksKey] )
- ? $lockRecords[$locksKey]
- : array( self::LOCK_SH => array(), self::LOCK_EX => array() ); // init
+ ? self::sanitizeLockArray( $lockRecords[$locksKey] )
+ : self::newLockArray(); // init
foreach ( $locksHeld[self::LOCK_EX] as $session => $expiry ) {
if ( $expiry < $now ) { // stale?
unset( $locksHeld[self::LOCK_EX][$session] );
@@ -141,7 +134,7 @@ class MemcLockManager extends QuorumLockManager {
}
if ( $status->isOK() ) {
// Register the session in the lock record array
- $locksHeld[$type][$this->session] = $now + $this->lockExpiry;
+ $locksHeld[$type][$this->session] = $now + $this->lockTTL;
// We will update this record if none of the other locks conflict
$lockRecords[$locksKey] = $locksHeld;
}
@@ -149,9 +142,15 @@ class MemcLockManager extends QuorumLockManager {
// If there were no lock conflicts, update all the lock records...
if ( $status->isOK() ) {
- foreach ( $lockRecords as $locksKey => $locksHeld ) {
- $memc->set( $locksKey, $locksHeld );
- wfDebug( __METHOD__ . ": acquired lock on key $locksKey.\n" );
+ foreach ( $paths as $path ) {
+ $locksKey = $this->recordKeyForPath( $path );
+ $locksHeld = $lockRecords[$locksKey];
+ $ok = $memc->set( $locksKey, $locksHeld, 7*86400 );
+ if ( !$ok ) {
+ $status->fatal( 'lockmanager-fail-acquirelock', $path );
+ } else {
+ wfDebug( __METHOD__ . ": acquired lock on key $locksKey.\n" );
+ }
}
}
@@ -186,17 +185,22 @@ class MemcLockManager extends QuorumLockManager {
foreach ( $paths as $path ) {
$locksKey = $this->recordKeyForPath( $path ); // lock record
if ( !isset( $lockRecords[$locksKey] ) ) {
+ $status->warning( 'lockmanager-fail-releaselock', $path );
continue; // nothing to do
}
- $locksHeld = $lockRecords[$locksKey];
- if ( is_array( $locksHeld ) && isset( $locksHeld[$type] ) ) {
- unset( $locksHeld[$type][$this->session] );
- $ok = $memc->set( $locksKey, $locksHeld );
+ $locksHeld = self::sanitizeLockArray( $lockRecords[$locksKey] );
+ if ( isset( $locksHeld[$type][$this->session] ) ) {
+ unset( $locksHeld[$type][$this->session] ); // unregister this session
+ if ( $locksHeld === self::newLockArray() ) {
+ $ok = $memc->delete( $locksKey );
+ } else {
+ $ok = $memc->set( $locksKey, $locksHeld );
+ }
+ if ( !$ok ) {
+ $status->fatal( 'lockmanager-fail-releaselock', $path );
+ }
} else {
- $ok = true;
- }
- if ( !$ok ) {
- $status->fatal( 'lockmanager-fail-releaselock', $path );
+ $status->warning( 'lockmanager-fail-releaselock', $path );
}
wfDebug( __METHOD__ . ": released lock on key $locksKey.\n" );
}
@@ -226,7 +230,7 @@ class MemcLockManager extends QuorumLockManager {
/**
* Get the MemcachedBagOStuff object for a $lockSrv
*
- * @param $lockSrv string Server name
+ * @param string $lockSrv Server name
* @return MemcachedBagOStuff|null
*/
protected function getCache( $lockSrv ) {
@@ -234,7 +238,7 @@ class MemcLockManager extends QuorumLockManager {
if ( isset( $this->bagOStuffs[$lockSrv] ) ) {
$memc = $this->bagOStuffs[$lockSrv];
if ( !isset( $this->serversUp[$lockSrv] ) ) {
- $this->serversUp[$lockSrv] = $memc->set( 'MemcLockManager:ping', 1, 1 );
+ $this->serversUp[$lockSrv] = $memc->set( __CLASS__ . ':ping', 1, 1 );
if ( !$this->serversUp[$lockSrv] ) {
trigger_error( __METHOD__ . ": Could not contact $lockSrv.", E_USER_WARNING );
}
@@ -251,14 +255,32 @@ class MemcLockManager extends QuorumLockManager {
* @return string
*/
protected function recordKeyForPath( $path ) {
- $hash = LockManager::sha1Base36( $path );
- list( $db, $prefix ) = wfSplitWikiID( $this->wikiId );
- return wfForeignMemcKey( $db, $prefix, __CLASS__, 'locks', $hash );
+ return implode( ':', array( __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ) );
+ }
+
+ /**
+ * @return Array An empty lock structure for a key
+ */
+ protected static function newLockArray() {
+ return array( self::LOCK_SH => array(), self::LOCK_EX => array() );
+ }
+
+ /**
+ * @param $a array
+ * @return Array An empty lock structure for a key
+ */
+ protected static function sanitizeLockArray( $a ) {
+ if ( is_array( $a ) && isset( $a[self::LOCK_EX] ) && isset( $a[self::LOCK_SH] ) ) {
+ return $a;
+ } else {
+ trigger_error( __METHOD__ . ": reset invalid lock array.", E_USER_WARNING );
+ return self::newLockArray();
+ }
}
/**
* @param $memc MemcachedBagOStuff
- * @param $keys Array List of keys to acquire
+ * @param array $keys List of keys to acquire
* @return bool
*/
protected function acquireMutexes( MemcachedBagOStuff $memc, array $keys ) {
@@ -284,10 +306,10 @@ class MemcLockManager extends QuorumLockManager {
continue; // acquire in order
}
}
- } while ( count( $lockedKeys ) < count( $keys ) && ( microtime( true ) - $start ) <= 6 );
+ } while ( count( $lockedKeys ) < count( $keys ) && ( microtime( true ) - $start ) <= 3 );
if ( count( $lockedKeys ) != count( $keys ) ) {
- $this->releaseMutexes( $lockedKeys ); // failed; release what was locked
+ $this->releaseMutexes( $memc, $lockedKeys ); // failed; release what was locked
return false;
}
@@ -296,7 +318,7 @@ class MemcLockManager extends QuorumLockManager {
/**
* @param $memc MemcachedBagOStuff
- * @param $keys Array List of acquired keys
+ * @param array $keys List of acquired keys
* @return void
*/
protected function releaseMutexes( MemcachedBagOStuff $memc, array $keys ) {
diff --git a/includes/filebackend/lockmanager/QuorumLockManager.php b/includes/filebackend/lockmanager/QuorumLockManager.php
new file mode 100644
index 00000000..b331b540
--- /dev/null
+++ b/includes/filebackend/lockmanager/QuorumLockManager.php
@@ -0,0 +1,230 @@
+<?php
+/**
+ * Version of LockManager that uses a quorum from peer servers for locks.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup LockManager
+ */
+
+/**
+ * Version of LockManager that uses a quorum from peer servers for locks.
+ * The resource space can also be sharded into separate peer groups.
+ *
+ * @ingroup LockManager
+ * @since 1.20
+ */
+abstract class QuorumLockManager extends LockManager {
+ /** @var Array Map of bucket indexes to peer server lists */
+ protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...))
+
+ /**
+ * @see LockManager::doLock()
+ * @param $paths array
+ * @param $type int
+ * @return Status
+ */
+ final protected function doLock( array $paths, $type ) {
+ $status = Status::newGood();
+
+ $pathsToLock = array(); // (bucket => paths)
+ // Get locks that need to be acquired (buckets => locks)...
+ foreach ( $paths as $path ) {
+ if ( isset( $this->locksHeld[$path][$type] ) ) {
+ ++$this->locksHeld[$path][$type];
+ } else {
+ $bucket = $this->getBucketFromPath( $path );
+ $pathsToLock[$bucket][] = $path;
+ }
+ }
+
+ $lockedPaths = array(); // files locked in this attempt
+ // Attempt to acquire these locks...
+ foreach ( $pathsToLock as $bucket => $paths ) {
+ // Try to acquire the locks for this bucket
+ $status->merge( $this->doLockingRequestBucket( $bucket, $paths, $type ) );
+ if ( !$status->isOK() ) {
+ $status->merge( $this->doUnlock( $lockedPaths, $type ) );
+ return $status;
+ }
+ // Record these locks as active
+ foreach ( $paths as $path ) {
+ $this->locksHeld[$path][$type] = 1; // locked
+ }
+ // Keep track of what locks were made in this attempt
+ $lockedPaths = array_merge( $lockedPaths, $paths );
+ }
+
+ return $status;
+ }
+
+ /**
+ * @see LockManager::doUnlock()
+ * @param $paths array
+ * @param $type int
+ * @return Status
+ */
+ final protected function doUnlock( array $paths, $type ) {
+ $status = Status::newGood();
+
+ $pathsToUnlock = array();
+ foreach ( $paths as $path ) {
+ if ( !isset( $this->locksHeld[$path][$type] ) ) {
+ $status->warning( 'lockmanager-notlocked', $path );
+ } else {
+ --$this->locksHeld[$path][$type];
+ // Reference count the locks held and release locks when zero
+ if ( $this->locksHeld[$path][$type] <= 0 ) {
+ unset( $this->locksHeld[$path][$type] );
+ $bucket = $this->getBucketFromPath( $path );
+ $pathsToUnlock[$bucket][] = $path;
+ }
+ if ( !count( $this->locksHeld[$path] ) ) {
+ unset( $this->locksHeld[$path] ); // no SH or EX locks left for key
+ }
+ }
+ }
+
+ // Remove these specific locks if possible, or at least release
+ // all locks once this process is currently not holding any locks.
+ foreach ( $pathsToUnlock as $bucket => $paths ) {
+ $status->merge( $this->doUnlockingRequestBucket( $bucket, $paths, $type ) );
+ }
+ if ( !count( $this->locksHeld ) ) {
+ $status->merge( $this->releaseAllLocks() );
+ }
+
+ return $status;
+ }
+
+ /**
+ * Attempt to acquire locks with the peers for a bucket.
+ * This is all or nothing; if any key is locked then this totally fails.
+ *
+ * @param $bucket integer
+ * @param array $paths List of resource keys to lock
+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @return Status
+ */
+ final protected function doLockingRequestBucket( $bucket, array $paths, $type ) {
+ $status = Status::newGood();
+
+ $yesVotes = 0; // locks made on trustable servers
+ $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers
+ $quorum = floor( $votesLeft/2 + 1 ); // simple majority
+ // Get votes for each peer, in order, until we have enough...
+ foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) {
+ if ( !$this->isServerUp( $lockSrv ) ) {
+ --$votesLeft;
+ $status->warning( 'lockmanager-fail-svr-acquire', $lockSrv );
+ continue; // server down?
+ }
+ // Attempt to acquire the lock on this peer
+ $status->merge( $this->getLocksOnServer( $lockSrv, $paths, $type ) );
+ if ( !$status->isOK() ) {
+ return $status; // vetoed; resource locked
+ }
+ ++$yesVotes; // success for this peer
+ if ( $yesVotes >= $quorum ) {
+ return $status; // lock obtained
+ }
+ --$votesLeft;
+ $votesNeeded = $quorum - $yesVotes;
+ if ( $votesNeeded > $votesLeft ) {
+ break; // short-circuit
+ }
+ }
+ // At this point, we must not have met the quorum
+ $status->setResult( false );
+
+ return $status;
+ }
+
+ /**
+ * Attempt to release locks with the peers for a bucket
+ *
+ * @param $bucket integer
+ * @param array $paths List of resource keys to lock
+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @return Status
+ */
+ final protected function doUnlockingRequestBucket( $bucket, array $paths, $type ) {
+ $status = Status::newGood();
+
+ foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) {
+ if ( !$this->isServerUp( $lockSrv ) ) {
+ $status->fatal( 'lockmanager-fail-svr-release', $lockSrv );
+ // Attempt to release the lock on this peer
+ } else {
+ $status->merge( $this->freeLocksOnServer( $lockSrv, $paths, $type ) );
+ }
+ }
+
+ return $status;
+ }
+
+ /**
+ * Get the bucket for resource path.
+ * This should avoid throwing any exceptions.
+ *
+ * @param $path string
+ * @return integer
+ */
+ protected function getBucketFromPath( $path ) {
+ $prefix = substr( sha1( $path ), 0, 2 ); // first 2 hex chars (8 bits)
+ return (int)base_convert( $prefix, 16, 10 ) % count( $this->srvsByBucket );
+ }
+
+ /**
+ * Check if a lock server is up
+ *
+ * @param $lockSrv string
+ * @return bool
+ */
+ abstract protected function isServerUp( $lockSrv );
+
+ /**
+ * Get a connection to a lock server and acquire locks on $paths
+ *
+ * @param $lockSrv string
+ * @param $paths array
+ * @param $type integer
+ * @return Status
+ */
+ abstract protected function getLocksOnServer( $lockSrv, array $paths, $type );
+
+ /**
+ * Get a connection to a lock server and release locks on $paths.
+ *
+ * Subclasses must effectively implement this or releaseAllLocks().
+ *
+ * @param $lockSrv string
+ * @param $paths array
+ * @param $type integer
+ * @return Status
+ */
+ abstract protected function freeLocksOnServer( $lockSrv, array $paths, $type );
+
+ /**
+ * Release all locks that this session is holding.
+ *
+ * Subclasses must effectively implement this or freeLocksOnServer().
+ *
+ * @return Status
+ */
+ abstract protected function releaseAllLocks();
+}
diff --git a/includes/filebackend/lockmanager/ScopedLock.php b/includes/filebackend/lockmanager/ScopedLock.php
new file mode 100644
index 00000000..edcb1d65
--- /dev/null
+++ b/includes/filebackend/lockmanager/ScopedLock.php
@@ -0,0 +1,102 @@
+<?php
+/**
+ * Resource locking handling.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup LockManager
+ * @author Aaron Schulz
+ */
+
+/**
+ * Self-releasing locks
+ *
+ * LockManager helper class to handle scoped locks, which
+ * release when an object is destroyed or goes out of scope.
+ *
+ * @ingroup LockManager
+ * @since 1.19
+ */
+class ScopedLock {
+ /** @var LockManager */
+ protected $manager;
+ /** @var Status */
+ protected $status;
+ /** @var Array List of resource paths*/
+ protected $paths;
+
+ protected $type; // integer lock type
+
+ /**
+ * @param $manager LockManager
+ * @param array $paths List of storage paths
+ * @param $type integer LockManager::LOCK_* constant
+ * @param $status Status
+ */
+ protected function __construct(
+ LockManager $manager, array $paths, $type, Status $status
+ ) {
+ $this->manager = $manager;
+ $this->paths = $paths;
+ $this->status = $status;
+ $this->type = $type;
+ }
+
+ /**
+ * Get a ScopedLock object representing a lock on resource paths.
+ * Any locks are released once this object goes out of scope.
+ * The status object is updated with any errors or warnings.
+ *
+ * @param $manager LockManager
+ * @param array $paths List of storage paths
+ * @param $type integer LockManager::LOCK_* constant
+ * @param $status Status
+ * @return ScopedLock|null Returns null on failure
+ */
+ public static function factory(
+ LockManager $manager, array $paths, $type, Status $status
+ ) {
+ $lockStatus = $manager->lock( $paths, $type );
+ $status->merge( $lockStatus );
+ if ( $lockStatus->isOK() ) {
+ return new self( $manager, $paths, $type, $status );
+ }
+ return null;
+ }
+
+ /**
+ * Release a scoped lock and set any errors in the attatched Status object.
+ * This is useful for early release of locks before function scope is destroyed.
+ * This is the same as setting the lock object to null.
+ *
+ * @param ScopedLock $lock
+ * @return void
+ * @since 1.21
+ */
+ public static function release( ScopedLock &$lock = null ) {
+ $lock = null;
+ }
+
+ function __destruct() {
+ $wasOk = $this->status->isOK();
+ $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) );
+ if ( $wasOk ) {
+ // Make sure status is OK, despite any unlockFiles() fatals
+ $this->status->setResult( true, $this->status->value );
+ }
+ }
+}