summaryrefslogtreecommitdiff
path: root/includes/filebackend
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2013-12-08 09:55:49 +0100
committerPierre Schmitz <pierre@archlinux.de>2013-12-08 09:55:49 +0100
commit4ac9fa081a7c045f6a9f1cfc529d82423f485b2e (patch)
treeaf68743f2f4a47d13f2b0eb05f5c4aaf86d8ea37 /includes/filebackend
parentaf4da56f1ad4d3ef7b06557bae365da2ea27a897 (diff)
Update to MediaWiki 1.22.0
Diffstat (limited to 'includes/filebackend')
-rw-r--r--includes/filebackend/FSFile.php29
-rw-r--r--includes/filebackend/FSFileBackend.php117
-rw-r--r--includes/filebackend/FileBackend.php143
-rw-r--r--includes/filebackend/FileBackendGroup.php28
-rw-r--r--includes/filebackend/FileBackendMultiWrite.php179
-rw-r--r--includes/filebackend/FileBackendStore.php576
-rw-r--r--includes/filebackend/FileOp.php193
-rw-r--r--includes/filebackend/FileOpBatch.php6
-rw-r--r--includes/filebackend/README2
-rw-r--r--includes/filebackend/SwiftFileBackend.php262
-rw-r--r--includes/filebackend/TempFSFile.php6
-rw-r--r--includes/filebackend/filejournal/DBFileJournal.php10
-rw-r--r--includes/filebackend/lockmanager/DBLockManager.php37
-rw-r--r--includes/filebackend/lockmanager/LockManager.php125
-rw-r--r--includes/filebackend/lockmanager/LockManagerGroup.php4
-rw-r--r--includes/filebackend/lockmanager/MemcLockManager.php41
-rw-r--r--includes/filebackend/lockmanager/QuorumLockManager.php140
-rw-r--r--includes/filebackend/lockmanager/RedisLockManager.php288
-rw-r--r--includes/filebackend/lockmanager/ScopedLock.php44
19 files changed, 1106 insertions, 1124 deletions
diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php
index 7d0dbd52..8f0a1334 100644
--- a/includes/filebackend/FSFile.php
+++ b/includes/filebackend/FSFile.php
@@ -28,7 +28,7 @@
*/
class FSFile {
protected $path; // path to file
- private $sha1Base36 = null; // File Sha1Base36
+ protected $sha1Base36; // file SHA-1 in base 36
/**
* Sets up the file object
@@ -98,7 +98,7 @@ class FSFile {
* Get an associative array containing information about
* a file with the given storage path.
*
- * @param $ext Mixed: the file extension, or true to extract it from the filename.
+ * @param Mixed $ext: the file extension, or true to extract it from the filename.
* Set it to false to ignore the extension.
*
* @return array
@@ -171,7 +171,7 @@ class FSFile {
/**
* Exract image size information
*
- * @param $gis array
+ * @param array $gis
* @return Array
*/
protected function extractImageSizeInfo( array $gis ) {
@@ -194,7 +194,7 @@ class FSFile {
* 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36
* fairly neatly.
*
- * @param $recache bool
+ * @param bool $recache
* @return bool|string False on failure
*/
public function getSha1Base36( $recache = false ) {
@@ -220,7 +220,7 @@ class FSFile {
/**
* Get the final file extension from a file system path
*
- * @param $path string
+ * @param string $path
* @return string
*/
public static function extensionFromPath( $path ) {
@@ -232,9 +232,8 @@ class FSFile {
* Get an associative array containing information about a file in the local filesystem.
*
* @param string $path absolute local filesystem path
- * @param $ext Mixed: the file extension, or true to extract it from the filename.
+ * @param Mixed $ext: the file extension, or true to extract it from the filename.
* Set it to false to ignore the extension.
- *
* @return array
*/
public static function getPropsFromPath( $path, $ext = true ) {
@@ -249,19 +248,11 @@ class FSFile {
* 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36
* fairly neatly.
*
- * @param $path string
- * @param $recache bool
- *
+ * @param string $path
* @return bool|string False on failure
*/
- public static function getSha1Base36FromPath( $path, $recache = false ) {
- static $sha1Base36 = array();
-
- if ( !isset( $sha1Base36[$path] ) || $recache ) {
- $fsFile = new self( $path );
- $sha1Base36[$path] = $fsFile->getSha1Base36();
- }
-
- return $sha1Base36[$path];
+ public static function getSha1Base36FromPath( $path ) {
+ $fsFile = new self( $path );
+ return $fsFile->getSha1Base36();
}
}
diff --git a/includes/filebackend/FSFileBackend.php b/includes/filebackend/FSFileBackend.php
index c9769989..6d642162 100644
--- a/includes/filebackend/FSFileBackend.php
+++ b/includes/filebackend/FSFileBackend.php
@@ -82,12 +82,6 @@ class FSFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::resolveContainerPath()
- * @param $container string
- * @param $relStoragePath string
- * @return null|string
- */
protected function resolveContainerPath( $container, $relStoragePath ) {
// Check that container has a root directory
if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) {
@@ -121,8 +115,8 @@ class FSFileBackend extends FileBackendStore {
* Given the short (unresolved) and full (resolved) name of
* a container, return the file system path of the container.
*
- * @param $shortCont string
- * @param $fullCont string
+ * @param string $shortCont
+ * @param string $fullCont
* @return string|null
*/
protected function containerFSRoot( $shortCont, $fullCont ) {
@@ -153,10 +147,6 @@ class FSFileBackend extends FileBackendStore {
return $fsPath;
}
- /**
- * @see FileBackendStore::isPathUsableInternal()
- * @return bool
- */
public function isPathUsableInternal( $storagePath ) {
$fsPath = $this->resolveToFSPath( $storagePath );
if ( $fsPath === null ) {
@@ -178,10 +168,6 @@ class FSFileBackend extends FileBackendStore {
return $ok;
}
- /**
- * @see FileBackendStore::doCreateInternal()
- * @return Status
- */
protected function doCreateInternal( array $params ) {
$status = Status::newGood();
@@ -235,10 +221,6 @@ class FSFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doStoreInternal()
- * @return Status
- */
protected function doStoreInternal( array $params ) {
$status = Status::newGood();
@@ -284,10 +266,6 @@ class FSFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doCopyInternal()
- * @return Status
- */
protected function doCopyInternal( array $params ) {
$status = Status::newGood();
@@ -319,7 +297,7 @@ class FSFileBackend extends FileBackendStore {
$status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd, $dest );
} else { // immediate write
$this->trapWarnings();
- $ok = copy( $source, $dest );
+ $ok = ( $source === $dest ) ? true : copy( $source, $dest );
$this->untrapWarnings();
// In some cases (at least over NFS), copy() returns true when it fails
if ( !$ok || ( filesize( $source ) !== filesize( $dest ) ) ) {
@@ -348,10 +326,6 @@ class FSFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doMoveInternal()
- * @return Status
- */
protected function doMoveInternal( array $params ) {
$status = Status::newGood();
@@ -383,7 +357,7 @@ class FSFileBackend extends FileBackendStore {
$status->value = new FSFileOpHandle( $this, $params, 'Move', $cmd );
} else { // immediate write
$this->trapWarnings();
- $ok = rename( $source, $dest );
+ $ok = ( $source === $dest ) ? true : rename( $source, $dest );
$this->untrapWarnings();
clearstatcache(); // file no longer at source
if ( !$ok ) {
@@ -405,10 +379,6 @@ class FSFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doDeleteInternal()
- * @return Status
- */
protected function doDeleteInternal( array $params ) {
$status = Status::newGood();
@@ -454,10 +424,6 @@ class FSFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doPrepareInternal()
- * @return Status
- */
protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
@@ -481,10 +447,6 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doSecureInternal()
- * @return Status
- */
protected function doSecureInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
@@ -512,10 +474,6 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doPublishInternal()
- * @return Status
- */
protected function doPublishInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
@@ -543,10 +501,6 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doCleanInternal()
- * @return Status
- */
protected function doCleanInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
@@ -560,10 +514,6 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doFileExists()
- * @return array|bool|null
- */
protected function doGetFileStat( array $params ) {
$source = $this->resolveToFSPath( $params['src'] );
if ( $source === null ) {
@@ -577,7 +527,7 @@ class FSFileBackend extends FileBackendStore {
if ( $stat ) {
return array(
'mtime' => wfTimestamp( TS_MW, $stat['mtime'] ),
- 'size' => $stat['size']
+ 'size' => $stat['size']
);
} elseif ( !$hadError ) {
return false; // file does not exist
@@ -593,10 +543,6 @@ class FSFileBackend extends FileBackendStore {
clearstatcache(); // clear the PHP file stat cache
}
- /**
- * @see FileBackendStore::doDirectoryExists()
- * @return bool|null
- */
protected function doDirectoryExists( $fullCont, $dirRel, array $params ) {
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
$contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
@@ -647,10 +593,6 @@ class FSFileBackend extends FileBackendStore {
return new FSFileBackendFileList( $dir, $params );
}
- /**
- * @see FileBackendStore::doGetLocalReferenceMulti()
- * @return Array
- */
protected function doGetLocalReferenceMulti( array $params ) {
$fsFiles = array(); // (path => FSFile)
@@ -666,10 +608,6 @@ class FSFileBackend extends FileBackendStore {
return $fsFiles;
}
- /**
- * @see FileBackendStore::doGetLocalCopyMulti()
- * @return Array
- */
protected function doGetLocalCopyMulti( array $params ) {
$tmpFiles = array(); // (path => TempFSFile)
@@ -702,18 +640,10 @@ class FSFileBackend extends FileBackendStore {
return $tmpFiles;
}
- /**
- * @see FileBackendStore::directoriesAreVirtual()
- * @return bool
- */
protected function directoriesAreVirtual() {
return false;
}
- /**
- * @see FileBackendStore::doExecuteOpHandlesInternal()
- * @return Array List of corresponding Status objects
- */
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
$statuses = array();
@@ -807,11 +737,12 @@ class FSFileBackend extends FileBackendStore {
}
/**
- * @param $errno integer
- * @param $errstr string
+ * @param integer $errno
+ * @param string $errstr
* @return bool
+ * @access private
*/
- private function handleWarning( $errno, $errstr ) {
+ public function handleWarning( $errno, $errstr ) {
wfDebugLog( 'FSFileBackend', $errstr ); // more detailed error logging
$this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true;
return true; // suppress from PHP handler
@@ -826,13 +757,15 @@ class FSFileOpHandle extends FileBackendStoreOpHandle {
public $chmodPath; // string; file to chmod
/**
- * @param $backend
- * @param $params array
- * @param $call
- * @param $cmd
- * @param $chmodPath null
+ * @param FSFileBackend $backend
+ * @param array $params
+ * @param string $call
+ * @param string $cmd
+ * @param integer|null $chmodPath
*/
- public function __construct( $backend, array $params, $call, $cmd, $chmodPath = null ) {
+ public function __construct(
+ FSFileBackend $backend, array $params, $call, $cmd, $chmodPath = null
+ ) {
$this->backend = $backend;
$this->params = $params;
$this->call = $call;
@@ -858,11 +791,11 @@ abstract class FSFileBackendList implements Iterator {
/**
* @param string $dir file system directory
- * @param $params array
+ * @param array $params
*/
public function __construct( $dir, array $params ) {
$path = realpath( $dir ); // normalize
- if( $path === false ) {
+ if ( $path === false ) {
$path = $dir;
}
$this->suffixStart = strlen( $path ) + 1; // size of "path/to/dir/"
@@ -921,8 +854,8 @@ abstract class FSFileBackendList implements Iterator {
try {
$this->iter->next();
$this->filterViaNext();
- } catch ( UnexpectedValueException $e ) {
- $this->iter = null;
+ } catch ( UnexpectedValueException $e ) { // bad permissions? deleted?
+ throw new FileBackendError( "File iterator gave UnexpectedValueException." );
}
++$this->pos;
}
@@ -936,8 +869,8 @@ abstract class FSFileBackendList implements Iterator {
try {
$this->iter->rewind();
$this->filterViaNext();
- } catch ( UnexpectedValueException $e ) {
- $this->iter = null;
+ } catch ( UnexpectedValueException $e ) { // bad permissions? deleted?
+ throw new FileBackendError( "File iterator gave UnexpectedValueException." );
}
}
@@ -958,12 +891,12 @@ abstract class FSFileBackendList implements Iterator {
* Return only the relative path and normalize slashes to FileBackend-style.
* Uses the "real path" since the suffix is based upon that.
*
- * @param $path string
+ * @param string $path
* @return string
*/
protected function getRelPath( $dir ) {
$path = realpath( $dir );
- if( $path === false ) {
+ if ( $path === false ) {
$path = $dir;
}
return strtr( substr( $path, $this->suffixStart ), '\\', '/' );
diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php
index f40b8c16..f586578b 100644
--- a/includes/filebackend/FileBackend.php
+++ b/includes/filebackend/FileBackend.php
@@ -1,7 +1,6 @@
<?php
/**
* @defgroup FileBackend File backend
- * @ingroup FileRepo
*
* File backend is used to interact with file storage systems,
* such as the local file system, NFS, or cloud storage systems.
@@ -95,7 +94,7 @@ abstract class FileBackend {
* Allowed values are "implicit", "explicit" and "off".
* - concurrency : How many file operations can be done in parallel.
*
- * @param $config Array
+ * @param array $config
* @throws MWException
*/
public function __construct( array $config ) {
@@ -191,7 +190,6 @@ abstract class FileBackend {
* 'content' => <string of new file contents>,
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
- * 'disposition' => <Content-Disposition header value>,
* 'headers' => <HTTP header name/value map> # since 1.21
* );
* @endcode
@@ -204,7 +202,6 @@ abstract class FileBackend {
* 'dst' => <storage path>,
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
- * 'disposition' => <Content-Disposition header value>,
* 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
@@ -218,7 +215,7 @@ abstract class FileBackend {
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
* 'ignoreMissingSource' => <boolean>, # since 1.21
- * 'disposition' => <Content-Disposition header value>
+ * 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
*
@@ -231,7 +228,7 @@ abstract class FileBackend {
* 'overwrite' => <boolean>,
* 'overwriteSame' => <boolean>,
* 'ignoreMissingSource' => <boolean>, # since 1.21
- * 'disposition' => <Content-Disposition header value>
+ * 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
*
@@ -249,7 +246,6 @@ abstract class FileBackend {
* array(
* 'op' => 'describe',
* 'src' => <storage path>,
- * 'disposition' => <Content-Disposition header value>,
* 'headers' => <HTTP header name/value map>
* )
* @endcode
@@ -265,19 +261,19 @@ abstract class FileBackend {
* - ignoreMissingSource : The operation will simply succeed and do
* nothing if the source file does not exist.
* - overwrite : Any destination file will be overwritten.
- * - overwriteSame : An error will not be given if a file already
- * exists at the destination that has the same
- * contents as the new contents to be written there.
- * - disposition : If supplied, the backend will return a Content-Disposition
- * header when GETs/HEADs of the destination file are made.
- * Backends that don't support metadata ignore this.
- * See http://tools.ietf.org/html/rfc6266. (since 1.20)
- * - headers : If supplied, the backend will return these headers when
- * GETs/HEADs of the destination file are made. Header values
- * should be smaller than 256 bytes, often options or numbers.
- * Existing headers will remain, but these will replace any
- * conflicting previous headers, and headers will be removed
- * if they are set to an empty string.
+ * - overwriteSame : If a file already exists at the destination with the
+ * same contents, then do nothing to the destination file
+ * instead of giving an error. This does not compare headers.
+ * This option is ignored if 'overwrite' is already provided.
+ * - headers : If supplied, the result of merging these headers with any
+ * existing source file headers (replacing conflicting ones)
+ * will be set as the destination file headers. Headers are
+ * deleted if their value is set to the empty string. When a
+ * file has headers they are included in responses to GET and
+ * HEAD requests to the backing store for that file.
+ * Header values should be no larger than 255 bytes, except for
+ * Content-Disposition. The system might ignore or truncate any
+ * headers that are too long to store (exact limits will vary).
* Backends that don't support metadata ignore this. (since 1.21)
*
* $opts is an associative of boolean flags, including:
@@ -318,9 +314,17 @@ abstract class FileBackend {
if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) {
return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
}
+ if ( !count( $ops ) ) {
+ return Status::newGood(); // nothing to do
+ }
if ( empty( $opts['force'] ) ) { // sanity
unset( $opts['nonLocking'] );
}
+ foreach ( $ops as &$op ) {
+ if ( isset( $op['disposition'] ) ) { // b/c (MW 1.20)
+ $op['headers']['Content-Disposition'] = $op['disposition'];
+ }
+ }
$scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doOperationsInternal( $ops, $opts );
}
@@ -452,7 +456,6 @@ abstract class FileBackend {
* 'op' => 'create',
* 'dst' => <storage path>,
* 'content' => <string of new file contents>,
- * 'disposition' => <Content-Disposition header value>,
* 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
@@ -463,7 +466,6 @@ abstract class FileBackend {
* 'op' => 'store',
* 'src' => <file system path>,
* 'dst' => <storage path>,
- * 'disposition' => <Content-Disposition header value>,
* 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
@@ -475,7 +477,7 @@ abstract class FileBackend {
* 'src' => <storage path>,
* 'dst' => <storage path>,
* 'ignoreMissingSource' => <boolean>, # since 1.21
- * 'disposition' => <Content-Disposition header value>
+ * 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
*
@@ -486,7 +488,7 @@ abstract class FileBackend {
* 'src' => <storage path>,
* 'dst' => <storage path>,
* 'ignoreMissingSource' => <boolean>, # since 1.21
- * 'disposition' => <Content-Disposition header value>
+ * 'headers' => <HTTP header name/value map> # since 1.21
* )
* @endcode
*
@@ -504,7 +506,6 @@ abstract class FileBackend {
* array(
* 'op' => 'describe',
* 'src' => <storage path>,
- * 'disposition' => <Content-Disposition header value>,
* 'headers' => <HTTP header name/value map>
* )
* @endcode
@@ -519,13 +520,11 @@ abstract class FileBackend {
* @par Boolean flags for operations (operation-specific):
* - ignoreMissingSource : The operation will simply succeed and do
* nothing if the source file does not exist.
- * - disposition : When supplied, the backend will add a Content-Disposition
- * header when GETs/HEADs of the destination file are made.
- * Backends that don't support file metadata will ignore this.
- * See http://tools.ietf.org/html/rfc6266 (since 1.20).
* - headers : If supplied with a header name/value map, the backend will
* reply with these headers when GETs/HEADs of the destination
* file are made. Header values should be smaller than 256 bytes.
+ * Content-Disposition headers can be longer, though the system
+ * might ignore or truncate ones that are too long to store.
* Existing headers will remain, but these will replace any
* conflicting previous headers, and headers will be removed
* if they are set to an empty string.
@@ -549,8 +548,14 @@ abstract class FileBackend {
if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) {
return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
}
+ if ( !count( $ops ) ) {
+ return Status::newGood(); // nothing to do
+ }
foreach ( $ops as &$op ) {
$op['overwrite'] = true; // avoids RTTs in key/value stores
+ if ( isset( $op['disposition'] ) ) { // b/c (MW 1.20)
+ $op['headers']['Content-Disposition'] = $op['disposition'];
+ }
}
$scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts
return $this->doQuickOperationsInternal( $ops );
@@ -683,6 +688,8 @@ abstract class FileBackend {
* The 'noAccess' and 'noListing' parameters works the same as in secure(),
* except they are only applied *if* the directory/container had to be created.
* These flags should always be set for directories that have private files.
+ * However, setting them is not guaranteed to actually do anything.
+ * Additional server configuration may be needed to achieve the desired effect.
*
* @param array $params
* $params include:
@@ -710,7 +717,9 @@ abstract class FileBackend {
* the container it belongs to. FS backends might add .htaccess
* files whereas key/value store backends might revoke container
* access to the storage user representing end-users in web requests.
- * This is not guaranteed to actually do anything.
+ *
+ * This is not guaranteed to actually make files or listings publically hidden.
+ * Additional server configuration may be needed to achieve the desired effect.
*
* @param array $params
* $params include:
@@ -740,6 +749,9 @@ abstract class FileBackend {
* access to the storage user representing end-users in web requests.
* This essentially can undo the result of secure() calls.
*
+ * This is not guaranteed to actually make files or listings publically viewable.
+ * Additional server configuration may be needed to achieve the desired effect.
+ *
* @param array $params
* $params include:
* - dir : storage directory
@@ -797,7 +809,9 @@ abstract class FileBackend {
final protected function getScopedPHPBehaviorForOps() {
if ( php_sapi_name() != 'cli' ) { // http://bugs.php.net/bug.php?id=47540
$old = ignore_user_abort( true ); // avoid half-finished operations
- return new ScopedCallback( function() use ( $old ) { ignore_user_abort( $old ); } );
+ return new ScopedCallback( function() use ( $old ) {
+ ignore_user_abort( $old );
+ } );
}
return null;
}
@@ -1029,7 +1043,7 @@ abstract class FileBackend {
*
* Storage backends with eventual consistency might return stale data.
*
- * @param $params array
+ * @param array $params
* $params include:
* - dir : storage directory
* @return bool|null Returns null on failure
@@ -1047,7 +1061,9 @@ abstract class FileBackend {
*
* Storage backends with eventual consistency might return stale data.
*
- * @param $params array
+ * Failures during iteration can result in FileBackendError exceptions (since 1.22).
+ *
+ * @param array $params
* $params include:
* - dir : storage directory
* - topOnly : only return direct child dirs of the directory
@@ -1062,7 +1078,9 @@ abstract class FileBackend {
*
* Storage backends with eventual consistency might return stale data.
*
- * @param $params array
+ * Failures during iteration can result in FileBackendError exceptions (since 1.22).
+ *
+ * @param array $params
* $params include:
* - dir : storage directory
* @return Traversable|Array|null Returns null on failure
@@ -1082,10 +1100,13 @@ abstract class FileBackend {
*
* Storage backends with eventual consistency might return stale data.
*
- * @param $params array
+ * Failures during iteration can result in FileBackendError exceptions (since 1.22).
+ *
+ * @param array $params
* $params include:
- * - dir : storage directory
- * - topOnly : only return direct child files of the directory (since 1.20)
+ * - dir : storage directory
+ * - topOnly : only return direct child files of the directory (since 1.20)
+ * - adviseStat : set to true if stat requests will be made on the files (since 1.22)
* @return Traversable|Array|null Returns null on failure
*/
abstract public function getFileList( array $params );
@@ -1096,9 +1117,12 @@ abstract class FileBackend {
*
* Storage backends with eventual consistency might return stale data.
*
- * @param $params array
+ * Failures during iteration can result in FileBackendError exceptions (since 1.22).
+ *
+ * @param array $params
* $params include:
- * - dir : storage directory
+ * - dir : storage directory
+ * - adviseStat : set to true if stat requests will be made on the files (since 1.22)
* @return Traversable|Array|null Returns null on failure
* @since 1.20
*/
@@ -1131,10 +1155,11 @@ abstract class FileBackend {
* Callers should consider using getScopedFileLocks() instead.
*
* @param array $paths Storage paths
- * @param $type integer LockManager::LOCK_* constant
+ * @param integer $type LockManager::LOCK_* constant
* @return Status
*/
final public function lockFiles( array $paths, $type ) {
+ $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
return $this->lockManager->lock( $paths, $type );
}
@@ -1142,10 +1167,11 @@ abstract class FileBackend {
* Unlock the files at the given storage paths in the backend.
*
* @param array $paths Storage paths
- * @param $type integer LockManager::LOCK_* constant
+ * @param integer $type LockManager::LOCK_* constant
* @return Status
*/
final public function unlockFiles( array $paths, $type ) {
+ $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
return $this->lockManager->unlock( $paths, $type );
}
@@ -1157,12 +1183,21 @@ abstract class FileBackend {
* Once the return value goes out scope, the locks will be released and
* the status updated. Unlock fatals will not change the status "OK" value.
*
- * @param array $paths Storage paths
- * @param $type integer LockManager::LOCK_* constant
- * @param $status Status Status to update on lock/unlock
+ * @see ScopedLock::factory()
+ *
+ * @param array $paths List of storage paths or map of lock types to path lists
+ * @param integer|string $type LockManager::LOCK_* constant or "mixed"
+ * @param Status $status Status to update on lock/unlock
* @return ScopedLock|null Returns null on failure
*/
final public function getScopedFileLocks( array $paths, $type, Status $status ) {
+ if ( $type === 'mixed' ) {
+ foreach ( $paths as &$typePaths ) {
+ $typePaths = array_map( 'FileBackend::normalizeStoragePath', $typePaths );
+ }
+ } else {
+ $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
+ }
return ScopedLock::factory( $this->lockManager, $paths, $type, $status );
}
@@ -1178,7 +1213,7 @@ abstract class FileBackend {
* @see FileBackend::doOperations()
*
* @param array $ops List of file operations to FileBackend::doOperations()
- * @param $status Status Status to update on lock/unlock
+ * @param Status $status Status to update on lock/unlock
* @return Array List of ScopedFileLocks or null values
* @since 1.20
*/
@@ -1219,7 +1254,7 @@ abstract class FileBackend {
* Check if a given path is a "mwstore://" path.
* This does not do any further validation or any existence checks.
*
- * @param $path string
+ * @param string $path
* @return bool
*/
final public static function isStoragePath( $path ) {
@@ -1231,7 +1266,7 @@ abstract class FileBackend {
* and a relative file path. The relative path may be the empty string.
* This does not do any path normalization or traversal checks.
*
- * @param $storagePath string
+ * @param string $storagePath
* @return Array (backend, container, rel object) or (null, null, null)
*/
final public static function splitStoragePath( $storagePath ) {
@@ -1253,7 +1288,7 @@ abstract class FileBackend {
* Normalize a storage path by cleaning up directory separators.
* Returns null if the path is not of the format of a valid storage path.
*
- * @param $storagePath string
+ * @param string $storagePath
* @return string|null
*/
final public static function normalizeStoragePath( $storagePath ) {
@@ -1274,7 +1309,7 @@ abstract class FileBackend {
* This returns a path like "mwstore://backend/container",
* "mwstore://backend/container/...", or null if there is no parent.
*
- * @param $storagePath string
+ * @param string $storagePath
* @return string|null
*/
final public static function parentStoragePath( $storagePath ) {
@@ -1286,7 +1321,7 @@ abstract class FileBackend {
/**
* Get the final extension from a storage or FS path
*
- * @param $path string
+ * @param string $path
* @return string
*/
final public static function extensionFromPath( $path ) {
@@ -1297,7 +1332,7 @@ abstract class FileBackend {
/**
* Check if a relative path has no directory traversals
*
- * @param $path string
+ * @param string $path
* @return bool
* @since 1.20
*/
@@ -1363,3 +1398,9 @@ abstract class FileBackend {
return $path;
}
}
+
+/**
+ * @ingroup FileBackend
+ * @since 1.22
+ */
+class FileBackendError extends MWException {}
diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php
index d790a996..be8a2076 100644
--- a/includes/filebackend/FileBackendGroup.php
+++ b/includes/filebackend/FileBackendGroup.php
@@ -95,17 +95,17 @@ class FileBackendGroup {
: 0644;
// Get the FS backend configuration
$autoBackends[] = array(
- 'name' => $backendName,
- 'class' => 'FSFileBackend',
- 'lockManager' => 'fsLockManager',
+ 'name' => $backendName,
+ 'class' => 'FSFileBackend',
+ 'lockManager' => 'fsLockManager',
'containerPaths' => array(
- "{$repoName}-public" => "{$directory}",
- "{$repoName}-thumb" => $thumbDir,
- "{$repoName}-transcoded" => $transcodedDir,
+ "{$repoName}-public" => "{$directory}",
+ "{$repoName}-thumb" => $thumbDir,
+ "{$repoName}-transcoded" => $transcodedDir,
"{$repoName}-deleted" => $deletedDir,
- "{$repoName}-temp" => "{$directory}/temp"
+ "{$repoName}-temp" => "{$directory}/temp"
),
- 'fileMode' => $fileMode,
+ 'fileMode' => $fileMode,
);
}
@@ -116,7 +116,7 @@ class FileBackendGroup {
/**
* Register an array of file backend configurations
*
- * @param $configs Array
+ * @param Array $configs
* @return void
* @throws MWException
*/
@@ -135,8 +135,8 @@ class FileBackendGroup {
unset( $config['class'] ); // backend won't need this
$this->backends[$name] = array(
- 'class' => $class,
- 'config' => $config,
+ 'class' => $class,
+ 'config' => $config,
'instance' => null
);
}
@@ -145,7 +145,7 @@ class FileBackendGroup {
/**
* Get the backend object with a given name
*
- * @param $name string
+ * @param string $name
* @return FileBackend
* @throws MWException
*/
@@ -165,7 +165,7 @@ class FileBackendGroup {
/**
* Get the config array for a backend object with a given name
*
- * @param $name string
+ * @param string $name
* @return Array
* @throws MWException
*/
@@ -180,7 +180,7 @@ class FileBackendGroup {
/**
* Get an appropriate backend object from a storage path
*
- * @param $storagePath string
+ * @param string $storagePath
* @return FileBackend|null Backend or null on failure
*/
public function backendFromPath( $storagePath ) {
diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php
index 939315d1..97584a71 100644
--- a/includes/filebackend/FileBackendMultiWrite.php
+++ b/includes/filebackend/FileBackendMultiWrite.php
@@ -75,10 +75,13 @@ class FileBackendMultiWrite extends FileBackend {
* - autoResync : Automatically resync the clone backends to the master backend
* when pre-operation sync checks fail. This should only be used
* if the master backend is stable and not missing any files.
+ * Use "conservative" to limit resyncing to copying newer master
+ * backend files over older (or non-existing) clone backend files.
+ * Cases that cannot be handled will result in operation abortion.
* - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend.
* - noPushDirConts : (hack) Only apply directory functions to the master backend.
*
- * @param $config Array
+ * @param Array $config
* @throws MWException
*/
public function __construct( array $config ) {
@@ -86,7 +89,9 @@ class FileBackendMultiWrite extends FileBackend {
$this->syncChecks = isset( $config['syncChecks'] )
? $config['syncChecks']
: self::CHECK_SIZE;
- $this->autoResync = !empty( $config['autoResync'] );
+ $this->autoResync = isset( $config['autoResync'] )
+ ? $config['autoResync']
+ : false;
$this->noPushQuickOps = isset( $config['noPushQuickOps'] )
? $config['noPushQuickOps']
: false;
@@ -131,26 +136,15 @@ class FileBackendMultiWrite extends FileBackend {
}
}
- /**
- * @see FileBackend::doOperationsInternal()
- * @return Status
- */
final protected function doOperationsInternal( array $ops, array $opts ) {
$status = Status::newGood();
$mbe = $this->backends[$this->masterIndex]; // convenience
- // Get the paths to lock from the master backend
- $realOps = $this->substOpBatchPaths( $ops, $mbe );
- $paths = $mbe->getPathsToLockForOpsInternal( $mbe->getOperationsInternal( $realOps ) );
- // Get the paths under the proxy backend's name
- $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
- $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
// Try to lock those files for the scope of this function...
if ( empty( $opts['nonLocking'] ) ) {
// Try to lock those files for the scope of this function...
- $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
- $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
+ $scopeLock = $this->getScopedLocksForOps( $ops, $status );
if ( !$status->isOK() ) {
return $status; // abort
}
@@ -177,6 +171,7 @@ class FileBackendMultiWrite extends FileBackend {
}
}
// Actually attempt the operation batch on the master backend...
+ $realOps = $this->substOpBatchPaths( $ops, $mbe );
$masterStatus = $mbe->doOperations( $realOps, $opts );
$status->merge( $masterStatus );
// Propagate the operations to the clone backends if there were no unexpected errors
@@ -304,11 +299,11 @@ class FileBackendMultiWrite extends FileBackend {
$mBackend = $this->backends[$this->masterIndex];
foreach ( $paths as $path ) {
$mPath = $this->substPaths( $path, $mBackend );
- $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) );
- $mExist = $mBackend->fileExists( array( 'src' => $mPath ) );
- // Check if the master backend is available...
- if ( $mExist === null ) {
+ $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath, 'latest' => true ) );
+ $mStat = $mBackend->getFileStat( array( 'src' => $mPath, 'latest' => true ) );
+ if ( $mStat === null || ( $mSha1 !== false && !$mStat ) ) { // sanity
$status->fatal( 'backend-fail-internal', $this->name );
+ continue; // file is not available on the master backend...
}
// Check of all clone backends agree with the master...
foreach ( $this->backends as $index => $cBackend ) {
@@ -316,15 +311,31 @@ class FileBackendMultiWrite extends FileBackend {
continue; // master
}
$cPath = $this->substPaths( $path, $cBackend );
- $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath ) );
+ $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath, 'latest' => true ) );
+ $cStat = $cBackend->getFileStat( array( 'src' => $cPath, 'latest' => true ) );
+ if ( $cStat === null || ( $cSha1 !== false && !$cStat ) ) { // sanity
+ $status->fatal( 'backend-fail-internal', $cBackend->getName() );
+ continue; // file is not available on the clone backend...
+ }
if ( $mSha1 === $cSha1 ) {
// already synced; nothing to do
- } elseif ( $mSha1 ) { // file is in master
- $fsFile = $mBackend->getLocalReference( array( 'src' => $mPath ) );
+ } elseif ( $mSha1 !== false ) { // file is in master
+ if ( $this->autoResync === 'conservative'
+ && $cStat && $cStat['mtime'] > $mStat['mtime'] )
+ {
+ $status->fatal( 'backend-fail-synced', $path );
+ continue; // don't rollback data
+ }
+ $fsFile = $mBackend->getLocalReference(
+ array( 'src' => $mPath, 'latest' => true ) );
$status->merge( $cBackend->quickStore(
array( 'src' => $fsFile->getPath(), 'dst' => $cPath )
) );
- } elseif ( $mExist === false ) { // file is not in master
+ } elseif ( $mStat === false ) { // file is not in master
+ if ( $this->autoResync === 'conservative' ) {
+ $status->fatal( 'backend-fail-synced', $path );
+ continue; // don't delete data
+ }
$status->merge( $cBackend->quickDelete( array( 'src' => $cPath ) ) );
}
}
@@ -366,7 +377,7 @@ class FileBackendMultiWrite extends FileBackend {
* for a set of operations with that of a given internal backend.
*
* @param array $ops List of file operation arrays
- * @param $backend FileBackendStore
+ * @param FileBackendStore $backend
* @return Array
*/
protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
@@ -387,7 +398,7 @@ class FileBackendMultiWrite extends FileBackend {
* Same as substOpBatchPaths() but for a single operation
*
* @param array $ops File operation array
- * @param $backend FileBackendStore
+ * @param FileBackendStore $backend
* @return Array
*/
protected function substOpPaths( array $ops, FileBackendStore $backend ) {
@@ -399,7 +410,7 @@ class FileBackendMultiWrite extends FileBackend {
* Substitute the backend of storage paths with an internal backend's name
*
* @param array|string $paths List of paths or single string path
- * @param $backend FileBackendStore
+ * @param FileBackendStore $backend
* @return Array|string
*/
protected function substPaths( $paths, FileBackendStore $backend ) {
@@ -424,10 +435,6 @@ class FileBackendMultiWrite extends FileBackend {
);
}
- /**
- * @see FileBackend::doQuickOperationsInternal()
- * @return Status
- */
protected function doQuickOperationsInternal( array $ops ) {
$status = Status::newGood();
// Do the operations on the master backend; setting Status fields...
@@ -457,14 +464,10 @@ class FileBackendMultiWrite extends FileBackend {
* @return bool Path container should have dir changes pushed to all backends
*/
protected function replicateContainerDirChanges( $path ) {
- list( , $shortCont, ) = self::splitStoragePath( $path );
+ list( , $shortCont, ) = self::splitStoragePath( $path );
return !in_array( $shortCont, $this->noPushDirConts );
}
- /**
- * @see FileBackend::doPrepare()
- * @return Status
- */
protected function doPrepare( array $params ) {
$status = Status::newGood();
$replicate = $this->replicateContainerDirChanges( $params['dir'] );
@@ -477,11 +480,6 @@ class FileBackendMultiWrite extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::doSecure()
- * @param $params array
- * @return Status
- */
protected function doSecure( array $params ) {
$status = Status::newGood();
$replicate = $this->replicateContainerDirChanges( $params['dir'] );
@@ -494,11 +492,6 @@ class FileBackendMultiWrite extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::doPublish()
- * @param $params array
- * @return Status
- */
protected function doPublish( array $params ) {
$status = Status::newGood();
$replicate = $this->replicateContainerDirChanges( $params['dir'] );
@@ -511,11 +504,6 @@ class FileBackendMultiWrite extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::doClean()
- * @param $params array
- * @return Status
- */
protected function doClean( array $params ) {
$status = Status::newGood();
$replicate = $this->replicateContainerDirChanges( $params['dir'] );
@@ -528,62 +516,32 @@ class FileBackendMultiWrite extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::concatenate()
- * @param $params array
- * @return Status
- */
public function concatenate( array $params ) {
// We are writing to an FS file, so we don't need to do this per-backend
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->concatenate( $realParams );
}
- /**
- * @see FileBackend::fileExists()
- * @param $params array
- * @return bool|null
- */
public function fileExists( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->fileExists( $realParams );
}
- /**
- * @see FileBackend::getFileTimestamp()
- * @param $params array
- * @return bool|string
- */
public function getFileTimestamp( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
}
- /**
- * @see FileBackend::getFileSize()
- * @param $params array
- * @return bool|int
- */
public function getFileSize( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getFileSize( $realParams );
}
- /**
- * @see FileBackend::getFileStat()
- * @param $params array
- * @return Array|bool|null
- */
public function getFileStat( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getFileStat( $realParams );
}
- /**
- * @see FileBackend::getFileContentsMulti()
- * @param $params array
- * @return bool|string
- */
public function getFileContentsMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
$contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams );
@@ -595,41 +553,21 @@ class FileBackendMultiWrite extends FileBackend {
return $contents;
}
- /**
- * @see FileBackend::getFileSha1Base36()
- * @param $params array
- * @return bool|string
- */
public function getFileSha1Base36( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
}
- /**
- * @see FileBackend::getFileProps()
- * @param $params array
- * @return Array
- */
public function getFileProps( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getFileProps( $realParams );
}
- /**
- * @see FileBackend::streamFile()
- * @param $params array
- * @return \Status
- */
public function streamFile( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->streamFile( $realParams );
}
- /**
- * @see FileBackend::getLocalReferenceMulti()
- * @param $params array
- * @return FSFile|null
- */
public function getLocalReferenceMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
$fsFilesM = $this->backends[$this->masterIndex]->getLocalReferenceMulti( $realParams );
@@ -641,11 +579,6 @@ class FileBackendMultiWrite extends FileBackend {
return $fsFiles;
}
- /**
- * @see FileBackend::getLocalCopyMulti()
- * @param $params array
- * @return null|TempFSFile
- */
public function getLocalCopyMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
$tempFilesM = $this->backends[$this->masterIndex]->getLocalCopyMulti( $realParams );
@@ -657,48 +590,26 @@ class FileBackendMultiWrite extends FileBackend {
return $tempFiles;
}
- /**
- * @see FileBackend::getFileHttpUrl()
- * @return string|null
- */
public function getFileHttpUrl( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams );
}
- /**
- * @see FileBackend::directoryExists()
- * @param $params array
- * @return bool|null
- */
public function directoryExists( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->directoryExists( $realParams );
}
- /**
- * @see FileBackend::getSubdirectoryList()
- * @param $params array
- * @return Array|null|Traversable
- */
public function getDirectoryList( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
}
- /**
- * @see FileBackend::getFileList()
- * @param $params array
- * @return Array|null|\Traversable
- */
public function getFileList( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
return $this->backends[$this->masterIndex]->getFileList( $realParams );
}
- /**
- * @see FileBackend::clearCache()
- */
public function clearCache( array $paths = null ) {
foreach ( $this->backends as $backend ) {
$realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
@@ -706,19 +617,17 @@ class FileBackendMultiWrite extends FileBackend {
}
}
- /**
- * @see FileBackend::getScopedLocksForOps()
- */
public function getScopedLocksForOps( array $ops, Status $status ) {
- $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $ops );
+ $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
+ $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps );
// Get the paths to lock from the master backend
$paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
// Get the paths under the proxy backend's name
- $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
- $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
- return array(
- $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
- $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
+ $pbPaths = array(
+ LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ),
+ LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] )
);
+ // Actually acquire the locks
+ return array( $this->getScopedFileLocks( $pbPaths, 'mixed', $status ) );
}
}
diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php
index 3f1d1857..0921e99f 100644
--- a/includes/filebackend/FileBackendStore.php
+++ b/includes/filebackend/FileBackendStore.php
@@ -38,28 +38,43 @@
abstract class FileBackendStore extends FileBackend {
/** @var BagOStuff */
protected $memCache;
- /** @var ProcessCacheLRU */
- protected $cheapCache; // Map of paths to small (RAM/disk) cache items
- /** @var ProcessCacheLRU */
- protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
+ /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */
+ protected $cheapCache;
+ /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */
+ protected $expensiveCache;
- /** @var Array Map of container names to sharding settings */
- protected $shardViaHashLevels = array(); // (container name => config array)
+ /** @var Array Map of container names to sharding config */
+ protected $shardViaHashLevels = array();
+
+ /** @var callback Method to get the MIME type of files */
+ protected $mimeCallback;
protected $maxFileSize = 4294967296; // integer bytes (4GiB)
const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
+ const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache"
+ const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
/**
* @see FileBackend::__construct()
+ * Additional $config params include:
+ * - mimeCallback : Callback that takes (storage path, content, file system path) and
+ * returns the MIME type of the file or 'unknown/unknown'. The file
+ * system path parameter should be used if the content one is null.
*
- * @param $config Array
+ * @param array $config
*/
public function __construct( array $config ) {
parent::__construct( $config );
+ $this->mimeCallback = isset( $config['mimeCallback'] )
+ ? $config['mimeCallback']
+ : function( $storagePath, $content, $fsPath ) {
+ // @TODO: handle the case of extension-less files using the contents
+ return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown';
+ };
$this->memCache = new EmptyBagOStuff(); // disabled by default
- $this->cheapCache = new ProcessCacheLRU( 300 );
- $this->expensiveCache = new ProcessCacheLRU( 5 );
+ $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
+ $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );
}
/**
@@ -79,7 +94,7 @@ abstract class FileBackendStore extends FileBackend {
* written under it, and that any file already there is writable.
* Backends using key/value stores should check if the container exists.
*
- * @param $storagePath string
+ * @param string $storagePath
* @return bool
*/
abstract public function isPathUsableInternal( $storagePath );
@@ -92,7 +107,6 @@ abstract class FileBackendStore extends FileBackend {
* $params include:
* - content : the raw file contents
* - dst : destination storage path
- * - disposition : Content-Disposition header value for the destination
* - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
@@ -104,8 +118,7 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function createInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
$status = Status::newFatal( 'backend-fail-maxsize',
$params['dst'], $this->maxFileSizeInternal() );
@@ -116,8 +129,6 @@ abstract class FileBackendStore extends FileBackend {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -135,7 +146,6 @@ abstract class FileBackendStore extends FileBackend {
* $params include:
* - src : source path on disk
* - dst : destination storage path
- * - disposition : Content-Disposition header value for the destination
* - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
@@ -147,8 +157,7 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function storeInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
$status = Status::newFatal( 'backend-fail-maxsize',
$params['dst'], $this->maxFileSizeInternal() );
@@ -159,8 +168,6 @@ abstract class FileBackendStore extends FileBackend {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -179,7 +186,7 @@ abstract class FileBackendStore extends FileBackend {
* - src : source storage path
* - dst : destination storage path
* - ignoreMissingSource : do nothing if the source file does not exist
- * - disposition : Content-Disposition header value for the destination
+ * - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
* set to a FileBackendStoreOpHandle object.
@@ -190,15 +197,12 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function copyInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = $this->doCopyInternal( $params );
$this->clearCache( array( $params['dst'] ) );
if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -223,13 +227,10 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function deleteInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = $this->doDeleteInternal( $params );
$this->clearCache( array( $params['src'] ) );
$this->deleteFileCache( $params['src'] ); // persistent cache
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -248,7 +249,7 @@ abstract class FileBackendStore extends FileBackend {
* - src : source storage path
* - dst : destination storage path
* - ignoreMissingSource : do nothing if the source file does not exist
- * - disposition : Content-Disposition header value for the destination
+ * - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
* set to a FileBackendStoreOpHandle object.
@@ -259,16 +260,13 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function moveInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = $this->doMoveInternal( $params );
$this->clearCache( array( $params['src'], $params['dst'] ) );
$this->deleteFileCache( $params['src'] ); // persistent cache
if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -278,10 +276,12 @@ abstract class FileBackendStore extends FileBackend {
*/
protected function doMoveInternal( array $params ) {
unset( $params['async'] ); // two steps, won't work here :)
+ $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
+ $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
// Copy source to dest
$status = $this->copyInternal( $params );
- if ( $status->isOK() ) {
- // Delete source (only fails due to races or medium going down)
+ if ( $nsrc !== $ndst && $status->isOK() ) {
+ // Delete source (only fails due to races or network problems)
$status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
$status->setResult( true, $status->value ); // ignore delete() errors
}
@@ -294,7 +294,6 @@ abstract class FileBackendStore extends FileBackend {
*
* $params include:
* - src : source storage path
- * - disposition : Content-Disposition header value for the destination
* - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
@@ -304,13 +303,14 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function describeInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
- $status = $this->doDescribeInternal( $params );
- $this->clearCache( array( $params['src'] ) );
- $this->deleteFileCache( $params['src'] ); // persistent cache
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
+ if ( count( $params['headers'] ) ) {
+ $status = $this->doDescribeInternal( $params );
+ $this->clearCache( array( $params['src'] ) );
+ $this->deleteFileCache( $params['src'] ); // persistent cache
+ } else {
+ $status = Status::newGood(); // nothing to do
+ }
return $status;
}
@@ -333,13 +333,8 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::concatenate()
- * @return Status
- */
final public function concatenate( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Try to lock the source files for the scope of this function
@@ -355,8 +350,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -426,20 +419,13 @@ abstract class FileBackendStore extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::doPrepare()
- * @return Status
- */
final protected function doPrepare( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
-
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
+
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -453,8 +439,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -466,20 +450,13 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::doSecure()
- * @return Status
- */
final protected function doSecure( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -493,8 +470,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -506,20 +481,13 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::doPublish()
- * @return Status
- */
final protected function doPublish( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -533,8 +501,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -546,13 +512,8 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::doClean()
- * @return Status
- */
final protected function doClean( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Recursive: first delete all empty subdirs recursively
@@ -570,8 +531,6 @@ abstract class FileBackendStore extends FileBackend {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -579,8 +538,6 @@ abstract class FileBackendStore extends FileBackend {
$filesLockEx = array( $params['dir'] );
$scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
if ( !$status->isOK() ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // abort
}
@@ -596,8 +553,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -609,56 +564,30 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::fileExists()
- * @return bool|null
- */
final public function fileExists( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return ( $stat === null ) ? null : (bool)$stat; // null => failure
}
- /**
- * @see FileBackend::getFileTimestamp()
- * @return bool
- */
final public function getFileTimestamp( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat ? $stat['mtime'] : false;
}
- /**
- * @see FileBackend::getFileSize()
- * @return bool
- */
final public function getFileSize( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat ? $stat['size'] : false;
}
- /**
- * @see FileBackend::getFileStat()
- * @return bool
- */
final public function getFileStat( array $params ) {
$path = self::normalizeStoragePath( $params['src'] );
if ( $path === null ) {
return false; // invalid storage path
}
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$latest = !empty( $params['latest'] ); // use latest data?
if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
$this->primeFileCache( array( $path ) ); // check persistent cache
@@ -669,14 +598,10 @@ abstract class FileBackendStore extends FileBackend {
// value was in fact fetched with the latest available data.
if ( is_array( $stat ) ) {
if ( !$latest || $stat['latest'] ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat;
}
} elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) {
if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return false;
}
}
@@ -696,12 +621,12 @@ abstract class FileBackendStore extends FileBackend {
}
} elseif ( $stat === false ) { // file does not exist
$this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
+ $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too
+ array( 'hash' => false, 'latest' => $latest ) );
wfDebug( __METHOD__ . ": File $path does not exist.\n" );
} else { // an error occurred
wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat;
}
@@ -710,19 +635,12 @@ abstract class FileBackendStore extends FileBackend {
*/
abstract protected function doGetFileStat( array $params );
- /**
- * @see FileBackend::getFileContentsMulti()
- * @return Array
- */
public function getFileContentsMulti( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$params = $this->setConcurrencyFlags( $params );
$contents = $this->doGetFileContentsMulti( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $contents;
}
@@ -740,25 +658,18 @@ abstract class FileBackendStore extends FileBackend {
return $contents;
}
- /**
- * @see FileBackend::getFileSha1Base36()
- * @return bool|string
- */
final public function getFileSha1Base36( array $params ) {
$path = self::normalizeStoragePath( $params['src'] );
if ( $path === null ) {
return false; // invalid storage path
}
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$latest = !empty( $params['latest'] ); // use latest data?
if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
$stat = $this->cheapCache->get( $path, 'sha1' );
// If we want the latest data, check that this cached
// value was in fact fetched with the latest available data.
if ( !$latest || $stat['latest'] ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat['hash'];
}
}
@@ -768,8 +679,6 @@ abstract class FileBackendStore extends FileBackend {
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
wfProfileOut( __METHOD__ . '-miss' );
$this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $hash;
}
@@ -786,27 +695,15 @@ abstract class FileBackendStore extends FileBackend {
}
}
- /**
- * @see FileBackend::getFileProps()
- * @return Array
- */
final public function getFileProps( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$fsFile = $this->getLocalReference( $params );
$props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $props;
}
- /**
- * @see FileBackend::getLocalReferenceMulti()
- * @return Array
- */
final public function getLocalReferenceMulti( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$params = $this->setConcurrencyFlags( $params );
@@ -836,8 +733,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $fsFiles;
}
@@ -849,19 +744,12 @@ abstract class FileBackendStore extends FileBackend {
return $this->doGetLocalCopyMulti( $params );
}
- /**
- * @see FileBackend::getLocalCopyMulti()
- * @return Array
- */
final public function getLocalCopyMulti( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$params = $this->setConcurrencyFlags( $params );
$tmpFiles = $this->doGetLocalCopyMulti( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $tmpFiles;
}
@@ -879,13 +767,8 @@ abstract class FileBackendStore extends FileBackend {
return null; // not supported
}
- /**
- * @see FileBackend::streamFile()
- * @return Status
- */
final public function streamFile( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
$info = $this->getFileStat( $params );
@@ -916,8 +799,6 @@ abstract class FileBackendStore extends FileBackend {
$status->fatal( 'backend-fail-stream', $params['src'] );
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -938,10 +819,6 @@ abstract class FileBackendStore extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::directoryExists()
- * @return bool|null
- */
final public function directoryExists( array $params ) {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
@@ -976,10 +853,6 @@ abstract class FileBackendStore extends FileBackend {
*/
abstract protected function doDirectoryExists( $container, $dir, array $params );
- /**
- * @see FileBackend::getDirectoryList()
- * @return Traversable|Array|null Returns null on failure
- */
final public function getDirectoryList( array $params ) {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) { // invalid storage path
@@ -1009,10 +882,6 @@ abstract class FileBackendStore extends FileBackend {
*/
abstract public function getDirectoryListInternal( $container, $dir, array $params );
- /**
- * @see FileBackend::getFileList()
- * @return Traversable|Array|null Returns null on failure
- */
final public function getFileList( array $params ) {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) { // invalid storage path
@@ -1055,13 +924,13 @@ abstract class FileBackendStore extends FileBackend {
*/
final public function getOperationsInternal( array $ops ) {
$supportedOps = array(
- 'store' => 'StoreFileOp',
- 'copy' => 'CopyFileOp',
- 'move' => 'MoveFileOp',
- 'delete' => 'DeleteFileOp',
- 'create' => 'CreateFileOp',
+ 'store' => 'StoreFileOp',
+ 'copy' => 'CopyFileOp',
+ 'move' => 'MoveFileOp',
+ 'delete' => 'DeleteFileOp',
+ 'create' => 'CreateFileOp',
'describe' => 'DescribeFileOp',
- 'null' => 'NullFileOp'
+ 'null' => 'NullFileOp'
);
$performOps = array(); // array of FileOp objects
@@ -1084,12 +953,13 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get a list of storage paths to lock for a list of operations
- * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
- * each corresponding to a list of storage paths to be locked.
- * All returned paths are normalized.
+ * Returns an array with LockManager::LOCK_UW (shared locks) and
+ * LockManager::LOCK_EX (exclusive locks) keys, each corresponding
+ * to a list of storage paths to be locked. All returned paths are
+ * normalized.
*
* @param array $performOps List of FileOp objects
- * @return Array ('sh' => list of paths, 'ex' => list of paths)
+ * @return Array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list)
*/
final public function getPathsToLockForOpsInternal( array $performOps ) {
// Build up a list of files to lock...
@@ -1103,28 +973,19 @@ abstract class FileBackendStore extends FileBackend {
// Get a shared lock on the parent directory of each path changed
$paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
- return $paths;
+ return array(
+ LockManager::LOCK_UW => $paths['sh'],
+ LockManager::LOCK_EX => $paths['ex']
+ );
}
- /**
- * @see FileBackend::getScopedLocksForOps()
- * @return Array
- */
public function getScopedLocksForOps( array $ops, Status $status ) {
$paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
- return array(
- $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
- $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
- );
+ return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) );
}
- /**
- * @see FileBackend::doOperationsInternal()
- * @return Status
- */
final protected function doOperationsInternal( array $ops, array $opts ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Fix up custom header name/value pairs...
@@ -1138,11 +999,8 @@ abstract class FileBackendStore extends FileBackend {
// Build up a list of files to lock...
$paths = $this->getPathsToLockForOpsInternal( $performOps );
// Try to lock those files for the scope of this function...
- $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
- $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
+ $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
if ( !$status->isOK() ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // abort
}
}
@@ -1164,19 +1022,11 @@ abstract class FileBackendStore extends FileBackend {
$status->merge( $subStatus );
$status->success = $subStatus->success; // not done in merge()
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
- /**
- * @see FileBackend::doQuickOperationsInternal()
- * @return Status
- * @throws MWException
- */
final protected function doQuickOperationsInternal( array $ops ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Fix up custom header name/value pairs...
@@ -1186,7 +1036,7 @@ abstract class FileBackendStore extends FileBackend {
$this->clearCache();
$supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
- $async = ( $this->parallelize === 'implicit' );
+ $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
$maxConcurrency = $this->concurrency; // throttle
$statuses = array(); // array of (index => Status)
@@ -1195,8 +1045,6 @@ abstract class FileBackendStore extends FileBackend {
// Perform the sync-only ops and build up op handles for the async ops...
foreach ( $ops as $index => $params ) {
if ( !in_array( $params['op'], $supportedOps ) ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
throw new MWException( "Operation '{$params['op']}' is not supported." );
}
$method = $params['op'] . 'Internal'; // e.g. "storeInternal"
@@ -1230,8 +1078,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -1245,8 +1091,7 @@ abstract class FileBackendStore extends FileBackend {
* @throws MWException
*/
final public function executeOpHandlesInternal( array $fileOpHandles ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
foreach ( $fileOpHandles as $fileOpHandle ) {
if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
@@ -1258,8 +1103,6 @@ abstract class FileBackendStore extends FileBackend {
foreach ( $fileOpHandles as $fileOpHandle ) {
$fileOpHandle->closeResources();
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $res;
}
@@ -1277,15 +1120,20 @@ abstract class FileBackendStore extends FileBackend {
}
/**
- * Strip long HTTP headers from a file operation
+ * Strip long HTTP headers from a file operation.
+ * Most headers are just numbers, but some are allowed to be long.
+ * This function is useful for cleaning up headers and avoiding backend
+ * specific errors, especially in the middle of batch file operations.
*
* @param array $op Same format as doOperation()
* @return Array
*/
protected function stripInvalidHeadersFromOp( array $op ) {
- if ( isset( $op['headers'] ) ) {
+ static $longs = array( 'Content-Disposition' );
+ if ( isset( $op['headers'] ) ) { // op sets HTTP headers
foreach ( $op['headers'] as $name => $value ) {
- if ( strlen( $name ) > 255 || strlen( $value ) > 255 ) {
+ $maxHVLen = in_array( $name, $longs ) ? INF : 255;
+ if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
trigger_error( "Header '$name: $value' is too long." );
unset( $op['headers'][$name] );
} elseif ( !strlen( $value ) ) {
@@ -1296,9 +1144,6 @@ abstract class FileBackendStore extends FileBackend {
return $op;
}
- /**
- * @see FileBackend::preloadCache()
- */
final public function preloadCache( array $paths ) {
$fullConts = array(); // full container names
foreach ( $paths as $path ) {
@@ -1310,9 +1155,6 @@ abstract class FileBackendStore extends FileBackend {
$this->primeFileCache( $paths );
}
- /**
- * @see FileBackend::clearCache()
- */
final public function clearCache( array $paths = null ) {
if ( is_array( $paths ) ) {
$paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
@@ -1353,7 +1195,7 @@ abstract class FileBackendStore extends FileBackend {
* Check if a container name is valid.
* This checks for for length and illegal characters.
*
- * @param $container string
+ * @param string $container
* @return bool
*/
final protected static function isValidContainerName( $container ) {
@@ -1375,7 +1217,7 @@ abstract class FileBackendStore extends FileBackend {
* this means that the path can only refer to a directory and can only
* be scanned by looking in all the container shards.
*
- * @param $storagePath string
+ * @param string $storagePath
* @return Array (container, path, container suffix) or (null, null, null) if invalid
*/
final protected function resolveStoragePath( $storagePath ) {
@@ -1405,16 +1247,22 @@ abstract class FileBackendStore extends FileBackend {
/**
* Like resolveStoragePath() except null values are returned if
- * the container is sharded and the shard could not be determined.
+ * the container is sharded and the shard could not be determined
+ * or if the path ends with '/'. The later case is illegal for FS
+ * backends and can confuse listings for object store backends.
+ *
+ * This function is used when resolving paths that must be valid
+ * locations for files. Directory and listing functions should
+ * generally just use resolveStoragePath() instead.
*
* @see FileBackendStore::resolveStoragePath()
*
- * @param $storagePath string
+ * @param string $storagePath
* @return Array (container, path) or (null, null) if invalid
*/
final protected function resolveStoragePathReal( $storagePath ) {
list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
- if ( $cShard !== null ) {
+ if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
return array( $container, $relPath );
}
return array( null, null );
@@ -1474,7 +1322,7 @@ abstract class FileBackendStore extends FileBackend {
* If greater than 0, then all file storage paths within
* the container are required to be hashed accordingly.
*
- * @param $container string
+ * @param string $container
* @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
*/
final protected function getContainerHashLevels( $container ) {
@@ -1494,7 +1342,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get a list of full container shard suffixes for a container
*
- * @param $container string
+ * @param string $container
* @return Array
*/
final protected function getContainerSuffixes( $container ) {
@@ -1512,7 +1360,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get the full container name, including the wiki ID prefix
*
- * @param $container string
+ * @param string $container
* @return string
*/
final protected function fullContainerName( $container ) {
@@ -1528,7 +1376,7 @@ abstract class FileBackendStore extends FileBackend {
* This is intended for internal use, such as encoding illegal chars.
* Subclasses can override this to be more restrictive.
*
- * @param $container string
+ * @param string $container
* @return string|null
*/
protected function resolveContainerName( $container ) {
@@ -1563,10 +1411,11 @@ abstract class FileBackendStore extends FileBackend {
* Set the cached info for a container
*
* @param string $container Resolved container name
- * @param $val mixed Information to cache
+ * @param array $val Information to cache
+ * @return void
*/
- final protected function setContainerCache( $container, $val ) {
- $this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 );
+ final protected function setContainerCache( $container, array $val ) {
+ $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 );
}
/**
@@ -1574,6 +1423,7 @@ abstract class FileBackendStore extends FileBackend {
* The cache key is salted for a while to prevent race conditions.
*
* @param string $container Resolved container name
+ * @return void
*/
final protected function deleteContainerCache( $container ) {
if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
@@ -1586,12 +1436,11 @@ abstract class FileBackendStore extends FileBackend {
* used in a list of container names, storage paths, or FileOp objects.
* This loads the persistent cache values into the process cache.
*
- * @param $items Array
+ * @param Array $items
* @return void
*/
final protected function primeContainerCache( array $items ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$paths = array(); // list of storage paths
$contNames = array(); // (cache key => resolved container name)
@@ -1623,9 +1472,6 @@ abstract class FileBackendStore extends FileBackend {
// Populate the container process cache for the backend...
$this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
-
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
}
/**
@@ -1654,14 +1500,17 @@ abstract class FileBackendStore extends FileBackend {
* salting for the case when a file is created at a path were there was none before.
*
* @param string $path Storage path
- * @param $val mixed Information to cache
+ * @param array $val Stat information to cache
+ * @return void
*/
- final protected function setFileCache( $path, $val ) {
+ final protected function setFileCache( $path, array $val ) {
$path = FileBackend::normalizeStoragePath( $path );
if ( $path === null ) {
return; // invalid storage path
}
- $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
+ $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
+ $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
+ $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
}
/**
@@ -1671,6 +1520,7 @@ abstract class FileBackendStore extends FileBackend {
* a file is created at a path were there was none before.
*
* @param string $path Storage path
+ * @return void
*/
final protected function deleteFileCache( $path ) {
$path = FileBackend::normalizeStoragePath( $path );
@@ -1691,8 +1541,7 @@ abstract class FileBackendStore extends FileBackend {
* @return void
*/
final protected function primeFileCache( array $items ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$paths = array(); // list of storage paths
$pathNames = array(); // (cache key => storage path)
@@ -1726,9 +1575,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
}
-
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
}
/**
@@ -1750,6 +1596,18 @@ abstract class FileBackendStore extends FileBackend {
}
return $opts;
}
+
+ /**
+ * Get the content type to use in HEAD/GET requests for a file
+ *
+ * @param string $storagePath
+ * @param string|null $content File data
+ * @param string|null $fsPath File system path
+ * @return MIME type
+ */
+ protected function getContentType( $storagePath, $content, $fsPath ) {
+ return call_user_func_array( $this->mimeCallback, func_get_args() );
+ }
}
/**
@@ -1786,26 +1644,20 @@ abstract class FileBackendStoreOpHandle {
*
* @ingroup FileBackend
*/
-abstract class FileBackendStoreShardListIterator implements Iterator {
+abstract class FileBackendStoreShardListIterator extends FilterIterator {
/** @var FileBackendStore */
protected $backend;
/** @var Array */
protected $params;
- /** @var Array */
- protected $shardSuffixes;
+
protected $container; // string; full container name
protected $directory; // string; resolved relative path
- /** @var Traversable */
- protected $iter;
- protected $curShard = 0; // integer
- protected $pos = 0; // integer
-
/** @var Array */
protected $multiShardPaths = array(); // (rel path => 1)
/**
- * @param $backend FileBackendStore
+ * @param FileBackendStore $backend
* @param string $container Full storage container name
* @param string $dir Storage directory relative to container
* @param array $suffixes List of container shard suffixes
@@ -1817,142 +1669,56 @@ abstract class FileBackendStoreShardListIterator implements Iterator {
$this->backend = $backend;
$this->container = $container;
$this->directory = $dir;
- $this->shardSuffixes = $suffixes;
$this->params = $params;
- }
-
- /**
- * @see Iterator::key()
- * @return integer
- */
- public function key() {
- return $this->pos;
- }
- /**
- * @see Iterator::valid()
- * @return bool
- */
- public function valid() {
- if ( $this->iter instanceof Iterator ) {
- return $this->iter->valid();
- } elseif ( is_array( $this->iter ) ) {
- return ( current( $this->iter ) !== false ); // no paths can have this value
+ $iter = new AppendIterator();
+ foreach ( $suffixes as $suffix ) {
+ $iter->append( $this->listFromShard( $this->container . $suffix ) );
}
- return false; // some failure?
- }
-
- /**
- * @see Iterator::current()
- * @return string|bool String or false
- */
- public function current() {
- return ( $this->iter instanceof Iterator )
- ? $this->iter->current()
- : current( $this->iter );
- }
- /**
- * @see Iterator::next()
- * @return void
- */
- public function next() {
- ++$this->pos;
- ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
- do {
- $continue = false; // keep scanning shards?
- $this->filterViaNext(); // filter out duplicates
- // Find the next non-empty shard if no elements are left
- if ( !$this->valid() ) {
- $this->nextShardIteratorIfNotValid();
- $continue = $this->valid(); // re-filter unless we ran out of shards
- }
- } while ( $continue );
- }
-
- /**
- * @see Iterator::rewind()
- * @return void
- */
- public function rewind() {
- $this->pos = 0;
- $this->curShard = 0;
- $this->setIteratorFromCurrentShard();
- do {
- $continue = false; // keep scanning shards?
- $this->filterViaNext(); // filter out duplicates
- // Find the next non-empty shard if no elements are left
- if ( !$this->valid() ) {
- $this->nextShardIteratorIfNotValid();
- $continue = $this->valid(); // re-filter unless we ran out of shards
- }
- } while ( $continue );
- }
-
- /**
- * Filter out duplicate items by advancing to the next ones
- */
- protected function filterViaNext() {
- while ( $this->valid() ) {
- $rel = $this->iter->current(); // path relative to given directory
- $path = $this->params['dir'] . "/{$rel}"; // full storage path
- if ( $this->backend->isSingleShardPathInternal( $path ) ) {
- break; // path is only on one shard; no issue with duplicates
- } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
- // Don't keep listing paths that are on multiple shards
- ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
- } else {
- $this->multiShardPaths[$rel] = 1;
- break;
- }
- }
+ parent::__construct( $iter );
}
- /**
- * If the list iterator for this container shard is out of items,
- * then move on to the next container that has items.
- * If there are none, then it advances to the last container.
- */
- protected function nextShardIteratorIfNotValid() {
- while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) {
- $this->setIteratorFromCurrentShard();
+ public function accept() {
+ $rel = $this->getInnerIterator()->current(); // path relative to given directory
+ $path = $this->params['dir'] . "/{$rel}"; // full storage path
+ if ( $this->backend->isSingleShardPathInternal( $path ) ) {
+ return true; // path is only on one shard; no issue with duplicates
+ } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
+ // Don't keep listing paths that are on multiple shards
+ return false;
+ } else {
+ $this->multiShardPaths[$rel] = 1;
+ return true;
}
}
- /**
- * Set the list iterator to that of the current container shard
- */
- protected function setIteratorFromCurrentShard() {
- $this->iter = $this->listFromShard(
- $this->container . $this->shardSuffixes[$this->curShard],
- $this->directory, $this->params );
- // Start loading results so that current() works
- if ( $this->iter ) {
- ( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter );
- }
+ public function rewind() {
+ parent::rewind();
+ $this->multiShardPaths = array();
}
/**
* Get the list for a given container shard
*
* @param string $container Resolved container name
- * @param string $dir Resolved path relative to container
- * @param array $params
- * @return Traversable|Array|null
+ * @return Iterator
*/
- abstract protected function listFromShard( $container, $dir, array $params );
+ abstract protected function listFromShard( $container );
}
/**
* Iterator for listing directories
*/
class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
- /**
- * @see FileBackendStoreShardListIterator::listFromShard()
- * @return Array|null|Traversable
- */
- protected function listFromShard( $container, $dir, array $params ) {
- return $this->backend->getDirectoryListInternal( $container, $dir, $params );
+ protected function listFromShard( $container ) {
+ $list = $this->backend->getDirectoryListInternal(
+ $container, $this->directory, $this->params );
+ if ( $list === null ) {
+ return new ArrayIterator( array() );
+ } else {
+ return is_array( $list ) ? new ArrayIterator( $list ) : $list;
+ }
}
}
@@ -1960,11 +1726,13 @@ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator
* Iterator for listing regular files
*/
class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
- /**
- * @see FileBackendStoreShardListIterator::listFromShard()
- * @return Array|null|Traversable
- */
- protected function listFromShard( $container, $dir, array $params ) {
- return $this->backend->getFileListInternal( $container, $dir, $params );
+ protected function listFromShard( $container ) {
+ $list = $this->backend->getFileListInternal(
+ $container, $this->directory, $this->params );
+ if ( $list === null ) {
+ return new ArrayIterator( array() );
+ } else {
+ return is_array( $list ) ? new ArrayIterator( $list ) : $list;
+ }
}
}
diff --git a/includes/filebackend/FileOp.php b/includes/filebackend/FileOp.php
index bb0ab578..fe833084 100644
--- a/includes/filebackend/FileOp.php
+++ b/includes/filebackend/FileOp.php
@@ -46,7 +46,7 @@ abstract class FileOp {
protected $doOperation = true; // boolean; operation is not a no-op
protected $sourceSha1; // string
- protected $destSameAsSource; // boolean
+ protected $overwriteSameCase; // boolean
protected $destExists; // boolean
/* Object life-cycle */
@@ -55,17 +55,19 @@ abstract class FileOp {
const STATE_ATTEMPTED = 3;
/**
- * Build a new file operation transaction
+ * Build a new batch file operation transaction
*
- * @param $backend FileBackendStore
- * @param $params Array
+ * @param FileBackendStore $backend
+ * @param Array $params
* @throws MWException
*/
final public function __construct( FileBackendStore $backend, array $params ) {
$this->backend = $backend;
list( $required, $optional ) = $this->allowedParams();
+ // @todo normalizeAnyStoragePaths() calls are overzealous, use a parameter list
foreach ( $required as $name ) {
if ( isset( $params[$name] ) ) {
+ // Normalize paths so the paths to the same file have the same string
$this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] );
} else {
throw new MWException( "File operation missing parameter '$name'." );
@@ -73,6 +75,7 @@ abstract class FileOp {
}
foreach ( $optional as $name ) {
if ( isset( $params[$name] ) ) {
+ // Normalize paths so the paths to the same file have the same string
$this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] );
}
}
@@ -82,7 +85,7 @@ abstract class FileOp {
/**
* Normalize $item or anything in $item that is a valid storage path
*
- * @param $item string|array
+ * @param string $item|array
* @return string|Array
*/
protected function normalizeAnyStoragePaths( $item ) {
@@ -102,7 +105,7 @@ abstract class FileOp {
/**
* Normalize a string if it is a valid storage path
*
- * @param $path string
+ * @param string $path
* @return string
*/
protected static function normalizeIfValidStoragePath( $path ) {
@@ -116,7 +119,7 @@ abstract class FileOp {
/**
* Set the batch UUID this operation belongs to
*
- * @param $batchId string
+ * @param string $batchId
* @return void
*/
final public function setBatchId( $batchId ) {
@@ -126,7 +129,7 @@ abstract class FileOp {
/**
* Get the value of the parameter with the given name
*
- * @param $name string
+ * @param string $name
* @return mixed Returns null if the parameter is not set
*/
final public function getParam( $name ) {
@@ -209,22 +212,22 @@ abstract class FileOp {
$pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
foreach ( array_unique( $pathsUsed ) as $path ) {
$nullEntries[] = array( // assertion for recovery
- 'op' => 'null',
- 'path' => $path,
+ 'op' => 'null',
+ 'path' => $path,
'newSha1' => $this->fileSha1( $path, $oPredicates )
);
}
foreach ( $this->storagePathsChanged() as $path ) {
if ( $nPredicates['sha1'][$path] === false ) { // deleted
$deleteEntries[] = array(
- 'op' => 'delete',
- 'path' => $path,
+ 'op' => 'delete',
+ 'path' => $path,
'newSha1' => ''
);
} else { // created/updated
$updateEntries[] = array(
- 'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create',
- 'path' => $path,
+ 'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create',
+ 'path' => $path,
'newSha1' => $nPredicates['sha1'][$path]
);
}
@@ -237,7 +240,7 @@ abstract class FileOp {
* This must update $predicates for each path that the op can change
* except when a failing status object is returned.
*
- * @param $predicates Array
+ * @param Array $predicates
* @return Status
*/
final public function precheck( array &$predicates ) {
@@ -314,7 +317,7 @@ abstract class FileOp {
/**
* Adjust params to FileBackendStore internal file calls
*
- * @param $params Array
+ * @param Array $params
* @return Array (required params list, optional params list)
*/
protected function setFlags( array $params ) {
@@ -341,10 +344,10 @@ abstract class FileOp {
/**
* Check for errors with regards to the destination file already existing.
- * Also set the destExists, destSameAsSource and sourceSha1 member variables.
+ * Also set the destExists, overwriteSameCase and sourceSha1 member variables.
* A bad status will be returned if there is no chance it can be overwritten.
*
- * @param $predicates Array
+ * @param Array $predicates
* @return Status
*/
protected function precheckDestExistence( array $predicates ) {
@@ -354,7 +357,7 @@ abstract class FileOp {
if ( $this->sourceSha1 === null ) { // file in storage?
$this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
}
- $this->destSameAsSource = false;
+ $this->overwriteSameCase = false;
$this->destExists = $this->fileExists( $this->params['dst'], $predicates );
if ( $this->destExists ) {
if ( $this->getParam( 'overwrite' ) ) {
@@ -368,7 +371,7 @@ abstract class FileOp {
// Give an error if the files are not identical
$status->fatal( 'backend-fail-notsame', $this->params['dst'] );
} else {
- $this->destSameAsSource = true; // OK
+ $this->overwriteSameCase = true; // OK
}
return $status; // do nothing; either OK or bad status
} else {
@@ -381,7 +384,7 @@ abstract class FileOp {
/**
* precheckDestExistence() helper function to get the source file SHA-1.
- * Subclasses should overwride this iff the source is not in storage.
+ * Subclasses should overwride this if the source is not in storage.
*
* @return string|bool Returns false on failure
*/
@@ -393,7 +396,7 @@ abstract class FileOp {
* Check if a file will exist in storage when this operation is attempted
*
* @param string $source Storage path
- * @param $predicates Array
+ * @param Array $predicates
* @return bool
*/
final protected function fileExists( $source, array $predicates ) {
@@ -409,7 +412,7 @@ abstract class FileOp {
* Get the SHA-1 of a file in storage when this operation is attempted
*
* @param string $source Storage path
- * @param $predicates Array
+ * @param Array $predicates
* @return string|bool False on failure
*/
final protected function fileSha1( $source, array $predicates ) {
@@ -435,7 +438,7 @@ abstract class FileOp {
/**
* Log a file operation failure and preserve any temp files
*
- * @param $action string
+ * @param string $action
* @return void
*/
final public function logFailure( $action ) {
@@ -457,7 +460,7 @@ abstract class FileOp {
class CreateFileOp extends FileOp {
protected function allowedParams() {
return array( array( 'content', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) );
+ array( 'overwrite', 'overwriteSame', 'headers' ) );
}
protected function doPrecheck( array &$predicates ) {
@@ -485,27 +488,18 @@ class CreateFileOp extends FileOp {
return $status; // safe to call attempt()
}
- /**
- * @return Status
- */
protected function doAttempt() {
- if ( !$this->destSameAsSource ) {
+ if ( !$this->overwriteSameCase ) {
// Create the file at the destination
return $this->backend->createInternal( $this->setFlags( $this->params ) );
}
return Status::newGood();
}
- /**
- * @return bool|String
- */
protected function getSourceSha1Base36() {
return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
}
- /**
- * @return array
- */
public function storagePathsChanged() {
return array( $this->params['dst'] );
}
@@ -516,18 +510,11 @@ class CreateFileOp extends FileOp {
* Parameters for this operation are outlined in FileBackend::doOperations().
*/
class StoreFileOp extends FileOp {
- /**
- * @return array
- */
protected function allowedParams() {
return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) );
+ array( 'overwrite', 'overwriteSame', 'headers' ) );
}
- /**
- * @param $predicates array
- * @return Status
- */
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source file exists on the file system
@@ -557,20 +544,14 @@ class StoreFileOp extends FileOp {
return $status; // safe to call attempt()
}
- /**
- * @return Status
- */
protected function doAttempt() {
- // Store the file at the destination
- if ( !$this->destSameAsSource ) {
+ if ( !$this->overwriteSameCase ) {
+ // Store the file at the destination
return $this->backend->storeInternal( $this->setFlags( $this->params ) );
}
return Status::newGood();
}
- /**
- * @return bool|string
- */
protected function getSourceSha1Base36() {
wfSuppressWarnings();
$hash = sha1_file( $this->params['src'] );
@@ -591,18 +572,11 @@ class StoreFileOp extends FileOp {
* Parameters for this operation are outlined in FileBackend::doOperations().
*/
class CopyFileOp extends FileOp {
- /**
- * @return array
- */
protected function allowedParams() {
return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
+ array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) );
}
- /**
- * @param $predicates array
- * @return Status
- */
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source file exists
@@ -634,30 +608,26 @@ class CopyFileOp extends FileOp {
return $status; // safe to call attempt()
}
- /**
- * @return Status
- */
protected function doAttempt() {
- // Do nothing if the src/dst paths are the same
- if ( $this->params['src'] !== $this->params['dst'] ) {
- // Copy the file into the destination
- if ( !$this->destSameAsSource ) {
- return $this->backend->copyInternal( $this->setFlags( $this->params ) );
- }
+ if ( $this->overwriteSameCase ) {
+ $status = Status::newGood(); // nothing to do
+ } elseif ( $this->params['src'] === $this->params['dst'] ) {
+ // Just update the destination file headers
+ $headers = $this->getParam( 'headers' ) ?: array();
+ $status = $this->backend->describeInternal( $this->setFlags( array(
+ 'src' => $this->params['dst'], 'headers' => $headers
+ ) ) );
+ } else {
+ // Copy the file to the destination
+ $status = $this->backend->copyInternal( $this->setFlags( $this->params ) );
}
- return Status::newGood();
+ return $status;
}
- /**
- * @return array
- */
public function storagePathsRead() {
return array( $this->params['src'] );
}
- /**
- * @return array
- */
public function storagePathsChanged() {
return array( $this->params['dst'] );
}
@@ -668,18 +638,11 @@ class CopyFileOp extends FileOp {
* Parameters for this operation are outlined in FileBackend::doOperations().
*/
class MoveFileOp extends FileOp {
- /**
- * @return array
- */
protected function allowedParams() {
return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
+ array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) );
}
- /**
- * @param $predicates array
- * @return Status
- */
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source file exists
@@ -713,34 +676,34 @@ class MoveFileOp extends FileOp {
return $status; // safe to call attempt()
}
- /**
- * @return Status
- */
protected function doAttempt() {
- // Do nothing if the src/dst paths are the same
- if ( $this->params['src'] !== $this->params['dst'] ) {
- if ( !$this->destSameAsSource ) {
- // Move the file into the destination
- return $this->backend->moveInternal( $this->setFlags( $this->params ) );
+ if ( $this->overwriteSameCase ) {
+ if ( $this->params['src'] === $this->params['dst'] ) {
+ // Do nothing to the destination (which is also the source)
+ $status = Status::newGood();
} else {
- // Just delete source as the destination needs no changes
- $params = array( 'src' => $this->params['src'] );
- return $this->backend->deleteInternal( $this->setFlags( $params ) );
+ // Just delete the source as the destination file needs no changes
+ $status = $this->backend->deleteInternal( $this->setFlags(
+ array( 'src' => $this->params['src'] )
+ ) );
}
+ } elseif ( $this->params['src'] === $this->params['dst'] ) {
+ // Just update the destination file headers
+ $headers = $this->getParam( 'headers' ) ?: array();
+ $status = $this->backend->describeInternal( $this->setFlags(
+ array( 'src' => $this->params['dst'], 'headers' => $headers )
+ ) );
+ } else {
+ // Move the file to the destination
+ $status = $this->backend->moveInternal( $this->setFlags( $this->params ) );
}
- return Status::newGood();
+ return $status;
}
- /**
- * @return array
- */
public function storagePathsRead() {
return array( $this->params['src'] );
}
- /**
- * @return array
- */
public function storagePathsChanged() {
return array( $this->params['src'], $this->params['dst'] );
}
@@ -751,17 +714,10 @@ class MoveFileOp extends FileOp {
* Parameters for this operation are outlined in FileBackend::doOperations().
*/
class DeleteFileOp extends FileOp {
- /**
- * @return array
- */
protected function allowedParams() {
return array( array( 'src' ), array( 'ignoreMissingSource' ) );
}
- /**
- * @param $predicates array
- * @return Status
- */
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source file exists
@@ -788,17 +744,11 @@ class DeleteFileOp extends FileOp {
return $status; // safe to call attempt()
}
- /**
- * @return Status
- */
protected function doAttempt() {
// Delete the source file
return $this->backend->deleteInternal( $this->setFlags( $this->params ) );
}
- /**
- * @return array
- */
public function storagePathsChanged() {
return array( $this->params['src'] );
}
@@ -809,17 +759,10 @@ class DeleteFileOp extends FileOp {
* Parameters for this operation are outlined in FileBackend::doOperations().
*/
class DescribeFileOp extends FileOp {
- /**
- * @return array
- */
protected function allowedParams() {
- return array( array( 'src' ), array( 'disposition', 'headers' ) );
+ return array( array( 'src' ), array( 'headers' ) );
}
- /**
- * @param $predicates array
- * @return Status
- */
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source file exists
@@ -840,17 +783,11 @@ class DescribeFileOp extends FileOp {
return $status; // safe to call attempt()
}
- /**
- * @return Status
- */
protected function doAttempt() {
// Update the source file's metadata
return $this->backend->describeInternal( $this->setFlags( $this->params ) );
}
- /**
- * @return array
- */
public function storagePathsChanged() {
return array( $this->params['src'] );
}
diff --git a/includes/filebackend/FileOpBatch.php b/includes/filebackend/FileOpBatch.php
index fc51d78a..785c0bc9 100644
--- a/includes/filebackend/FileOpBatch.php
+++ b/includes/filebackend/FileOpBatch.php
@@ -51,7 +51,7 @@ class FileOpBatch {
*
* @param array $performOps List of FileOp operations
* @param array $opts Batch operation options
- * @param $journal FileJournal Journal to log operations to
+ * @param FileJournal $journal Journal to log operations to
* @return Status
*/
public static function attempt( array $performOps, array $opts, FileJournal $journal ) {
@@ -145,8 +145,8 @@ class FileOpBatch {
* within any given sub-batch do not depend on each other.
* This will abort remaining ops on failure.
*
- * @param $pPerformOps Array
- * @param $status Status
+ * @param Array $pPerformOps
+ * @param Status $status
* @return bool Success
*/
protected static function runParallelBatches( array $pPerformOps, Status $status ) {
diff --git a/includes/filebackend/README b/includes/filebackend/README
index 6ab54810..569f3376 100644
--- a/includes/filebackend/README
+++ b/includes/filebackend/README
@@ -47,7 +47,7 @@ directories. See FileBackend.php for full documentation for each function.
The following basic operations are supported for reading from a backend:
On files:
-* state a file for basic information (timestamp, size)
+* stat a file for basic information (timestamp, size)
* read a file into a string or several files into a map of path names to strings
* download a file or set of files to a temporary file (on a mounted file system)
* get the SHA1 hash of a file
diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php
index 0f3d97a3..db090a98 100644
--- a/includes/filebackend/SwiftFileBackend.php
+++ b/includes/filebackend/SwiftFileBackend.php
@@ -24,7 +24,7 @@
*/
/**
- * @brief Class for an OpenStack Swift based file backend.
+ * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend.
*
* This requires the SwiftCloudFiles MediaWiki extension, which includes
* the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
@@ -104,7 +104,7 @@ class SwiftFileBackend extends FileBackendStore {
*/
public function __construct( array $config ) {
parent::__construct( $config );
- if ( !MWInit::classExists( 'CF_Constants' ) ) {
+ if ( !class_exists( 'CF_Constants' ) ) {
throw new MWException( 'SwiftCloudFiles extension not installed.' );
}
// Required settings
@@ -132,7 +132,7 @@ class SwiftFileBackend extends FileBackendStore {
: false;
$this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
? $config['swiftCDNExpiry']
- : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
+ : 12 * 3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
$this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
? $config['swiftCDNPurgable']
: true;
@@ -172,10 +172,6 @@ class SwiftFileBackend extends FileBackendStore {
return $relStoragePath;
}
- /**
- * @see FileBackendStore::isPathUsableInternal()
- * @return bool
- */
public function isPathUsableInternal( $storagePath ) {
list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
if ( $rel === null ) {
@@ -194,6 +190,18 @@ class SwiftFileBackend extends FileBackendStore {
}
/**
+ * @param array $headers
+ * @return array
+ */
+ protected function sanitizeHdrs( array $headers ) {
+ // By default, Swift has annoyingly low maximum header value limits
+ if ( isset( $headers['Content-Disposition'] ) ) {
+ $headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] );
+ }
+ return $headers;
+ }
+
+ /**
* @param string $disposition Content-Disposition header value
* @return string Truncated Content-Disposition header value to meet Swift limits
*/
@@ -211,10 +219,6 @@ class SwiftFileBackend extends FileBackendStore {
return $res;
}
- /**
- * @see FileBackendStore::doCreateInternal()
- * @return Status
- */
protected function doCreateInternal( array $params ) {
$status = Status::newGood();
@@ -248,17 +252,10 @@ class SwiftFileBackend extends FileBackendStore {
// The MD5 here will be checked within Swift against its own MD5.
$obj->set_etag( md5( $params['content'] ) );
// Use the same content type as StreamFile for security
- $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
- if ( !strlen( $obj->content_type ) ) { // special case
- $obj->content_type = 'unknown/unknown';
- }
- // Set the Content-Disposition header if requested
- if ( isset( $params['disposition'] ) ) {
- $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
- }
+ $obj->content_type = $this->getContentType( $params['dst'], $params['content'], null );
// Set any other custom headers if requested
if ( isset( $params['headers'] ) ) {
- $obj->headers += $params['headers'];
+ $obj->headers += $this->sanitizeHdrs( $params['headers'] );
}
if ( !empty( $params['async'] ) ) { // deferred
$op = $obj->write_async( $params['content'] );
@@ -290,10 +287,6 @@ class SwiftFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doStoreInternal()
- * @return Status
- */
protected function doStoreInternal( array $params ) {
$status = Status::newGood();
@@ -333,17 +326,10 @@ class SwiftFileBackend extends FileBackendStore {
// The MD5 here will be checked within Swift against its own MD5.
$obj->set_etag( md5_file( $params['src'] ) );
// Use the same content type as StreamFile for security
- $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
- if ( !strlen( $obj->content_type ) ) { // special case
- $obj->content_type = 'unknown/unknown';
- }
- // Set the Content-Disposition header if requested
- if ( isset( $params['disposition'] ) ) {
- $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
- }
+ $obj->content_type = $this->getContentType( $params['dst'], null, $params['src'] );
// Set any other custom headers if requested
if ( isset( $params['headers'] ) ) {
- $obj->headers += $params['headers'];
+ $obj->headers += $this->sanitizeHdrs( $params['headers'] );
}
if ( !empty( $params['async'] ) ) { // deferred
wfSuppressWarnings();
@@ -387,10 +373,6 @@ class SwiftFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doCopyInternal()
- * @return Status
- */
protected function doCopyInternal( array $params ) {
$status = Status::newGood();
@@ -424,8 +406,9 @@ class SwiftFileBackend extends FileBackendStore {
try {
$dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
$hdrs = array(); // source file headers to override with new values
- if ( isset( $params['disposition'] ) ) {
- $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
+ // Set any other custom headers if requested
+ if ( isset( $params['headers'] ) ) {
+ $hdrs += $this->sanitizeHdrs( $params['headers'] );
}
if ( !empty( $params['async'] ) ) { // deferred
$op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
@@ -459,10 +442,6 @@ class SwiftFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doMoveInternal()
- * @return Status
- */
protected function doMoveInternal( array $params ) {
$status = Status::newGood();
@@ -497,8 +476,9 @@ class SwiftFileBackend extends FileBackendStore {
$srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
$dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
$hdrs = array(); // source file headers to override with new values
- if ( isset( $params['disposition'] ) ) {
- $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
+ // Set any other custom headers if requested
+ if ( isset( $params['headers'] ) ) {
+ $hdrs += $this->sanitizeHdrs( $params['headers'] );
}
if ( !empty( $params['async'] ) ) { // deferred
$op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
@@ -534,10 +514,6 @@ class SwiftFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doDeleteInternal()
- * @return Status
- */
protected function doDeleteInternal( array $params ) {
$status = Status::newGood();
@@ -590,10 +566,6 @@ class SwiftFileBackend extends FileBackendStore {
}
}
- /**
- * @see FileBackendStore::doDescribeInternal()
- * @return Status
- */
protected function doDescribeInternal( array $params ) {
$status = Status::newGood();
@@ -603,19 +575,15 @@ class SwiftFileBackend extends FileBackendStore {
return $status;
}
- $hdrs = isset( $params['headers'] ) ? $params['headers'] : array();
- // Set the Content-Disposition header if requested
- if ( isset( $params['disposition'] ) ) {
- $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
- }
-
try {
$sContObj = $this->getContainer( $srcCont );
// Get the latest version of the current metadata
$srcObj = $sContObj->get_object( $srcRel,
$this->headersFromParams( array( 'latest' => true ) ) );
// Merge in the metadata updates...
- $srcObj->headers = $hdrs + $srcObj->headers;
+ if ( isset( $params['headers'] ) ) {
+ $srcObj->headers = $this->sanitizeHdrs( $params['headers'] ) + $srcObj->headers;
+ }
$srcObj->sync_metadata(); // save to Swift
$this->purgeCDNCache( array( $srcObj ) );
} catch ( CDNNotEnabledException $e ) {
@@ -631,10 +599,6 @@ class SwiftFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doPrepareInternal()
- * @return Status
- */
protected function doPrepareInternal( $fullCont, $dir, array $params ) {
$status = Status::newGood();
@@ -748,10 +712,6 @@ class SwiftFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doCleanInternal()
- * @return Status
- */
protected function doCleanInternal( $fullCont, $dir, array $params ) {
$status = Status::newGood();
@@ -787,10 +747,6 @@ class SwiftFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doFileExists()
- * @return array|bool|null
- */
protected function doGetFileStat( array $params ) {
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
@@ -805,8 +761,8 @@ class SwiftFileBackend extends FileBackendStore {
$stat = array(
// Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
- 'size' => (int)$srcObj->content_length,
- 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' )
+ 'size' => (int)$srcObj->content_length,
+ 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' )
);
} catch ( NoSuchContainerException $e ) {
} catch ( NoSuchObjectException $e ) {
@@ -821,7 +777,7 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Fill in any missing object metadata and save it to Swift
*
- * @param $obj CF_Object
+ * @param CF_Object $obj
* @param string $path Storage path to object
* @return bool Success
* @throws Exception cloudfiles exceptions
@@ -852,10 +808,6 @@ class SwiftFileBackend extends FileBackendStore {
return false; // failed
}
- /**
- * @see FileBackendStore::doGetFileContentsMulti()
- * @return Array
- */
protected function doGetFileContentsMulti( array $params ) {
$contents = array();
@@ -965,24 +917,25 @@ class SwiftFileBackend extends FileBackendStore {
* @param string $fullCont Resolved container name
* @param string $dir Resolved storage directory with no trailing slash
* @param string|null $after Storage path of file to list items after
- * @param $limit integer Max number of items to list
- * @param array $params Includes flag for 'topOnly'
- * @return Array List of relative paths of dirs directly under $dir
+ * @param integer $limit Max number of items to list
+ * @param array $params Parameters for getDirectoryList()
+ * @return Array List of resolved paths of directories directly under $dir
+ * @throws FileBackendError
*/
public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
$dirs = array();
if ( $after === INF ) {
return $dirs; // nothing more
}
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . '-' . $this->name );
try {
$container = $this->getContainer( $fullCont );
$prefix = ( $dir == '' ) ? null : "{$dir}/";
// Non-recursive: only list dirs right under $dir
if ( !empty( $params['topOnly'] ) ) {
$objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
- foreach ( $objects as $object ) { // files and dirs
+ foreach ( $objects as $object ) { // files and directories
if ( substr( $object, -1 ) === '/' ) {
$dirs[] = $object; // directories end in '/'
}
@@ -1013,6 +966,7 @@ class SwiftFileBackend extends FileBackendStore {
}
}
}
+ // Page on the unfiltered directory listing (what is returned may be filtered)
if ( count( $objects ) < $limit ) {
$after = INF; // avoid a second RTT
} else {
@@ -1022,9 +976,9 @@ class SwiftFileBackend extends FileBackendStore {
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, null, __METHOD__,
array( 'cont' => $fullCont, 'dir' => $dir ) );
+ throw new FileBackendError( "Got " . get_class( $e ) . " exception." );
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
return $dirs;
}
@@ -1038,33 +992,49 @@ class SwiftFileBackend extends FileBackendStore {
* @param string $fullCont Resolved container name
* @param string $dir Resolved storage directory with no trailing slash
* @param string|null $after Storage path of file to list items after
- * @param $limit integer Max number of items to list
- * @param array $params Includes flag for 'topOnly'
- * @return Array List of relative paths of files under $dir
+ * @param integer $limit Max number of items to list
+ * @param array $params Parameters for getDirectoryList()
+ * @return Array List of resolved paths of files under $dir
+ * @throws FileBackendError
*/
public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
$files = array();
if ( $after === INF ) {
return $files; // nothing more
}
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . '-' . $this->name );
try {
$container = $this->getContainer( $fullCont );
$prefix = ( $dir == '' ) ? null : "{$dir}/";
// Non-recursive: only list files right under $dir
if ( !empty( $params['topOnly'] ) ) { // files and dirs
- $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
- foreach ( $objects as $object ) {
- if ( substr( $object, -1 ) !== '/' ) {
- $files[] = $object; // directories end in '/'
+ if ( !empty( $params['adviseStat'] ) ) {
+ $limit = min( $limit, self::CACHE_CHEAP_SIZE );
+ // Note: get_objects() does not include directories
+ $objects = $this->loadObjectListing( $params, $dir,
+ $container->get_objects( $limit, $after, $prefix, null, '/' ) );
+ $files = $objects;
+ } else {
+ $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
+ foreach ( $objects as $object ) { // files and directories
+ if ( substr( $object, -1 ) !== '/' ) {
+ $files[] = $object; // directories end in '/'
+ }
}
}
// Recursive: list all files under $dir and its subdirs
} else { // files
- $objects = $container->list_objects( $limit, $after, $prefix );
+ if ( !empty( $params['adviseStat'] ) ) {
+ $limit = min( $limit, self::CACHE_CHEAP_SIZE );
+ $objects = $this->loadObjectListing( $params, $dir,
+ $container->get_objects( $limit, $after, $prefix ) );
+ } else {
+ $objects = $container->list_objects( $limit, $after, $prefix );
+ }
$files = $objects;
}
+ // Page on the unfiltered object listing (what is returned may be filtered)
if ( count( $objects ) < $limit ) {
$after = INF; // avoid a second RTT
} else {
@@ -1074,29 +1044,57 @@ class SwiftFileBackend extends FileBackendStore {
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, null, __METHOD__,
array( 'cont' => $fullCont, 'dir' => $dir ) );
+ throw new FileBackendError( "Got " . get_class( $e ) . " exception." );
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
return $files;
}
/**
- * @see FileBackendStore::doGetFileSha1base36()
- * @return bool
+ * Load a list of objects that belong under $dir into stat cache
+ * and return a list of the names of the objects in the same order.
+ *
+ * @param array $params Parameters for getDirectoryList()
+ * @param string $dir Resolved container directory path
+ * @param array $cfObjects List of CF_Object items
+ * @return array List of object names
*/
+ private function loadObjectListing( array $params, $dir, array $cfObjects ) {
+ $names = array();
+ $storageDir = rtrim( $params['dir'], '/' );
+ $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/"
+ // Iterate over the list *backwards* as this primes the stat cache, which is LRU.
+ // If this fills the cache and the caller stats an uncached file before stating
+ // the ones on the listing, there would be zero cache hits if this went forwards.
+ for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) {
+ $object = current( $cfObjects );
+ $path = "{$storageDir}/" . substr( $object->name, $suffixStart );
+ $val = array(
+ // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
+ 'mtime' => wfTimestamp( TS_MW, $object->last_modified ),
+ 'size' => (int)$object->content_length,
+ 'latest' => false // eventually consistent
+ );
+ $this->cheapCache->set( $path, 'stat', $val );
+ $names[] = $object->name;
+ }
+ return array_reverse( $names ); // keep the paths in original order
+ }
+
protected function doGetFileSha1base36( array $params ) {
$stat = $this->getFileStat( $params );
if ( $stat ) {
+ if ( !isset( $stat['sha1'] ) ) {
+ // Stat entries filled by file listings don't include SHA1
+ $this->clearCache( array( $params['src'] ) );
+ $stat = $this->getFileStat( $params );
+ }
return $stat['sha1'];
} else {
return false;
}
}
- /**
- * @see FileBackendStore::doStreamFile()
- * @return Status
- */
protected function doStreamFile( array $params ) {
$status = Status::newGood();
@@ -1128,10 +1126,6 @@ class SwiftFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FileBackendStore::doGetLocalCopyMulti()
- * @return null|TempFSFile
- */
protected function doGetLocalCopyMulti( array $params ) {
$tmpFiles = array();
@@ -1201,10 +1195,6 @@ class SwiftFileBackend extends FileBackendStore {
return $tmpFiles;
}
- /**
- * @see FileBackendStore::getFileHttpUrl()
- * @return string|null
- */
public function getFileHttpUrl( array $params ) {
if ( $this->swiftTempUrlKey != '' ||
( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
@@ -1237,8 +1227,8 @@ class SwiftFileBackend extends FileBackendStore {
str_replace( '/swift/v1', '', // S3 API is the rgw default
$sContObj->cfs_http->getStorageUrl() . $spath ),
array(
- 'Signature' => $signature,
- 'Expires' => $expires,
+ 'Signature' => $signature,
+ 'Expires' => $expires,
'AWSAccessKeyId' => $this->rgwS3AccessKey )
);
}
@@ -1250,10 +1240,6 @@ class SwiftFileBackend extends FileBackendStore {
return null;
}
- /**
- * @see FileBackendStore::directoriesAreVirtual()
- * @return bool
- */
protected function directoriesAreVirtual() {
return true;
}
@@ -1274,10 +1260,6 @@ class SwiftFileBackend extends FileBackendStore {
return $hdrs;
}
- /**
- * @see FileBackendStore::doExecuteOpHandlesInternal()
- * @return Array List of corresponding Status objects
- */
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
$statuses = array();
@@ -1314,7 +1296,7 @@ class SwiftFileBackend extends FileBackendStore {
* matches the expression and the request is not for a listing.
* Setting this to '*' effectively makes a container public.
* -".rlistings:<regex>" : Grants access if the request is from a referrer host that
- * matches the expression and the request for a listing.
+ * matches the expression and the request is for a listing.
*
* $writeGrps is a list of the possible criteria for a request to have
* access to write to a container. Each item is of the following format:
@@ -1325,7 +1307,7 @@ class SwiftFileBackend extends FileBackendStore {
* In general, we don't allow listings to end-users. It's not useful, isn't well-defined
* (lists are truncated to 10000 item with no way to page), and is just a performance risk.
*
- * @param $contObj CF_Container Swift container
+ * @param CF_Container $contObj Swift container
* @param array $readGrps List of read access routes
* @param array $writeGrps List of write access routes
* @return Status
@@ -1395,12 +1377,12 @@ class SwiftFileBackend extends FileBackendStore {
if ( is_array( $creds ) ) { // cache hit
$this->auth->load_cached_credentials(
$creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
- $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
+ $this->sessionStarted = time() - ceil( $this->authTTL / 2 ); // skew for worst case
} else { // cache miss
try {
$this->auth->authenticate();
$creds = $this->auth->export_credentials();
- $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
+ $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL / 2 ) ); // cache
$this->sessionStarted = time();
} catch ( CloudFilesException $e ) {
$this->connException = $e; // don't keep re-trying
@@ -1433,7 +1415,7 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Get the cache key for a container
*
- * @param $username string
+ * @param string $username
* @return string
*/
private function getCredsCacheKey( $username ) {
@@ -1496,10 +1478,6 @@ class SwiftFileBackend extends FileBackendStore {
$conn->delete_container( $container );
}
- /**
- * @see FileBackendStore::doPrimeContainerCache()
- * @return void
- */
protected function doPrimeContainerCache( array $containerInfo ) {
try {
$conn = $this->getConnection(); // Swift proxy connection
@@ -1517,9 +1495,9 @@ class SwiftFileBackend extends FileBackendStore {
* Log an unexpected exception for this backend.
* This also sets the Status object to have a fatal error.
*
- * @param $e Exception
- * @param $status Status|null
- * @param $func string
+ * @param Exception $e
+ * @param Status $status|null
+ * @param string $func
* @param array $params
* @return void
*/
@@ -1554,7 +1532,15 @@ class SwiftFileOpHandle extends FileBackendStoreOpHandle {
/** @var Array */
public $affectedObjects = array();
- public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
+ /**
+ * @param SwiftFileBackend $backend
+ * @param array $params
+ * @param string $call
+ * @param CF_Async_Op $cfOp
+ */
+ public function __construct(
+ SwiftFileBackend $backend, array $params, $call, CF_Async_Op $cfOp
+ ) {
$this->backend = $backend;
$this->params = $params;
$this->call = $call;
@@ -1586,7 +1572,7 @@ abstract class SwiftFileBackendList implements Iterator {
const PAGE_SIZE = 9000; // file listing buffer size
/**
- * @param $backend SwiftFileBackend
+ * @param SwiftFileBackend $backend
* @param string $fullCont Resolved container name
* @param string $dir Resolved directory relative to container
* @param array $params
@@ -1660,10 +1646,10 @@ abstract class SwiftFileBackendList implements Iterator {
*
* @param string $container Resolved container name
* @param string $dir Resolved path relative to container
- * @param $after string|null
- * @param $limit integer
+ * @param string $after|null
+ * @param integer $limit
* @param array $params
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|Array
*/
abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
}
@@ -1682,7 +1668,7 @@ class SwiftFileBackendDirList extends SwiftFileBackendList {
/**
* @see SwiftFileBackendList::pageFromList()
- * @return Array|null
+ * @return Array
*/
protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
@@ -1703,7 +1689,7 @@ class SwiftFileBackendFileList extends SwiftFileBackendList {
/**
* @see SwiftFileBackendList::pageFromList()
- * @return Array|null
+ * @return Array
*/
protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php
index 11e125c1..8266e420 100644
--- a/includes/filebackend/TempFSFile.php
+++ b/includes/filebackend/TempFSFile.php
@@ -37,8 +37,8 @@ class TempFSFile extends FSFile {
* Make a new temporary file on the file system.
* Temporary files may be purged when the file object falls out of scope.
*
- * @param $prefix string
- * @param $extension string
+ * @param string $prefix
+ * @param string $extension
* @return TempFSFile|null
*/
public static function factory( $prefix, $extension = '' ) {
@@ -81,7 +81,7 @@ class TempFSFile extends FSFile {
/**
* Clean up the temporary file only after an object goes out of scope
*
- * @param $object Object
+ * @param Object $object
* @return TempFSFile This object
*/
public function bind( $object ) {
diff --git a/includes/filebackend/filejournal/DBFileJournal.php b/includes/filebackend/filejournal/DBFileJournal.php
index 73f29a95..9250aa5e 100644
--- a/includes/filebackend/filejournal/DBFileJournal.php
+++ b/includes/filebackend/filejournal/DBFileJournal.php
@@ -65,11 +65,11 @@ class DBFileJournal extends FileJournal {
foreach ( $entries as $entry ) {
$data[] = array(
'fj_batch_uuid' => $batchId,
- 'fj_backend' => $this->backend,
- 'fj_op' => $entry['op'],
- 'fj_path' => $entry['path'],
- 'fj_new_sha1' => $entry['newSha1'],
- 'fj_timestamp' => $dbw->timestamp( $now )
+ 'fj_backend' => $this->backend,
+ 'fj_op' => $entry['op'],
+ 'fj_path' => $entry['path'],
+ 'fj_new_sha1' => $entry['newSha1'],
+ 'fj_timestamp' => $dbw->timestamp( $now )
);
}
diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php
index f02387dc..3e934ba5 100644
--- a/includes/filebackend/lockmanager/DBLockManager.php
+++ b/includes/filebackend/lockmanager/DBLockManager.php
@@ -110,6 +110,19 @@ abstract class DBLockManager extends QuorumLockManager {
$this->session = wfRandomString( 31 );
}
+ // @TODO: change this code to work in one batch
+ protected function getLocksOnServer( $lockSrv, array $pathsByType ) {
+ $status = Status::newGood();
+ foreach ( $pathsByType as $type => $paths ) {
+ $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) );
+ }
+ return $status;
+ }
+
+ protected function freeLocksOnServer( $lockSrv, array $pathsByType ) {
+ return Status::newGood();
+ }
+
/**
* @see QuorumLockManager::isServerUp()
* @return bool
@@ -252,7 +265,7 @@ class MySqlLockManager extends DBLockManager {
* @see DBLockManager::getLocksOnServer()
* @return Status
*/
- protected function getLocksOnServer( $lockSrv, array $paths, $type ) {
+ protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) {
$status = Status::newGood();
$db = $this->getConnection( $lockSrv ); // checked in isServerUp()
@@ -319,14 +332,6 @@ class MySqlLockManager extends DBLockManager {
}
/**
- * @see QuorumLockManager::freeLocksOnServer()
- * @return Status
- */
- protected function freeLocksOnServer( $lockSrv, array $paths, $type ) {
- return Status::newGood(); // not supported
- }
-
- /**
* @see QuorumLockManager::releaseAllLocks()
* @return Status
*/
@@ -361,7 +366,7 @@ class PostgreSqlLockManager extends DBLockManager {
self::LOCK_EX => self::LOCK_EX
);
- protected function getLocksOnServer( $lockSrv, array $paths, $type ) {
+ protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) {
$status = Status::newGood();
if ( !count( $paths ) ) {
return $status; // nothing to lock
@@ -369,7 +374,9 @@ class PostgreSqlLockManager extends DBLockManager {
$db = $this->getConnection( $lockSrv ); // checked in isServerUp()
$bigints = array_unique( array_map(
- function( $key ) { return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 ); },
+ function( $key ) {
+ return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 );
+ },
array_map( array( $this, 'sha1Base16Absolute' ), $paths )
) );
@@ -406,14 +413,6 @@ class PostgreSqlLockManager extends DBLockManager {
}
/**
- * @see QuorumLockManager::freeLocksOnServer()
- * @return Status
- */
- protected function freeLocksOnServer( $lockSrv, array $paths, $type ) {
- return Status::newGood(); // not supported
- }
-
- /**
* @see QuorumLockManager::releaseAllLocks()
* @return Status
*/
diff --git a/includes/filebackend/lockmanager/LockManager.php b/includes/filebackend/lockmanager/LockManager.php
index 0512a01b..dad8a624 100644
--- a/includes/filebackend/lockmanager/LockManager.php
+++ b/includes/filebackend/lockmanager/LockManager.php
@@ -56,7 +56,7 @@ abstract class LockManager {
protected $domain; // string; domain (usually wiki ID)
protected $lockTTL; // integer; maximum time locks can be held
- /* Lock types; stronger locks have higher values */
+ /** Lock types; stronger locks have higher values */
const LOCK_SH = 1; // shared lock (for reads)
const LOCK_UW = 2; // shared lock (for reads used to write elsewhere)
const LOCK_EX = 3; // exclusive lock (for writes)
@@ -76,10 +76,10 @@ abstract class LockManager {
if ( isset( $config['lockTTL'] ) ) {
$this->lockTTL = max( 1, $config['lockTTL'] );
} elseif ( PHP_SAPI === 'cli' ) {
- $this->lockTTL = 2*3600;
+ $this->lockTTL = 2 * 3600;
} else {
$met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode
- $this->lockTTL = max( 5*60, 2*(int)$met );
+ $this->lockTTL = max( 5 * 60, 2 * (int)$met );
}
}
@@ -88,11 +88,36 @@ abstract class LockManager {
*
* @param array $paths List of resource names
* @param $type integer LockManager::LOCK_* constant
+ * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21)
* @return Status
*/
- final public function lock( array $paths, $type = self::LOCK_EX ) {
+ final public function lock( array $paths, $type = self::LOCK_EX, $timeout = 0 ) {
+ return $this->lockByType( array( $type => $paths ), $timeout );
+ }
+
+ /**
+ * Lock the resources at the given abstract paths
+ *
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
+ * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21)
+ * @return Status
+ * @since 1.22
+ */
+ final public function lockByType( array $pathsByType, $timeout = 0 ) {
wfProfileIn( __METHOD__ );
- $status = $this->doLock( array_unique( $paths ), $this->lockTypeMap[$type] );
+ $status = Status::newGood();
+ $pathsByType = $this->normalizePathsByType( $pathsByType );
+ $msleep = array( 0, 50, 100, 300, 500 ); // retry backoff times
+ $start = microtime( true );
+ do {
+ $status = $this->doLockByType( $pathsByType );
+ $elapsed = microtime( true ) - $start;
+ if ( $status->isOK() || $elapsed >= $timeout || $elapsed < 0 ) {
+ break; // success, timeout, or clock set back
+ }
+ usleep( 1e3 * ( next( $msleep ) ?: 1000 ) ); // use 1 sec after enough times
+ $elapsed = microtime( true ) - $start;
+ } while ( $elapsed < $timeout && $elapsed >= 0 );
wfProfileOut( __METHOD__ );
return $status;
}
@@ -100,13 +125,25 @@ abstract class LockManager {
/**
* Unlock the resources at the given abstract paths
*
- * @param array $paths List of storage paths
+ * @param array $paths List of paths
* @param $type integer LockManager::LOCK_* constant
* @return Status
*/
final public function unlock( array $paths, $type = self::LOCK_EX ) {
+ return $this->unlockByType( array( $type => $paths ) );
+ }
+
+ /**
+ * Unlock the resources at the given abstract paths
+ *
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
+ * @return Status
+ * @since 1.22
+ */
+ final public function unlockByType( array $pathsByType ) {
wfProfileIn( __METHOD__ );
- $status = $this->doUnlock( array_unique( $paths ), $this->lockTypeMap[$type] );
+ $pathsByType = $this->normalizePathsByType( $pathsByType );
+ $status = $this->doUnlockByType( $pathsByType );
wfProfileOut( __METHOD__ );
return $status;
}
@@ -136,20 +173,74 @@ abstract class LockManager {
}
/**
+ * Normalize the $paths array by converting LOCK_UW locks into the
+ * appropriate type and removing any duplicated paths for each lock type.
+ *
+ * @param array $paths Map of LockManager::LOCK_* constants to lists of paths
+ * @return Array
+ * @since 1.22
+ */
+ final protected function normalizePathsByType( array $pathsByType ) {
+ $res = array();
+ foreach ( $pathsByType as $type => $paths ) {
+ $res[$this->lockTypeMap[$type]] = array_unique( $paths );
+ }
+ return $res;
+ }
+
+ /**
+ * @see LockManager::lockByType()
+ * @param array $paths Map of LockManager::LOCK_* constants to lists of paths
+ * @return Status
+ * @since 1.22
+ */
+ protected function doLockByType( array $pathsByType ) {
+ $status = Status::newGood();
+ $lockedByType = array(); // map of (type => paths)
+ foreach ( $pathsByType as $type => $paths ) {
+ $status->merge( $this->doLock( $paths, $type ) );
+ if ( $status->isOK() ) {
+ $lockedByType[$type] = $paths;
+ } else {
+ // Release the subset of locks that were acquired
+ foreach ( $lockedByType as $type => $paths ) {
+ $status->merge( $this->doUnlock( $paths, $type ) );
+ }
+ break;
+ }
+ }
+ return $status;
+ }
+
+ /**
* Lock resources with the given keys and lock type
*
- * @param array $paths List of storage paths
+ * @param array $paths List of paths
* @param $type integer LockManager::LOCK_* constant
- * @return string
+ * @return Status
*/
abstract protected function doLock( array $paths, $type );
/**
+ * @see LockManager::unlockByType()
+ * @param array $paths Map of LockManager::LOCK_* constants to lists of paths
+ * @return Status
+ * @since 1.22
+ */
+ protected function doUnlockByType( array $pathsByType ) {
+ $status = Status::newGood();
+ foreach ( $pathsByType as $type => $paths ) {
+ $status->merge( $this->doUnlock( $paths, $type ) );
+ }
+ return $status;
+ }
+
+ /**
* Unlock resources with the given keys and lock type
*
- * @param array $paths List of storage paths
+ * @param array $paths List of paths
* @param $type integer LockManager::LOCK_* constant
- * @return string
+ * @return Status
*/
abstract protected function doUnlock( array $paths, $type );
}
@@ -159,22 +250,10 @@ abstract class LockManager {
* @since 1.19
*/
class NullLockManager extends LockManager {
- /**
- * @see LockManager::doLock()
- * @param $paths array
- * @param $type int
- * @return Status
- */
protected function doLock( array $paths, $type ) {
return Status::newGood();
}
- /**
- * @see LockManager::doUnlock()
- * @param $paths array
- * @param $type int
- * @return Status
- */
protected function doUnlock( array $paths, $type ) {
return Status::newGood();
}
diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php
index ac0bd49b..9aff2415 100644
--- a/includes/filebackend/lockmanager/LockManagerGroup.php
+++ b/includes/filebackend/lockmanager/LockManagerGroup.php
@@ -97,8 +97,8 @@ class LockManagerGroup {
$class = $config['class'];
unset( $config['class'] ); // lock manager won't need this
$this->managers[$name] = array(
- 'class' => $class,
- 'config' => $config,
+ 'class' => $class,
+ 'config' => $config,
'instance' => null
);
}
diff --git a/includes/filebackend/lockmanager/MemcLockManager.php b/includes/filebackend/lockmanager/MemcLockManager.php
index fafc588a..5eab03ee 100644
--- a/includes/filebackend/lockmanager/MemcLockManager.php
+++ b/includes/filebackend/lockmanager/MemcLockManager.php
@@ -88,11 +88,44 @@ class MemcLockManager extends QuorumLockManager {
$this->session = wfRandomString( 32 );
}
+ // @TODO: change this code to work in one batch
+ protected function getLocksOnServer( $lockSrv, array $pathsByType ) {
+ $status = Status::newGood();
+
+ $lockedPaths = array();
+ foreach ( $pathsByType as $type => $paths ) {
+ $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) );
+ if ( $status->isOK() ) {
+ $lockedPaths[$type] = isset( $lockedPaths[$type] )
+ ? array_merge( $lockedPaths[$type], $paths )
+ : $paths;
+ } else {
+ foreach ( $lockedPaths as $type => $paths ) {
+ $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) );
+ }
+ break;
+ }
+ }
+
+ return $status;
+ }
+
+ // @TODO: change this code to work in one batch
+ protected function freeLocksOnServer( $lockSrv, array $pathsByType ) {
+ $status = Status::newGood();
+
+ foreach ( $pathsByType as $type => $paths ) {
+ $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) );
+ }
+
+ return $status;
+ }
+
/**
* @see QuorumLockManager::getLocksOnServer()
* @return Status
*/
- protected function getLocksOnServer( $lockSrv, array $paths, $type ) {
+ protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) {
$status = Status::newGood();
$memc = $this->getCache( $lockSrv );
@@ -145,7 +178,7 @@ class MemcLockManager extends QuorumLockManager {
foreach ( $paths as $path ) {
$locksKey = $this->recordKeyForPath( $path );
$locksHeld = $lockRecords[$locksKey];
- $ok = $memc->set( $locksKey, $locksHeld, 7*86400 );
+ $ok = $memc->set( $locksKey, $locksHeld, 7 * 86400 );
if ( !$ok ) {
$status->fatal( 'lockmanager-fail-acquirelock', $path );
} else {
@@ -164,7 +197,7 @@ class MemcLockManager extends QuorumLockManager {
* @see QuorumLockManager::freeLocksOnServer()
* @return Status
*/
- protected function freeLocksOnServer( $lockSrv, array $paths, $type ) {
+ protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) {
$status = Status::newGood();
$memc = $this->getCache( $lockSrv );
@@ -297,7 +330,7 @@ class MemcLockManager extends QuorumLockManager {
$start = microtime( true );
do {
if ( ( ++$rounds % 4 ) == 0 ) {
- usleep( 1000*50 ); // 50 ms
+ usleep( 1000 * 50 ); // 50 ms
}
foreach ( array_diff( $keys, $lockedKeys ) as $key ) {
if ( $memc->add( "$key:mutex", 1, 180 ) ) { // lock record
diff --git a/includes/filebackend/lockmanager/QuorumLockManager.php b/includes/filebackend/lockmanager/QuorumLockManager.php
index b331b540..8356d32a 100644
--- a/includes/filebackend/lockmanager/QuorumLockManager.php
+++ b/includes/filebackend/lockmanager/QuorumLockManager.php
@@ -31,81 +31,86 @@
abstract class QuorumLockManager extends LockManager {
/** @var Array Map of bucket indexes to peer server lists */
protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...))
+ /** @var Array Map of degraded buckets */
+ protected $degradedBuckets = array(); // (buckey index => UNIX timestamp)
- /**
- * @see LockManager::doLock()
- * @param $paths array
- * @param $type int
- * @return Status
- */
final protected function doLock( array $paths, $type ) {
+ return $this->doLockByType( array( $type => $paths ) );
+ }
+
+ final protected function doUnlock( array $paths, $type ) {
+ return $this->doUnlockByType( array( $type => $paths ) );
+ }
+
+ protected function doLockByType( array $pathsByType ) {
$status = Status::newGood();
- $pathsToLock = array(); // (bucket => paths)
+ $pathsToLock = array(); // (bucket => type => paths)
// Get locks that need to be acquired (buckets => locks)...
- foreach ( $paths as $path ) {
- if ( isset( $this->locksHeld[$path][$type] ) ) {
- ++$this->locksHeld[$path][$type];
- } else {
- $bucket = $this->getBucketFromPath( $path );
- $pathsToLock[$bucket][] = $path;
+ foreach ( $pathsByType as $type => $paths ) {
+ foreach ( $paths as $path ) {
+ if ( isset( $this->locksHeld[$path][$type] ) ) {
+ ++$this->locksHeld[$path][$type];
+ } else {
+ $bucket = $this->getBucketFromPath( $path );
+ $pathsToLock[$bucket][$type][] = $path;
+ }
}
}
- $lockedPaths = array(); // files locked in this attempt
+ $lockedPaths = array(); // files locked in this attempt (type => paths)
// Attempt to acquire these locks...
- foreach ( $pathsToLock as $bucket => $paths ) {
+ foreach ( $pathsToLock as $bucket => $pathsToLockByType ) {
// Try to acquire the locks for this bucket
- $status->merge( $this->doLockingRequestBucket( $bucket, $paths, $type ) );
+ $status->merge( $this->doLockingRequestBucket( $bucket, $pathsToLockByType ) );
if ( !$status->isOK() ) {
- $status->merge( $this->doUnlock( $lockedPaths, $type ) );
+ $status->merge( $this->doUnlockByType( $lockedPaths ) );
return $status;
}
// Record these locks as active
- foreach ( $paths as $path ) {
- $this->locksHeld[$path][$type] = 1; // locked
+ foreach ( $pathsToLockByType as $type => $paths ) {
+ foreach ( $paths as $path ) {
+ $this->locksHeld[$path][$type] = 1; // locked
+ // Keep track of what locks were made in this attempt
+ $lockedPaths[$type][] = $path;
+ }
}
- // Keep track of what locks were made in this attempt
- $lockedPaths = array_merge( $lockedPaths, $paths );
}
return $status;
}
- /**
- * @see LockManager::doUnlock()
- * @param $paths array
- * @param $type int
- * @return Status
- */
- final protected function doUnlock( array $paths, $type ) {
+ protected function doUnlockByType( array $pathsByType ) {
$status = Status::newGood();
- $pathsToUnlock = array();
- foreach ( $paths as $path ) {
- if ( !isset( $this->locksHeld[$path][$type] ) ) {
- $status->warning( 'lockmanager-notlocked', $path );
- } else {
- --$this->locksHeld[$path][$type];
- // Reference count the locks held and release locks when zero
- if ( $this->locksHeld[$path][$type] <= 0 ) {
- unset( $this->locksHeld[$path][$type] );
- $bucket = $this->getBucketFromPath( $path );
- $pathsToUnlock[$bucket][] = $path;
- }
- if ( !count( $this->locksHeld[$path] ) ) {
- unset( $this->locksHeld[$path] ); // no SH or EX locks left for key
+ $pathsToUnlock = array(); // (bucket => type => paths)
+ foreach ( $pathsByType as $type => $paths ) {
+ foreach ( $paths as $path ) {
+ if ( !isset( $this->locksHeld[$path][$type] ) ) {
+ $status->warning( 'lockmanager-notlocked', $path );
+ } else {
+ --$this->locksHeld[$path][$type];
+ // Reference count the locks held and release locks when zero
+ if ( $this->locksHeld[$path][$type] <= 0 ) {
+ unset( $this->locksHeld[$path][$type] );
+ $bucket = $this->getBucketFromPath( $path );
+ $pathsToUnlock[$bucket][$type][] = $path;
+ }
+ if ( !count( $this->locksHeld[$path] ) ) {
+ unset( $this->locksHeld[$path] ); // no SH or EX locks left for key
+ }
}
}
}
// Remove these specific locks if possible, or at least release
// all locks once this process is currently not holding any locks.
- foreach ( $pathsToUnlock as $bucket => $paths ) {
- $status->merge( $this->doUnlockingRequestBucket( $bucket, $paths, $type ) );
+ foreach ( $pathsToUnlock as $bucket => $pathsToUnlockByType ) {
+ $status->merge( $this->doUnlockingRequestBucket( $bucket, $pathsToUnlockByType ) );
}
if ( !count( $this->locksHeld ) ) {
$status->merge( $this->releaseAllLocks() );
+ $this->degradedBuckets = array(); // safe to retry the normal quorum
}
return $status;
@@ -116,25 +121,25 @@ abstract class QuorumLockManager extends LockManager {
* This is all or nothing; if any key is locked then this totally fails.
*
* @param $bucket integer
- * @param array $paths List of resource keys to lock
- * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
- final protected function doLockingRequestBucket( $bucket, array $paths, $type ) {
+ final protected function doLockingRequestBucket( $bucket, array $pathsByType ) {
$status = Status::newGood();
$yesVotes = 0; // locks made on trustable servers
$votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers
- $quorum = floor( $votesLeft/2 + 1 ); // simple majority
+ $quorum = floor( $votesLeft / 2 + 1 ); // simple majority
// Get votes for each peer, in order, until we have enough...
foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) {
if ( !$this->isServerUp( $lockSrv ) ) {
--$votesLeft;
$status->warning( 'lockmanager-fail-svr-acquire', $lockSrv );
+ $this->degradedBuckets[$bucket] = time();
continue; // server down?
}
// Attempt to acquire the lock on this peer
- $status->merge( $this->getLocksOnServer( $lockSrv, $paths, $type ) );
+ $status->merge( $this->getLocksOnServer( $lockSrv, $pathsByType ) );
if ( !$status->isOK() ) {
return $status; // vetoed; resource locked
}
@@ -158,21 +163,33 @@ abstract class QuorumLockManager extends LockManager {
* Attempt to release locks with the peers for a bucket
*
* @param $bucket integer
- * @param array $paths List of resource keys to lock
- * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
- final protected function doUnlockingRequestBucket( $bucket, array $paths, $type ) {
+ final protected function doUnlockingRequestBucket( $bucket, array $pathsByType ) {
$status = Status::newGood();
+ $yesVotes = 0; // locks freed on trustable servers
+ $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers
+ $quorum = floor( $votesLeft / 2 + 1 ); // simple majority
+ $isDegraded = isset( $this->degradedBuckets[$bucket] ); // not the normal quorum?
foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) {
if ( !$this->isServerUp( $lockSrv ) ) {
- $status->fatal( 'lockmanager-fail-svr-release', $lockSrv );
+ $status->warning( 'lockmanager-fail-svr-release', $lockSrv );
// Attempt to release the lock on this peer
} else {
- $status->merge( $this->freeLocksOnServer( $lockSrv, $paths, $type ) );
+ $status->merge( $this->freeLocksOnServer( $lockSrv, $pathsByType ) );
+ ++$yesVotes; // success for this peer
+ // Normally the first peers form the quorum, and the others are ignored.
+ // Ignore them in this case, but not when an alternative quorum was used.
+ if ( $yesVotes >= $quorum && !$isDegraded ) {
+ break; // lock released
+ }
}
}
+ // Set a bad status if the quorum was not met.
+ // Assumes the same "up" servers as during the acquire step.
+ $status->setResult( $yesVotes >= $quorum );
return $status;
}
@@ -190,7 +207,8 @@ abstract class QuorumLockManager extends LockManager {
}
/**
- * Check if a lock server is up
+ * Check if a lock server is up.
+ * This should process cache results to reduce RTT.
*
* @param $lockSrv string
* @return bool
@@ -198,14 +216,13 @@ abstract class QuorumLockManager extends LockManager {
abstract protected function isServerUp( $lockSrv );
/**
- * Get a connection to a lock server and acquire locks on $paths
+ * Get a connection to a lock server and acquire locks
*
* @param $lockSrv string
- * @param $paths array
- * @param $type integer
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
- abstract protected function getLocksOnServer( $lockSrv, array $paths, $type );
+ abstract protected function getLocksOnServer( $lockSrv, array $pathsByType );
/**
* Get a connection to a lock server and release locks on $paths.
@@ -213,11 +230,10 @@ abstract class QuorumLockManager extends LockManager {
* Subclasses must effectively implement this or releaseAllLocks().
*
* @param $lockSrv string
- * @param $paths array
- * @param $type integer
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
- abstract protected function freeLocksOnServer( $lockSrv, array $paths, $type );
+ abstract protected function freeLocksOnServer( $lockSrv, array $pathsByType );
/**
* Release all locks that this session is holding.
diff --git a/includes/filebackend/lockmanager/RedisLockManager.php b/includes/filebackend/lockmanager/RedisLockManager.php
new file mode 100644
index 00000000..43b0198a
--- /dev/null
+++ b/includes/filebackend/lockmanager/RedisLockManager.php
@@ -0,0 +1,288 @@
+<?php
+/**
+ * Version of LockManager based on using redis servers.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup LockManager
+ */
+
+/**
+ * Manage locks using redis servers.
+ *
+ * Version of LockManager based on using redis servers.
+ * This is meant for multi-wiki systems that may share files.
+ * All locks are non-blocking, which avoids deadlocks.
+ *
+ * All lock requests for a resource, identified by a hash string, will map to one
+ * bucket. Each bucket maps to one or several peer servers, each running redis.
+ * A majority of peers must agree for a lock to be acquired.
+ *
+ * This class requires Redis 2.6 as it makes use Lua scripts for fast atomic operations.
+ *
+ * @ingroup LockManager
+ * @since 1.22
+ */
+class RedisLockManager extends QuorumLockManager {
+ /** @var Array Mapping of lock types to the type actually used */
+ protected $lockTypeMap = array(
+ self::LOCK_SH => self::LOCK_SH,
+ self::LOCK_UW => self::LOCK_SH,
+ self::LOCK_EX => self::LOCK_EX
+ );
+
+ /** @var RedisConnectionPool */
+ protected $redisPool;
+ /** @var Array Map server names to hostname/IP and port numbers */
+ protected $lockServers = array();
+
+ protected $session = ''; // string; random UUID
+
+ /**
+ * Construct a new instance from configuration.
+ *
+ * $config paramaters include:
+ * - lockServers : Associative array of server names to "<IP>:<port>" strings.
+ * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0,
+ * each having an odd-numbered list of server names (peers) as values.
+ * - redisConfig : Configuration for RedisConnectionPool::__construct().
+ *
+ * @param Array $config
+ * @throws MWException
+ */
+ public function __construct( array $config ) {
+ parent::__construct( $config );
+
+ $this->lockServers = $config['lockServers'];
+ // Sanitize srvsByBucket config to prevent PHP errors
+ $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' );
+ $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive
+
+ $config['redisConfig']['serializer'] = 'none';
+ $this->redisPool = RedisConnectionPool::singleton( $config['redisConfig'] );
+
+ $this->session = wfRandomString( 32 );
+ }
+
+ // @TODO: change this code to work in one batch
+ protected function getLocksOnServer( $lockSrv, array $pathsByType ) {
+ $status = Status::newGood();
+
+ $lockedPaths = array();
+ foreach ( $pathsByType as $type => $paths ) {
+ $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) );
+ if ( $status->isOK() ) {
+ $lockedPaths[$type] = isset( $lockedPaths[$type] )
+ ? array_merge( $lockedPaths[$type], $paths )
+ : $paths;
+ } else {
+ foreach ( $lockedPaths as $type => $paths ) {
+ $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) );
+ }
+ break;
+ }
+ }
+
+ return $status;
+ }
+
+ // @TODO: change this code to work in one batch
+ protected function freeLocksOnServer( $lockSrv, array $pathsByType ) {
+ $status = Status::newGood();
+
+ foreach ( $pathsByType as $type => $paths ) {
+ $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) );
+ }
+
+ return $status;
+ }
+
+ protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) {
+ $status = Status::newGood();
+
+ $server = $this->lockServers[$lockSrv];
+ $conn = $this->redisPool->getConnection( $server );
+ if ( !$conn ) {
+ foreach ( $paths as $path ) {
+ $status->fatal( 'lockmanager-fail-acquirelock', $path );
+ }
+ return $status;
+ }
+
+ $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records
+
+ try {
+ static $script =
+<<<LUA
+ if ARGV[1] ~= 'EX' and ARGV[1] ~= 'SH' then
+ return redis.error_reply('Unrecognized lock type given (must be EX or SH)')
+ end
+ local failed = {}
+ -- Check that all the locks can be acquired
+ for i,resourceKey in ipairs(KEYS) do
+ local keyIsFree = true
+ local currentLocks = redis.call('hKeys',resourceKey)
+ for i,lockKey in ipairs(currentLocks) do
+ local _, _, type, session = string.find(lockKey,"(%w+):(%w+)")
+ -- Check any locks that are not owned by this session
+ if session ~= ARGV[2] then
+ local lockTimestamp = redis.call('hGet',resourceKey,lockKey)
+ if 1*lockTimestamp < ( ARGV[4] - ARGV[3] ) then
+ -- Lock is stale, so just prune it out
+ redis.call('hDel',resourceKey,lockKey)
+ elseif ARGV[1] == 'EX' or type == 'EX' then
+ keyIsFree = false
+ break
+ end
+ end
+ end
+ if not keyIsFree then
+ failed[#failed+1] = resourceKey
+ end
+ end
+ -- If all locks could be acquired, then do so
+ if #failed == 0 then
+ for i,resourceKey in ipairs(KEYS) do
+ redis.call('hSet',resourceKey,ARGV[1] .. ':' .. ARGV[2],ARGV[4])
+ -- In addition to invalidation logic, be sure to garbage collect
+ redis.call('expire',resourceKey,ARGV[3])
+ end
+ end
+ return failed
+LUA;
+ $res = $conn->luaEval( $script,
+ array_merge(
+ $keys, // KEYS[0], KEYS[1],...KEYS[N]
+ array(
+ $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1]
+ $this->session, // ARGV[2]
+ $this->lockTTL, // ARGV[3]
+ time() // ARGV[4]
+ )
+ ),
+ count( $keys ) # number of first argument(s) that are keys
+ );
+ } catch ( RedisException $e ) {
+ $res = false;
+ $this->redisPool->handleException( $server, $conn, $e );
+ }
+
+ if ( $res === false ) {
+ foreach ( $paths as $path ) {
+ $status->fatal( 'lockmanager-fail-acquirelock', $path );
+ }
+ } else {
+ $pathsByKey = array_combine( $keys, $paths );
+ foreach ( $res as $key ) {
+ $status->fatal( 'lockmanager-fail-acquirelock', $pathsByKey[$key] );
+ }
+ }
+
+ return $status;
+ }
+
+ protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) {
+ $status = Status::newGood();
+
+ $server = $this->lockServers[$lockSrv];
+ $conn = $this->redisPool->getConnection( $server );
+ if ( !$conn ) {
+ foreach ( $paths as $path ) {
+ $status->fatal( 'lockmanager-fail-releaselock', $path );
+ }
+ return $status;
+ }
+
+ $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records
+
+ try {
+ static $script =
+<<<LUA
+ if ARGV[1] ~= 'EX' and ARGV[1] ~= 'SH' then
+ return redis.error_reply('Unrecognized lock type given (must be EX or SH)')
+ end
+ local failed = {}
+ for i,resourceKey in ipairs(KEYS) do
+ local released = redis.call('hDel',resourceKey,ARGV[1] .. ':' .. ARGV[2])
+ if released > 0 then
+ -- Remove the whole structure if it is now empty
+ if redis.call('hLen',resourceKey) == 0 then
+ redis.call('del',resourceKey)
+ end
+ else
+ failed[#failed+1] = resourceKey
+ end
+ end
+ return failed
+LUA;
+ $res = $conn->luaEval( $script,
+ array_merge(
+ $keys, // KEYS[0], KEYS[1],...KEYS[N]
+ array(
+ $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1]
+ $this->session // ARGV[2]
+ )
+ ),
+ count( $keys ) # number of first argument(s) that are keys
+ );
+ } catch ( RedisException $e ) {
+ $res = false;
+ $this->redisPool->handleException( $server, $conn, $e );
+ }
+
+ if ( $res === false ) {
+ foreach ( $paths as $path ) {
+ $status->fatal( 'lockmanager-fail-releaselock', $path );
+ }
+ } else {
+ $pathsByKey = array_combine( $keys, $paths );
+ foreach ( $res as $key ) {
+ $status->fatal( 'lockmanager-fail-releaselock', $pathsByKey[$key] );
+ }
+ }
+
+ return $status;
+ }
+
+ protected function releaseAllLocks() {
+ return Status::newGood(); // not supported
+ }
+
+ protected function isServerUp( $lockSrv ) {
+ return (bool)$this->redisPool->getConnection( $this->lockServers[$lockSrv] );
+ }
+
+ /**
+ * @param $path string
+ * @return string
+ */
+ protected function recordKeyForPath( $path ) {
+ return implode( ':', array( __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ) );
+ }
+
+ /**
+ * Make sure remaining locks get cleared for sanity
+ */
+ function __destruct() {
+ while ( count( $this->locksHeld ) ) {
+ foreach ( $this->locksHeld as $path => $locks ) {
+ $this->doUnlock( array( $path ), self::LOCK_EX );
+ $this->doUnlock( array( $path ), self::LOCK_SH );
+ }
+ }
+ }
+}
diff --git a/includes/filebackend/lockmanager/ScopedLock.php b/includes/filebackend/lockmanager/ScopedLock.php
index edcb1d65..5faad4a6 100644
--- a/includes/filebackend/lockmanager/ScopedLock.php
+++ b/includes/filebackend/lockmanager/ScopedLock.php
@@ -36,24 +36,18 @@ class ScopedLock {
protected $manager;
/** @var Status */
protected $status;
- /** @var Array List of resource paths*/
- protected $paths;
-
- protected $type; // integer lock type
+ /** @var Array Map of lock types to resource paths */
+ protected $pathsByType;
/**
- * @param $manager LockManager
- * @param array $paths List of storage paths
- * @param $type integer LockManager::LOCK_* constant
- * @param $status Status
+ * @param LockManager $manager
+ * @param array $pathsByType Map of lock types to path lists
+ * @param Status $status
*/
- protected function __construct(
- LockManager $manager, array $paths, $type, Status $status
- ) {
+ protected function __construct( LockManager $manager, array $pathsByType, Status $status ) {
$this->manager = $manager;
- $this->paths = $paths;
+ $this->pathsByType = $pathsByType;
$this->status = $status;
- $this->type = $type;
}
/**
@@ -61,19 +55,24 @@ class ScopedLock {
* Any locks are released once this object goes out of scope.
* The status object is updated with any errors or warnings.
*
- * @param $manager LockManager
- * @param array $paths List of storage paths
- * @param $type integer LockManager::LOCK_* constant
- * @param $status Status
+ * $type can be "mixed" and $paths can be a map of types to paths (since 1.22).
+ * Otherwise $type should be an integer and $paths should be a list of paths.
+ *
+ * @param LockManager $manager
+ * @param array $paths List of storage paths or map of lock types to path lists
+ * @param integer|string $type LockManager::LOCK_* constant or "mixed"
+ * @param Status $status
+ * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.22)
* @return ScopedLock|null Returns null on failure
*/
public static function factory(
- LockManager $manager, array $paths, $type, Status $status
+ LockManager $manager, array $paths, $type, Status $status, $timeout = 0
) {
- $lockStatus = $manager->lock( $paths, $type );
+ $pathsByType = is_integer( $type ) ? array( $type => $paths ) : $paths;
+ $lockStatus = $manager->lockByType( $pathsByType, $timeout );
$status->merge( $lockStatus );
if ( $lockStatus->isOK() ) {
- return new self( $manager, $paths, $type, $status );
+ return new self( $manager, $pathsByType, $status );
}
return null;
}
@@ -91,9 +90,12 @@ class ScopedLock {
$lock = null;
}
+ /**
+ * Release the locks when this goes out of scope
+ */
function __destruct() {
$wasOk = $this->status->isOK();
- $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) );
+ $this->status->merge( $this->manager->unlockByType( $this->pathsByType ) );
if ( $wasOk ) {
// Make sure status is OK, despite any unlockFiles() fatals
$this->status->setResult( true, $this->status->value );