summaryrefslogtreecommitdiff
path: root/includes/filebackend/FileBackendStore.php
diff options
context:
space:
mode:
Diffstat (limited to 'includes/filebackend/FileBackendStore.php')
-rw-r--r--includes/filebackend/FileBackendStore.php576
1 files changed, 172 insertions, 404 deletions
diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php
index 3f1d1857..0921e99f 100644
--- a/includes/filebackend/FileBackendStore.php
+++ b/includes/filebackend/FileBackendStore.php
@@ -38,28 +38,43 @@
abstract class FileBackendStore extends FileBackend {
/** @var BagOStuff */
protected $memCache;
- /** @var ProcessCacheLRU */
- protected $cheapCache; // Map of paths to small (RAM/disk) cache items
- /** @var ProcessCacheLRU */
- protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
+ /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */
+ protected $cheapCache;
+ /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */
+ protected $expensiveCache;
- /** @var Array Map of container names to sharding settings */
- protected $shardViaHashLevels = array(); // (container name => config array)
+ /** @var Array Map of container names to sharding config */
+ protected $shardViaHashLevels = array();
+
+ /** @var callback Method to get the MIME type of files */
+ protected $mimeCallback;
protected $maxFileSize = 4294967296; // integer bytes (4GiB)
const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
+ const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache"
+ const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
/**
* @see FileBackend::__construct()
+ * Additional $config params include:
+ * - mimeCallback : Callback that takes (storage path, content, file system path) and
+ * returns the MIME type of the file or 'unknown/unknown'. The file
+ * system path parameter should be used if the content one is null.
*
- * @param $config Array
+ * @param array $config
*/
public function __construct( array $config ) {
parent::__construct( $config );
+ $this->mimeCallback = isset( $config['mimeCallback'] )
+ ? $config['mimeCallback']
+ : function( $storagePath, $content, $fsPath ) {
+ // @TODO: handle the case of extension-less files using the contents
+ return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown';
+ };
$this->memCache = new EmptyBagOStuff(); // disabled by default
- $this->cheapCache = new ProcessCacheLRU( 300 );
- $this->expensiveCache = new ProcessCacheLRU( 5 );
+ $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
+ $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );
}
/**
@@ -79,7 +94,7 @@ abstract class FileBackendStore extends FileBackend {
* written under it, and that any file already there is writable.
* Backends using key/value stores should check if the container exists.
*
- * @param $storagePath string
+ * @param string $storagePath
* @return bool
*/
abstract public function isPathUsableInternal( $storagePath );
@@ -92,7 +107,6 @@ abstract class FileBackendStore extends FileBackend {
* $params include:
* - content : the raw file contents
* - dst : destination storage path
- * - disposition : Content-Disposition header value for the destination
* - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
@@ -104,8 +118,7 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function createInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
$status = Status::newFatal( 'backend-fail-maxsize',
$params['dst'], $this->maxFileSizeInternal() );
@@ -116,8 +129,6 @@ abstract class FileBackendStore extends FileBackend {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -135,7 +146,6 @@ abstract class FileBackendStore extends FileBackend {
* $params include:
* - src : source path on disk
* - dst : destination storage path
- * - disposition : Content-Disposition header value for the destination
* - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
@@ -147,8 +157,7 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function storeInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
$status = Status::newFatal( 'backend-fail-maxsize',
$params['dst'], $this->maxFileSizeInternal() );
@@ -159,8 +168,6 @@ abstract class FileBackendStore extends FileBackend {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -179,7 +186,7 @@ abstract class FileBackendStore extends FileBackend {
* - src : source storage path
* - dst : destination storage path
* - ignoreMissingSource : do nothing if the source file does not exist
- * - disposition : Content-Disposition header value for the destination
+ * - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
* set to a FileBackendStoreOpHandle object.
@@ -190,15 +197,12 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function copyInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = $this->doCopyInternal( $params );
$this->clearCache( array( $params['dst'] ) );
if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -223,13 +227,10 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function deleteInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = $this->doDeleteInternal( $params );
$this->clearCache( array( $params['src'] ) );
$this->deleteFileCache( $params['src'] ); // persistent cache
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -248,7 +249,7 @@ abstract class FileBackendStore extends FileBackend {
* - src : source storage path
* - dst : destination storage path
* - ignoreMissingSource : do nothing if the source file does not exist
- * - disposition : Content-Disposition header value for the destination
+ * - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
* set to a FileBackendStoreOpHandle object.
@@ -259,16 +260,13 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function moveInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = $this->doMoveInternal( $params );
$this->clearCache( array( $params['src'], $params['dst'] ) );
$this->deleteFileCache( $params['src'] ); // persistent cache
if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -278,10 +276,12 @@ abstract class FileBackendStore extends FileBackend {
*/
protected function doMoveInternal( array $params ) {
unset( $params['async'] ); // two steps, won't work here :)
+ $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
+ $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
// Copy source to dest
$status = $this->copyInternal( $params );
- if ( $status->isOK() ) {
- // Delete source (only fails due to races or medium going down)
+ if ( $nsrc !== $ndst && $status->isOK() ) {
+ // Delete source (only fails due to races or network problems)
$status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
$status->setResult( true, $status->value ); // ignore delete() errors
}
@@ -294,7 +294,6 @@ abstract class FileBackendStore extends FileBackend {
*
* $params include:
* - src : source storage path
- * - disposition : Content-Disposition header value for the destination
* - headers : HTTP header name/value map
* - async : Status will be returned immediately if supported.
* If the status is OK, then its value field will be
@@ -304,13 +303,14 @@ abstract class FileBackendStore extends FileBackend {
* @return Status
*/
final public function describeInternal( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
- $status = $this->doDescribeInternal( $params );
- $this->clearCache( array( $params['src'] ) );
- $this->deleteFileCache( $params['src'] ); // persistent cache
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
+ if ( count( $params['headers'] ) ) {
+ $status = $this->doDescribeInternal( $params );
+ $this->clearCache( array( $params['src'] ) );
+ $this->deleteFileCache( $params['src'] ); // persistent cache
+ } else {
+ $status = Status::newGood(); // nothing to do
+ }
return $status;
}
@@ -333,13 +333,8 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::concatenate()
- * @return Status
- */
final public function concatenate( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Try to lock the source files for the scope of this function
@@ -355,8 +350,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -426,20 +419,13 @@ abstract class FileBackendStore extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::doPrepare()
- * @return Status
- */
final protected function doPrepare( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
-
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
+
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -453,8 +439,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -466,20 +450,13 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::doSecure()
- * @return Status
- */
final protected function doSecure( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -493,8 +470,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -506,20 +481,13 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::doPublish()
- * @return Status
- */
final protected function doPublish( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -533,8 +501,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -546,13 +512,8 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::doClean()
- * @return Status
- */
final protected function doClean( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Recursive: first delete all empty subdirs recursively
@@ -570,8 +531,6 @@ abstract class FileBackendStore extends FileBackend {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // invalid storage path
}
@@ -579,8 +538,6 @@ abstract class FileBackendStore extends FileBackend {
$filesLockEx = array( $params['dir'] );
$scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
if ( !$status->isOK() ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // abort
}
@@ -596,8 +553,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -609,56 +564,30 @@ abstract class FileBackendStore extends FileBackend {
return Status::newGood();
}
- /**
- * @see FileBackend::fileExists()
- * @return bool|null
- */
final public function fileExists( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return ( $stat === null ) ? null : (bool)$stat; // null => failure
}
- /**
- * @see FileBackend::getFileTimestamp()
- * @return bool
- */
final public function getFileTimestamp( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat ? $stat['mtime'] : false;
}
- /**
- * @see FileBackend::getFileSize()
- * @return bool
- */
final public function getFileSize( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat ? $stat['size'] : false;
}
- /**
- * @see FileBackend::getFileStat()
- * @return bool
- */
final public function getFileStat( array $params ) {
$path = self::normalizeStoragePath( $params['src'] );
if ( $path === null ) {
return false; // invalid storage path
}
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$latest = !empty( $params['latest'] ); // use latest data?
if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
$this->primeFileCache( array( $path ) ); // check persistent cache
@@ -669,14 +598,10 @@ abstract class FileBackendStore extends FileBackend {
// value was in fact fetched with the latest available data.
if ( is_array( $stat ) ) {
if ( !$latest || $stat['latest'] ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat;
}
} elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) {
if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return false;
}
}
@@ -696,12 +621,12 @@ abstract class FileBackendStore extends FileBackend {
}
} elseif ( $stat === false ) { // file does not exist
$this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
+ $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too
+ array( 'hash' => false, 'latest' => $latest ) );
wfDebug( __METHOD__ . ": File $path does not exist.\n" );
} else { // an error occurred
wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat;
}
@@ -710,19 +635,12 @@ abstract class FileBackendStore extends FileBackend {
*/
abstract protected function doGetFileStat( array $params );
- /**
- * @see FileBackend::getFileContentsMulti()
- * @return Array
- */
public function getFileContentsMulti( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$params = $this->setConcurrencyFlags( $params );
$contents = $this->doGetFileContentsMulti( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $contents;
}
@@ -740,25 +658,18 @@ abstract class FileBackendStore extends FileBackend {
return $contents;
}
- /**
- * @see FileBackend::getFileSha1Base36()
- * @return bool|string
- */
final public function getFileSha1Base36( array $params ) {
$path = self::normalizeStoragePath( $params['src'] );
if ( $path === null ) {
return false; // invalid storage path
}
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$latest = !empty( $params['latest'] ); // use latest data?
if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
$stat = $this->cheapCache->get( $path, 'sha1' );
// If we want the latest data, check that this cached
// value was in fact fetched with the latest available data.
if ( !$latest || $stat['latest'] ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $stat['hash'];
}
}
@@ -768,8 +679,6 @@ abstract class FileBackendStore extends FileBackend {
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
wfProfileOut( __METHOD__ . '-miss' );
$this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $hash;
}
@@ -786,27 +695,15 @@ abstract class FileBackendStore extends FileBackend {
}
}
- /**
- * @see FileBackend::getFileProps()
- * @return Array
- */
final public function getFileProps( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$fsFile = $this->getLocalReference( $params );
$props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $props;
}
- /**
- * @see FileBackend::getLocalReferenceMulti()
- * @return Array
- */
final public function getLocalReferenceMulti( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$params = $this->setConcurrencyFlags( $params );
@@ -836,8 +733,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $fsFiles;
}
@@ -849,19 +744,12 @@ abstract class FileBackendStore extends FileBackend {
return $this->doGetLocalCopyMulti( $params );
}
- /**
- * @see FileBackend::getLocalCopyMulti()
- * @return Array
- */
final public function getLocalCopyMulti( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$params = $this->setConcurrencyFlags( $params );
$tmpFiles = $this->doGetLocalCopyMulti( $params );
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $tmpFiles;
}
@@ -879,13 +767,8 @@ abstract class FileBackendStore extends FileBackend {
return null; // not supported
}
- /**
- * @see FileBackend::streamFile()
- * @return Status
- */
final public function streamFile( array $params ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
$info = $this->getFileStat( $params );
@@ -916,8 +799,6 @@ abstract class FileBackendStore extends FileBackend {
$status->fatal( 'backend-fail-stream', $params['src'] );
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -938,10 +819,6 @@ abstract class FileBackendStore extends FileBackend {
return $status;
}
- /**
- * @see FileBackend::directoryExists()
- * @return bool|null
- */
final public function directoryExists( array $params ) {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
@@ -976,10 +853,6 @@ abstract class FileBackendStore extends FileBackend {
*/
abstract protected function doDirectoryExists( $container, $dir, array $params );
- /**
- * @see FileBackend::getDirectoryList()
- * @return Traversable|Array|null Returns null on failure
- */
final public function getDirectoryList( array $params ) {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) { // invalid storage path
@@ -1009,10 +882,6 @@ abstract class FileBackendStore extends FileBackend {
*/
abstract public function getDirectoryListInternal( $container, $dir, array $params );
- /**
- * @see FileBackend::getFileList()
- * @return Traversable|Array|null Returns null on failure
- */
final public function getFileList( array $params ) {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) { // invalid storage path
@@ -1055,13 +924,13 @@ abstract class FileBackendStore extends FileBackend {
*/
final public function getOperationsInternal( array $ops ) {
$supportedOps = array(
- 'store' => 'StoreFileOp',
- 'copy' => 'CopyFileOp',
- 'move' => 'MoveFileOp',
- 'delete' => 'DeleteFileOp',
- 'create' => 'CreateFileOp',
+ 'store' => 'StoreFileOp',
+ 'copy' => 'CopyFileOp',
+ 'move' => 'MoveFileOp',
+ 'delete' => 'DeleteFileOp',
+ 'create' => 'CreateFileOp',
'describe' => 'DescribeFileOp',
- 'null' => 'NullFileOp'
+ 'null' => 'NullFileOp'
);
$performOps = array(); // array of FileOp objects
@@ -1084,12 +953,13 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get a list of storage paths to lock for a list of operations
- * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
- * each corresponding to a list of storage paths to be locked.
- * All returned paths are normalized.
+ * Returns an array with LockManager::LOCK_UW (shared locks) and
+ * LockManager::LOCK_EX (exclusive locks) keys, each corresponding
+ * to a list of storage paths to be locked. All returned paths are
+ * normalized.
*
* @param array $performOps List of FileOp objects
- * @return Array ('sh' => list of paths, 'ex' => list of paths)
+ * @return Array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list)
*/
final public function getPathsToLockForOpsInternal( array $performOps ) {
// Build up a list of files to lock...
@@ -1103,28 +973,19 @@ abstract class FileBackendStore extends FileBackend {
// Get a shared lock on the parent directory of each path changed
$paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
- return $paths;
+ return array(
+ LockManager::LOCK_UW => $paths['sh'],
+ LockManager::LOCK_EX => $paths['ex']
+ );
}
- /**
- * @see FileBackend::getScopedLocksForOps()
- * @return Array
- */
public function getScopedLocksForOps( array $ops, Status $status ) {
$paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
- return array(
- $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
- $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
- );
+ return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) );
}
- /**
- * @see FileBackend::doOperationsInternal()
- * @return Status
- */
final protected function doOperationsInternal( array $ops, array $opts ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Fix up custom header name/value pairs...
@@ -1138,11 +999,8 @@ abstract class FileBackendStore extends FileBackend {
// Build up a list of files to lock...
$paths = $this->getPathsToLockForOpsInternal( $performOps );
// Try to lock those files for the scope of this function...
- $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
- $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
+ $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
if ( !$status->isOK() ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status; // abort
}
}
@@ -1164,19 +1022,11 @@ abstract class FileBackendStore extends FileBackend {
$status->merge( $subStatus );
$status->success = $subStatus->success; // not done in merge()
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
- /**
- * @see FileBackend::doQuickOperationsInternal()
- * @return Status
- * @throws MWException
- */
final protected function doQuickOperationsInternal( array $ops ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$status = Status::newGood();
// Fix up custom header name/value pairs...
@@ -1186,7 +1036,7 @@ abstract class FileBackendStore extends FileBackend {
$this->clearCache();
$supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
- $async = ( $this->parallelize === 'implicit' );
+ $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
$maxConcurrency = $this->concurrency; // throttle
$statuses = array(); // array of (index => Status)
@@ -1195,8 +1045,6 @@ abstract class FileBackendStore extends FileBackend {
// Perform the sync-only ops and build up op handles for the async ops...
foreach ( $ops as $index => $params ) {
if ( !in_array( $params['op'], $supportedOps ) ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
throw new MWException( "Operation '{$params['op']}' is not supported." );
}
$method = $params['op'] . 'Internal'; // e.g. "storeInternal"
@@ -1230,8 +1078,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -1245,8 +1091,7 @@ abstract class FileBackendStore extends FileBackend {
* @throws MWException
*/
final public function executeOpHandlesInternal( array $fileOpHandles ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
foreach ( $fileOpHandles as $fileOpHandle ) {
if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
@@ -1258,8 +1103,6 @@ abstract class FileBackendStore extends FileBackend {
foreach ( $fileOpHandles as $fileOpHandle ) {
$fileOpHandle->closeResources();
}
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
return $res;
}
@@ -1277,15 +1120,20 @@ abstract class FileBackendStore extends FileBackend {
}
/**
- * Strip long HTTP headers from a file operation
+ * Strip long HTTP headers from a file operation.
+ * Most headers are just numbers, but some are allowed to be long.
+ * This function is useful for cleaning up headers and avoiding backend
+ * specific errors, especially in the middle of batch file operations.
*
* @param array $op Same format as doOperation()
* @return Array
*/
protected function stripInvalidHeadersFromOp( array $op ) {
- if ( isset( $op['headers'] ) ) {
+ static $longs = array( 'Content-Disposition' );
+ if ( isset( $op['headers'] ) ) { // op sets HTTP headers
foreach ( $op['headers'] as $name => $value ) {
- if ( strlen( $name ) > 255 || strlen( $value ) > 255 ) {
+ $maxHVLen = in_array( $name, $longs ) ? INF : 255;
+ if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
trigger_error( "Header '$name: $value' is too long." );
unset( $op['headers'][$name] );
} elseif ( !strlen( $value ) ) {
@@ -1296,9 +1144,6 @@ abstract class FileBackendStore extends FileBackend {
return $op;
}
- /**
- * @see FileBackend::preloadCache()
- */
final public function preloadCache( array $paths ) {
$fullConts = array(); // full container names
foreach ( $paths as $path ) {
@@ -1310,9 +1155,6 @@ abstract class FileBackendStore extends FileBackend {
$this->primeFileCache( $paths );
}
- /**
- * @see FileBackend::clearCache()
- */
final public function clearCache( array $paths = null ) {
if ( is_array( $paths ) ) {
$paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
@@ -1353,7 +1195,7 @@ abstract class FileBackendStore extends FileBackend {
* Check if a container name is valid.
* This checks for for length and illegal characters.
*
- * @param $container string
+ * @param string $container
* @return bool
*/
final protected static function isValidContainerName( $container ) {
@@ -1375,7 +1217,7 @@ abstract class FileBackendStore extends FileBackend {
* this means that the path can only refer to a directory and can only
* be scanned by looking in all the container shards.
*
- * @param $storagePath string
+ * @param string $storagePath
* @return Array (container, path, container suffix) or (null, null, null) if invalid
*/
final protected function resolveStoragePath( $storagePath ) {
@@ -1405,16 +1247,22 @@ abstract class FileBackendStore extends FileBackend {
/**
* Like resolveStoragePath() except null values are returned if
- * the container is sharded and the shard could not be determined.
+ * the container is sharded and the shard could not be determined
+ * or if the path ends with '/'. The later case is illegal for FS
+ * backends and can confuse listings for object store backends.
+ *
+ * This function is used when resolving paths that must be valid
+ * locations for files. Directory and listing functions should
+ * generally just use resolveStoragePath() instead.
*
* @see FileBackendStore::resolveStoragePath()
*
- * @param $storagePath string
+ * @param string $storagePath
* @return Array (container, path) or (null, null) if invalid
*/
final protected function resolveStoragePathReal( $storagePath ) {
list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
- if ( $cShard !== null ) {
+ if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
return array( $container, $relPath );
}
return array( null, null );
@@ -1474,7 +1322,7 @@ abstract class FileBackendStore extends FileBackend {
* If greater than 0, then all file storage paths within
* the container are required to be hashed accordingly.
*
- * @param $container string
+ * @param string $container
* @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
*/
final protected function getContainerHashLevels( $container ) {
@@ -1494,7 +1342,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get a list of full container shard suffixes for a container
*
- * @param $container string
+ * @param string $container
* @return Array
*/
final protected function getContainerSuffixes( $container ) {
@@ -1512,7 +1360,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* Get the full container name, including the wiki ID prefix
*
- * @param $container string
+ * @param string $container
* @return string
*/
final protected function fullContainerName( $container ) {
@@ -1528,7 +1376,7 @@ abstract class FileBackendStore extends FileBackend {
* This is intended for internal use, such as encoding illegal chars.
* Subclasses can override this to be more restrictive.
*
- * @param $container string
+ * @param string $container
* @return string|null
*/
protected function resolveContainerName( $container ) {
@@ -1563,10 +1411,11 @@ abstract class FileBackendStore extends FileBackend {
* Set the cached info for a container
*
* @param string $container Resolved container name
- * @param $val mixed Information to cache
+ * @param array $val Information to cache
+ * @return void
*/
- final protected function setContainerCache( $container, $val ) {
- $this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 );
+ final protected function setContainerCache( $container, array $val ) {
+ $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 );
}
/**
@@ -1574,6 +1423,7 @@ abstract class FileBackendStore extends FileBackend {
* The cache key is salted for a while to prevent race conditions.
*
* @param string $container Resolved container name
+ * @return void
*/
final protected function deleteContainerCache( $container ) {
if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
@@ -1586,12 +1436,11 @@ abstract class FileBackendStore extends FileBackend {
* used in a list of container names, storage paths, or FileOp objects.
* This loads the persistent cache values into the process cache.
*
- * @param $items Array
+ * @param Array $items
* @return void
*/
final protected function primeContainerCache( array $items ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$paths = array(); // list of storage paths
$contNames = array(); // (cache key => resolved container name)
@@ -1623,9 +1472,6 @@ abstract class FileBackendStore extends FileBackend {
// Populate the container process cache for the backend...
$this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
-
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
}
/**
@@ -1654,14 +1500,17 @@ abstract class FileBackendStore extends FileBackend {
* salting for the case when a file is created at a path were there was none before.
*
* @param string $path Storage path
- * @param $val mixed Information to cache
+ * @param array $val Stat information to cache
+ * @return void
*/
- final protected function setFileCache( $path, $val ) {
+ final protected function setFileCache( $path, array $val ) {
$path = FileBackend::normalizeStoragePath( $path );
if ( $path === null ) {
return; // invalid storage path
}
- $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
+ $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
+ $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
+ $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
}
/**
@@ -1671,6 +1520,7 @@ abstract class FileBackendStore extends FileBackend {
* a file is created at a path were there was none before.
*
* @param string $path Storage path
+ * @return void
*/
final protected function deleteFileCache( $path ) {
$path = FileBackend::normalizeStoragePath( $path );
@@ -1691,8 +1541,7 @@ abstract class FileBackendStore extends FileBackend {
* @return void
*/
final protected function primeFileCache( array $items ) {
- wfProfileIn( __METHOD__ );
- wfProfileIn( __METHOD__ . '-' . $this->name );
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$paths = array(); // list of storage paths
$pathNames = array(); // (cache key => storage path)
@@ -1726,9 +1575,6 @@ abstract class FileBackendStore extends FileBackend {
}
}
}
-
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
}
/**
@@ -1750,6 +1596,18 @@ abstract class FileBackendStore extends FileBackend {
}
return $opts;
}
+
+ /**
+ * Get the content type to use in HEAD/GET requests for a file
+ *
+ * @param string $storagePath
+ * @param string|null $content File data
+ * @param string|null $fsPath File system path
+ * @return MIME type
+ */
+ protected function getContentType( $storagePath, $content, $fsPath ) {
+ return call_user_func_array( $this->mimeCallback, func_get_args() );
+ }
}
/**
@@ -1786,26 +1644,20 @@ abstract class FileBackendStoreOpHandle {
*
* @ingroup FileBackend
*/
-abstract class FileBackendStoreShardListIterator implements Iterator {
+abstract class FileBackendStoreShardListIterator extends FilterIterator {
/** @var FileBackendStore */
protected $backend;
/** @var Array */
protected $params;
- /** @var Array */
- protected $shardSuffixes;
+
protected $container; // string; full container name
protected $directory; // string; resolved relative path
- /** @var Traversable */
- protected $iter;
- protected $curShard = 0; // integer
- protected $pos = 0; // integer
-
/** @var Array */
protected $multiShardPaths = array(); // (rel path => 1)
/**
- * @param $backend FileBackendStore
+ * @param FileBackendStore $backend
* @param string $container Full storage container name
* @param string $dir Storage directory relative to container
* @param array $suffixes List of container shard suffixes
@@ -1817,142 +1669,56 @@ abstract class FileBackendStoreShardListIterator implements Iterator {
$this->backend = $backend;
$this->container = $container;
$this->directory = $dir;
- $this->shardSuffixes = $suffixes;
$this->params = $params;
- }
-
- /**
- * @see Iterator::key()
- * @return integer
- */
- public function key() {
- return $this->pos;
- }
- /**
- * @see Iterator::valid()
- * @return bool
- */
- public function valid() {
- if ( $this->iter instanceof Iterator ) {
- return $this->iter->valid();
- } elseif ( is_array( $this->iter ) ) {
- return ( current( $this->iter ) !== false ); // no paths can have this value
+ $iter = new AppendIterator();
+ foreach ( $suffixes as $suffix ) {
+ $iter->append( $this->listFromShard( $this->container . $suffix ) );
}
- return false; // some failure?
- }
-
- /**
- * @see Iterator::current()
- * @return string|bool String or false
- */
- public function current() {
- return ( $this->iter instanceof Iterator )
- ? $this->iter->current()
- : current( $this->iter );
- }
- /**
- * @see Iterator::next()
- * @return void
- */
- public function next() {
- ++$this->pos;
- ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
- do {
- $continue = false; // keep scanning shards?
- $this->filterViaNext(); // filter out duplicates
- // Find the next non-empty shard if no elements are left
- if ( !$this->valid() ) {
- $this->nextShardIteratorIfNotValid();
- $continue = $this->valid(); // re-filter unless we ran out of shards
- }
- } while ( $continue );
- }
-
- /**
- * @see Iterator::rewind()
- * @return void
- */
- public function rewind() {
- $this->pos = 0;
- $this->curShard = 0;
- $this->setIteratorFromCurrentShard();
- do {
- $continue = false; // keep scanning shards?
- $this->filterViaNext(); // filter out duplicates
- // Find the next non-empty shard if no elements are left
- if ( !$this->valid() ) {
- $this->nextShardIteratorIfNotValid();
- $continue = $this->valid(); // re-filter unless we ran out of shards
- }
- } while ( $continue );
- }
-
- /**
- * Filter out duplicate items by advancing to the next ones
- */
- protected function filterViaNext() {
- while ( $this->valid() ) {
- $rel = $this->iter->current(); // path relative to given directory
- $path = $this->params['dir'] . "/{$rel}"; // full storage path
- if ( $this->backend->isSingleShardPathInternal( $path ) ) {
- break; // path is only on one shard; no issue with duplicates
- } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
- // Don't keep listing paths that are on multiple shards
- ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
- } else {
- $this->multiShardPaths[$rel] = 1;
- break;
- }
- }
+ parent::__construct( $iter );
}
- /**
- * If the list iterator for this container shard is out of items,
- * then move on to the next container that has items.
- * If there are none, then it advances to the last container.
- */
- protected function nextShardIteratorIfNotValid() {
- while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) {
- $this->setIteratorFromCurrentShard();
+ public function accept() {
+ $rel = $this->getInnerIterator()->current(); // path relative to given directory
+ $path = $this->params['dir'] . "/{$rel}"; // full storage path
+ if ( $this->backend->isSingleShardPathInternal( $path ) ) {
+ return true; // path is only on one shard; no issue with duplicates
+ } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
+ // Don't keep listing paths that are on multiple shards
+ return false;
+ } else {
+ $this->multiShardPaths[$rel] = 1;
+ return true;
}
}
- /**
- * Set the list iterator to that of the current container shard
- */
- protected function setIteratorFromCurrentShard() {
- $this->iter = $this->listFromShard(
- $this->container . $this->shardSuffixes[$this->curShard],
- $this->directory, $this->params );
- // Start loading results so that current() works
- if ( $this->iter ) {
- ( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter );
- }
+ public function rewind() {
+ parent::rewind();
+ $this->multiShardPaths = array();
}
/**
* Get the list for a given container shard
*
* @param string $container Resolved container name
- * @param string $dir Resolved path relative to container
- * @param array $params
- * @return Traversable|Array|null
+ * @return Iterator
*/
- abstract protected function listFromShard( $container, $dir, array $params );
+ abstract protected function listFromShard( $container );
}
/**
* Iterator for listing directories
*/
class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
- /**
- * @see FileBackendStoreShardListIterator::listFromShard()
- * @return Array|null|Traversable
- */
- protected function listFromShard( $container, $dir, array $params ) {
- return $this->backend->getDirectoryListInternal( $container, $dir, $params );
+ protected function listFromShard( $container ) {
+ $list = $this->backend->getDirectoryListInternal(
+ $container, $this->directory, $this->params );
+ if ( $list === null ) {
+ return new ArrayIterator( array() );
+ } else {
+ return is_array( $list ) ? new ArrayIterator( $list ) : $list;
+ }
}
}
@@ -1960,11 +1726,13 @@ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator
* Iterator for listing regular files
*/
class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
- /**
- * @see FileBackendStoreShardListIterator::listFromShard()
- * @return Array|null|Traversable
- */
- protected function listFromShard( $container, $dir, array $params ) {
- return $this->backend->getFileListInternal( $container, $dir, $params );
+ protected function listFromShard( $container ) {
+ $list = $this->backend->getFileListInternal(
+ $container, $this->directory, $this->params );
+ if ( $list === null ) {
+ return new ArrayIterator( array() );
+ } else {
+ return is_array( $list ) ? new ArrayIterator( $list ) : $list;
+ }
}
}
generated by cgit v1.2.3-54-g00ecf (git 2.45.2) at 2024-11-23 03:38:06 +0000