summaryrefslogtreecommitdiff
path: root/includes/filebackend/FileBackendStore.php
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2013-01-18 16:46:04 +0100
committerPierre Schmitz <pierre@archlinux.de>2013-01-18 16:46:04 +0100
commit63601400e476c6cf43d985f3e7b9864681695ed4 (patch)
treef7846203a952e38aaf66989d0a4702779f549962 /includes/filebackend/FileBackendStore.php
parent8ff01378c9e0207f9169b81966a51def645b6a51 (diff)
Update to MediaWiki 1.20.2
this update includes: * adjusted Arch Linux skin * updated FluxBBAuthPlugin * patch for https://bugzilla.wikimedia.org/show_bug.cgi?id=44024
Diffstat (limited to 'includes/filebackend/FileBackendStore.php')
-rw-r--r--includes/filebackend/FileBackendStore.php1766
1 files changed, 1766 insertions, 0 deletions
diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php
new file mode 100644
index 00000000..083dfea9
--- /dev/null
+++ b/includes/filebackend/FileBackendStore.php
@@ -0,0 +1,1766 @@
+<?php
+/**
+ * Base class for all backends using particular storage medium.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup FileBackend
+ * @author Aaron Schulz
+ */
+
+/**
+ * @brief Base class for all backends using particular storage medium.
+ *
+ * This class defines the methods as abstract that subclasses must implement.
+ * Outside callers should *not* use functions with "Internal" in the name.
+ *
+ * The FileBackend operations are implemented using basic functions
+ * such as storeInternal(), copyInternal(), deleteInternal() and the like.
+ * This class is also responsible for path resolution and sanitization.
+ *
+ * @ingroup FileBackend
+ * @since 1.19
+ */
+abstract class FileBackendStore extends FileBackend {
+ /** @var BagOStuff */
+ protected $memCache;
+ /** @var ProcessCacheLRU */
+ protected $cheapCache; // Map of paths to small (RAM/disk) cache items
+ /** @var ProcessCacheLRU */
+ protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
+
+ /** @var Array Map of container names to sharding settings */
+ protected $shardViaHashLevels = array(); // (container name => config array)
+
+ protected $maxFileSize = 4294967296; // integer bytes (4GiB)
+
+ /**
+ * @see FileBackend::__construct()
+ *
+ * @param $config Array
+ */
+ public function __construct( array $config ) {
+ parent::__construct( $config );
+ $this->memCache = new EmptyBagOStuff(); // disabled by default
+ $this->cheapCache = new ProcessCacheLRU( 300 );
+ $this->expensiveCache = new ProcessCacheLRU( 5 );
+ }
+
+ /**
+ * Get the maximum allowable file size given backend
+ * medium restrictions and basic performance constraints.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * @return integer Bytes
+ */
+ final public function maxFileSizeInternal() {
+ return $this->maxFileSize;
+ }
+
+ /**
+ * Check if a file can be created at a given storage path.
+ * FS backends should check if the parent directory exists and the file is writable.
+ * Backends using key/value stores should check if the container exists.
+ *
+ * @param $storagePath string
+ * @return bool
+ */
+ abstract public function isPathUsableInternal( $storagePath );
+
+ /**
+ * Create a file in the backend with the given contents.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * $params include:
+ * - content : the raw file contents
+ * - dst : destination storage path
+ * - overwrite : overwrite any file that exists at the destination
+ * - disposition : Content-Disposition header value for the destination
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function createInternal( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
+ $status = Status::newFatal( 'backend-fail-maxsize',
+ $params['dst'], $this->maxFileSizeInternal() );
+ } else {
+ $status = $this->doCreateInternal( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->deleteFileCache( $params['dst'] ); // persistent cache
+ }
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::createInternal()
+ */
+ abstract protected function doCreateInternal( array $params );
+
+ /**
+ * Store a file into the backend from a file on disk.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * $params include:
+ * - src : source path on disk
+ * - dst : destination storage path
+ * - overwrite : overwrite any file that exists at the destination
+ * - disposition : Content-Disposition header value for the destination
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function storeInternal( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
+ $status = Status::newFatal( 'backend-fail-maxsize',
+ $params['dst'], $this->maxFileSizeInternal() );
+ } else {
+ $status = $this->doStoreInternal( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->deleteFileCache( $params['dst'] ); // persistent cache
+ }
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::storeInternal()
+ */
+ abstract protected function doStoreInternal( array $params );
+
+ /**
+ * Copy a file from one storage path to another in the backend.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * $params include:
+ * - src : source storage path
+ * - dst : destination storage path
+ * - overwrite : overwrite any file that exists at the destination
+ * - disposition : Content-Disposition header value for the destination
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function copyInternal( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = $this->doCopyInternal( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->deleteFileCache( $params['dst'] ); // persistent cache
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::copyInternal()
+ */
+ abstract protected function doCopyInternal( array $params );
+
+ /**
+ * Delete a file at the storage path.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * $params include:
+ * - src : source storage path
+ * - ignoreMissingSource : do nothing if the source file does not exist
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function deleteInternal( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = $this->doDeleteInternal( $params );
+ $this->clearCache( array( $params['src'] ) );
+ $this->deleteFileCache( $params['src'] ); // persistent cache
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::deleteInternal()
+ */
+ abstract protected function doDeleteInternal( array $params );
+
+ /**
+ * Move a file from one storage path to another in the backend.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * $params include:
+ * - src : source storage path
+ * - dst : destination storage path
+ * - overwrite : overwrite any file that exists at the destination
+ * - disposition : Content-Disposition header value for the destination
+ * - async : Status will be returned immediately if supported.
+ * If the status is OK, then its value field will be
+ * set to a FileBackendStoreOpHandle object.
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function moveInternal( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = $this->doMoveInternal( $params );
+ $this->clearCache( array( $params['src'], $params['dst'] ) );
+ $this->deleteFileCache( $params['src'] ); // persistent cache
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->deleteFileCache( $params['dst'] ); // persistent cache
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::moveInternal()
+ * @return Status
+ */
+ protected function doMoveInternal( array $params ) {
+ unset( $params['async'] ); // two steps, won't work here :)
+ // Copy source to dest
+ $status = $this->copyInternal( $params );
+ if ( $status->isOK() ) {
+ // Delete source (only fails due to races or medium going down)
+ $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
+ $status->setResult( true, $status->value ); // ignore delete() errors
+ }
+ return $status;
+ }
+
+ /**
+ * No-op file operation that does nothing.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function nullInternal( array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackend::concatenate()
+ * @return Status
+ */
+ final public function concatenate( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = Status::newGood();
+
+ // Try to lock the source files for the scope of this function
+ $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
+ if ( $status->isOK() ) {
+ // Actually do the file concatenation...
+ $start_time = microtime( true );
+ $status->merge( $this->doConcatenate( $params ) );
+ $sec = microtime( true ) - $start_time;
+ if ( !$status->isOK() ) {
+ wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
+ count( $params['srcs'] ) . " file(s) [$sec sec]" );
+ }
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::concatenate()
+ * @return Status
+ */
+ protected function doConcatenate( array $params ) {
+ $status = Status::newGood();
+ $tmpPath = $params['dst']; // convenience
+
+ // Check that the specified temp file is valid...
+ wfSuppressWarnings();
+ $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
+ wfRestoreWarnings();
+ if ( !$ok ) { // not present or not empty
+ $status->fatal( 'backend-fail-opentemp', $tmpPath );
+ return $status;
+ }
+
+ // Build up the temp file using the source chunks (in order)...
+ $tmpHandle = fopen( $tmpPath, 'ab' );
+ if ( $tmpHandle === false ) {
+ $status->fatal( 'backend-fail-opentemp', $tmpPath );
+ return $status;
+ }
+ foreach ( $params['srcs'] as $virtualSource ) {
+ // Get a local FS version of the chunk
+ $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
+ if ( !$tmpFile ) {
+ $status->fatal( 'backend-fail-read', $virtualSource );
+ return $status;
+ }
+ // Get a handle to the local FS version
+ $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
+ if ( $sourceHandle === false ) {
+ fclose( $tmpHandle );
+ $status->fatal( 'backend-fail-read', $virtualSource );
+ return $status;
+ }
+ // Append chunk to file (pass chunk size to avoid magic quotes)
+ if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
+ fclose( $sourceHandle );
+ fclose( $tmpHandle );
+ $status->fatal( 'backend-fail-writetemp', $tmpPath );
+ return $status;
+ }
+ fclose( $sourceHandle );
+ }
+ if ( !fclose( $tmpHandle ) ) {
+ $status->fatal( 'backend-fail-closetemp', $tmpPath );
+ return $status;
+ }
+
+ clearstatcache(); // temp file changed
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::doPrepare()
+ * @return Status
+ */
+ final protected function doPrepare( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+
+ $status = Status::newGood();
+ list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status; // invalid storage path
+ }
+
+ if ( $shard !== null ) { // confined to a single container/shard
+ $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
+ } else { // directory is on several shards
+ wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
+ $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
+ }
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::doPrepare()
+ * @return Status
+ */
+ protected function doPrepareInternal( $container, $dir, array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackend::doSecure()
+ * @return Status
+ */
+ final protected function doSecure( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = Status::newGood();
+
+ list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status; // invalid storage path
+ }
+
+ if ( $shard !== null ) { // confined to a single container/shard
+ $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
+ } else { // directory is on several shards
+ wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
+ $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
+ }
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::doSecure()
+ * @return Status
+ */
+ protected function doSecureInternal( $container, $dir, array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackend::doPublish()
+ * @return Status
+ */
+ final protected function doPublish( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = Status::newGood();
+
+ list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status; // invalid storage path
+ }
+
+ if ( $shard !== null ) { // confined to a single container/shard
+ $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
+ } else { // directory is on several shards
+ wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
+ $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
+ }
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::doPublish()
+ * @return Status
+ */
+ protected function doPublishInternal( $container, $dir, array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackend::doClean()
+ * @return Status
+ */
+ final protected function doClean( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = Status::newGood();
+
+ // Recursive: first delete all empty subdirs recursively
+ if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
+ $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
+ if ( $subDirsRel !== null ) { // no errors
+ foreach ( $subDirsRel as $subDirRel ) {
+ $subDir = $params['dir'] . "/{$subDirRel}"; // full path
+ $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
+ }
+ }
+ }
+
+ list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status; // invalid storage path
+ }
+
+ // Attempt to lock this directory...
+ $filesLockEx = array( $params['dir'] );
+ $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
+ if ( !$status->isOK() ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status; // abort
+ }
+
+ if ( $shard !== null ) { // confined to a single container/shard
+ $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
+ $this->deleteContainerCache( $fullCont ); // purge cache
+ } else { // directory is on several shards
+ wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
+ $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
+ $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
+ }
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::doClean()
+ * @return Status
+ */
+ protected function doCleanInternal( $container, $dir, array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackend::fileExists()
+ * @return bool|null
+ */
+ final public function fileExists( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $stat = $this->getFileStat( $params );
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return ( $stat === null ) ? null : (bool)$stat; // null => failure
+ }
+
+ /**
+ * @see FileBackend::getFileTimestamp()
+ * @return bool
+ */
+ final public function getFileTimestamp( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $stat = $this->getFileStat( $params );
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $stat ? $stat['mtime'] : false;
+ }
+
+ /**
+ * @see FileBackend::getFileSize()
+ * @return bool
+ */
+ final public function getFileSize( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $stat = $this->getFileStat( $params );
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $stat ? $stat['size'] : false;
+ }
+
+ /**
+ * @see FileBackend::getFileStat()
+ * @return bool
+ */
+ final public function getFileStat( array $params ) {
+ $path = self::normalizeStoragePath( $params['src'] );
+ if ( $path === null ) {
+ return false; // invalid storage path
+ }
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $latest = !empty( $params['latest'] ); // use latest data?
+ if ( !$this->cheapCache->has( $path, 'stat' ) ) {
+ $this->primeFileCache( array( $path ) ); // check persistent cache
+ }
+ if ( $this->cheapCache->has( $path, 'stat' ) ) {
+ $stat = $this->cheapCache->get( $path, 'stat' );
+ // If we want the latest data, check that this cached
+ // value was in fact fetched with the latest available data.
+ if ( !$latest || $stat['latest'] ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $stat;
+ }
+ }
+ wfProfileIn( __METHOD__ . '-miss' );
+ wfProfileIn( __METHOD__ . '-miss-' . $this->name );
+ $stat = $this->doGetFileStat( $params );
+ wfProfileOut( __METHOD__ . '-miss-' . $this->name );
+ wfProfileOut( __METHOD__ . '-miss' );
+ if ( is_array( $stat ) ) { // don't cache negatives
+ $stat['latest'] = $latest;
+ $this->cheapCache->set( $path, 'stat', $stat );
+ $this->setFileCache( $path, $stat ); // update persistent cache
+ if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
+ $this->cheapCache->set( $path, 'sha1',
+ array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
+ }
+ } else {
+ wfDebug( __METHOD__ . ": File $path does not exist.\n" );
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $stat;
+ }
+
+ /**
+ * @see FileBackendStore::getFileStat()
+ */
+ abstract protected function doGetFileStat( array $params );
+
+ /**
+ * @see FileBackend::getFileContents()
+ * @return bool|string
+ */
+ public function getFileContents( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $tmpFile = $this->getLocalReference( $params );
+ if ( !$tmpFile ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return false;
+ }
+ wfSuppressWarnings();
+ $data = file_get_contents( $tmpFile->getPath() );
+ wfRestoreWarnings();
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $data;
+ }
+
+ /**
+ * @see FileBackend::getFileSha1Base36()
+ * @return bool|string
+ */
+ final public function getFileSha1Base36( array $params ) {
+ $path = self::normalizeStoragePath( $params['src'] );
+ if ( $path === null ) {
+ return false; // invalid storage path
+ }
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $latest = !empty( $params['latest'] ); // use latest data?
+ if ( $this->cheapCache->has( $path, 'sha1' ) ) {
+ $stat = $this->cheapCache->get( $path, 'sha1' );
+ // If we want the latest data, check that this cached
+ // value was in fact fetched with the latest available data.
+ if ( !$latest || $stat['latest'] ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $stat['hash'];
+ }
+ }
+ wfProfileIn( __METHOD__ . '-miss' );
+ wfProfileIn( __METHOD__ . '-miss-' . $this->name );
+ $hash = $this->doGetFileSha1Base36( $params );
+ wfProfileOut( __METHOD__ . '-miss-' . $this->name );
+ wfProfileOut( __METHOD__ . '-miss' );
+ if ( $hash ) { // don't cache negatives
+ $this->cheapCache->set( $path, 'sha1',
+ array( 'hash' => $hash, 'latest' => $latest ) );
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $hash;
+ }
+
+ /**
+ * @see FileBackendStore::getFileSha1Base36()
+ * @return bool|string
+ */
+ protected function doGetFileSha1Base36( array $params ) {
+ $fsFile = $this->getLocalReference( $params );
+ if ( !$fsFile ) {
+ return false;
+ } else {
+ return $fsFile->getSha1Base36();
+ }
+ }
+
+ /**
+ * @see FileBackend::getFileProps()
+ * @return Array
+ */
+ final public function getFileProps( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $fsFile = $this->getLocalReference( $params );
+ $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $props;
+ }
+
+ /**
+ * @see FileBackend::getLocalReference()
+ * @return TempFSFile|null
+ */
+ public function getLocalReference( array $params ) {
+ $path = self::normalizeStoragePath( $params['src'] );
+ if ( $path === null ) {
+ return null; // invalid storage path
+ }
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $latest = !empty( $params['latest'] ); // use latest data?
+ if ( $this->expensiveCache->has( $path, 'localRef' ) ) {
+ $val = $this->expensiveCache->get( $path, 'localRef' );
+ // If we want the latest data, check that this cached
+ // value was in fact fetched with the latest available data.
+ if ( !$latest || $val['latest'] ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $val['object'];
+ }
+ }
+ $tmpFile = $this->getLocalCopy( $params );
+ if ( $tmpFile ) { // don't cache negatives
+ $this->expensiveCache->set( $path, 'localRef',
+ array( 'object' => $tmpFile, 'latest' => $latest ) );
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $tmpFile;
+ }
+
+ /**
+ * @see FileBackend::streamFile()
+ * @return Status
+ */
+ final public function streamFile( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = Status::newGood();
+
+ $info = $this->getFileStat( $params );
+ if ( !$info ) { // let StreamFile handle the 404
+ $status->fatal( 'backend-fail-notexists', $params['src'] );
+ }
+
+ // Set output buffer and HTTP headers for stream
+ $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
+ $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
+ if ( $res == StreamFile::NOT_MODIFIED ) {
+ // do nothing; client cache is up to date
+ } elseif ( $res == StreamFile::READY_STREAM ) {
+ wfProfileIn( __METHOD__ . '-send' );
+ wfProfileIn( __METHOD__ . '-send-' . $this->name );
+ $status = $this->doStreamFile( $params );
+ wfProfileOut( __METHOD__ . '-send-' . $this->name );
+ wfProfileOut( __METHOD__ . '-send' );
+ } else {
+ $status->fatal( 'backend-fail-stream', $params['src'] );
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackendStore::streamFile()
+ * @return Status
+ */
+ protected function doStreamFile( array $params ) {
+ $status = Status::newGood();
+
+ $fsFile = $this->getLocalReference( $params );
+ if ( !$fsFile ) {
+ $status->fatal( 'backend-fail-stream', $params['src'] );
+ } elseif ( !readfile( $fsFile->getPath() ) ) {
+ $status->fatal( 'backend-fail-stream', $params['src'] );
+ }
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::directoryExists()
+ * @return bool|null
+ */
+ final public function directoryExists( array $params ) {
+ list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ return false; // invalid storage path
+ }
+ if ( $shard !== null ) { // confined to a single container/shard
+ return $this->doDirectoryExists( $fullCont, $dir, $params );
+ } else { // directory is on several shards
+ wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ $res = false; // response
+ foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
+ $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
+ if ( $exists ) {
+ $res = true;
+ break; // found one!
+ } elseif ( $exists === null ) { // error?
+ $res = null; // if we don't find anything, it is indeterminate
+ }
+ }
+ return $res;
+ }
+ }
+
+ /**
+ * @see FileBackendStore::directoryExists()
+ *
+ * @param $container string Resolved container name
+ * @param $dir string Resolved path relative to container
+ * @param $params Array
+ * @return bool|null
+ */
+ abstract protected function doDirectoryExists( $container, $dir, array $params );
+
+ /**
+ * @see FileBackend::getDirectoryList()
+ * @return Traversable|Array|null Returns null on failure
+ */
+ final public function getDirectoryList( array $params ) {
+ list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) { // invalid storage path
+ return null;
+ }
+ if ( $shard !== null ) {
+ // File listing is confined to a single container/shard
+ return $this->getDirectoryListInternal( $fullCont, $dir, $params );
+ } else {
+ wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
+ // File listing spans multiple containers/shards
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ return new FileBackendStoreShardDirIterator( $this,
+ $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
+ }
+ }
+
+ /**
+ * Do not call this function from places outside FileBackend
+ *
+ * @see FileBackendStore::getDirectoryList()
+ *
+ * @param $container string Resolved container name
+ * @param $dir string Resolved path relative to container
+ * @param $params Array
+ * @return Traversable|Array|null Returns null on failure
+ */
+ abstract public function getDirectoryListInternal( $container, $dir, array $params );
+
+ /**
+ * @see FileBackend::getFileList()
+ * @return Traversable|Array|null Returns null on failure
+ */
+ final public function getFileList( array $params ) {
+ list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) { // invalid storage path
+ return null;
+ }
+ if ( $shard !== null ) {
+ // File listing is confined to a single container/shard
+ return $this->getFileListInternal( $fullCont, $dir, $params );
+ } else {
+ wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
+ // File listing spans multiple containers/shards
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
+ return new FileBackendStoreShardFileIterator( $this,
+ $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
+ }
+ }
+
+ /**
+ * Do not call this function from places outside FileBackend
+ *
+ * @see FileBackendStore::getFileList()
+ *
+ * @param $container string Resolved container name
+ * @param $dir string Resolved path relative to container
+ * @param $params Array
+ * @return Traversable|Array|null Returns null on failure
+ */
+ abstract public function getFileListInternal( $container, $dir, array $params );
+
+ /**
+ * Return a list of FileOp objects from a list of operations.
+ * Do not call this function from places outside FileBackend.
+ *
+ * The result must have the same number of items as the input.
+ * An exception is thrown if an unsupported operation is requested.
+ *
+ * @param $ops Array Same format as doOperations()
+ * @return Array List of FileOp objects
+ * @throws MWException
+ */
+ final public function getOperationsInternal( array $ops ) {
+ $supportedOps = array(
+ 'store' => 'StoreFileOp',
+ 'copy' => 'CopyFileOp',
+ 'move' => 'MoveFileOp',
+ 'delete' => 'DeleteFileOp',
+ 'create' => 'CreateFileOp',
+ 'null' => 'NullFileOp'
+ );
+
+ $performOps = array(); // array of FileOp objects
+ // Build up ordered array of FileOps...
+ foreach ( $ops as $operation ) {
+ $opName = $operation['op'];
+ if ( isset( $supportedOps[$opName] ) ) {
+ $class = $supportedOps[$opName];
+ // Get params for this operation
+ $params = $operation;
+ // Append the FileOp class
+ $performOps[] = new $class( $this, $params );
+ } else {
+ throw new MWException( "Operation '$opName' is not supported." );
+ }
+ }
+
+ return $performOps;
+ }
+
+ /**
+ * Get a list of storage paths to lock for a list of operations
+ * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
+ * each corresponding to a list of storage paths to be locked.
+ *
+ * @param $performOps Array List of FileOp objects
+ * @return Array ('sh' => list of paths, 'ex' => list of paths)
+ */
+ final public function getPathsToLockForOpsInternal( array $performOps ) {
+ // Build up a list of files to lock...
+ $paths = array( 'sh' => array(), 'ex' => array() );
+ foreach ( $performOps as $fileOp ) {
+ $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
+ $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
+ }
+ // Optimization: if doing an EX lock anyway, don't also set an SH one
+ $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
+ // Get a shared lock on the parent directory of each path changed
+ $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
+
+ return $paths;
+ }
+
+ /**
+ * @see FileBackend::getScopedLocksForOps()
+ * @return Array
+ */
+ public function getScopedLocksForOps( array $ops, Status $status ) {
+ $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
+ return array(
+ $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
+ $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
+ );
+ }
+
+ /**
+ * @see FileBackend::doOperationsInternal()
+ * @return Status
+ */
+ final protected function doOperationsInternal( array $ops, array $opts ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = Status::newGood();
+
+ // Build up a list of FileOps...
+ $performOps = $this->getOperationsInternal( $ops );
+
+ // Acquire any locks as needed...
+ if ( empty( $opts['nonLocking'] ) ) {
+ // Build up a list of files to lock...
+ $paths = $this->getPathsToLockForOpsInternal( $performOps );
+ // Try to lock those files for the scope of this function...
+ $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
+ $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
+ if ( !$status->isOK() ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status; // abort
+ }
+ }
+
+ // Clear any file cache entries (after locks acquired)
+ if ( empty( $opts['preserveCache'] ) ) {
+ $this->clearCache();
+ }
+
+ // Load from the persistent file and container caches
+ $this->primeFileCache( $performOps );
+ $this->primeContainerCache( $performOps );
+
+ // Actually attempt the operation batch...
+ $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
+
+ // Merge errors into status fields
+ $status->merge( $subStatus );
+ $status->success = $subStatus->success; // not done in merge()
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::doQuickOperationsInternal()
+ * @return Status
+ * @throws MWException
+ */
+ final protected function doQuickOperationsInternal( array $ops ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ $status = Status::newGood();
+
+ $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
+ $async = ( $this->parallelize === 'implicit' );
+ $maxConcurrency = $this->concurrency; // throttle
+
+ $statuses = array(); // array of (index => Status)
+ $fileOpHandles = array(); // list of (index => handle) arrays
+ $curFileOpHandles = array(); // current handle batch
+ // Perform the sync-only ops and build up op handles for the async ops...
+ foreach ( $ops as $index => $params ) {
+ if ( !in_array( $params['op'], $supportedOps ) ) {
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ throw new MWException( "Operation '{$params['op']}' is not supported." );
+ }
+ $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
+ $subStatus = $this->$method( array( 'async' => $async ) + $params );
+ if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
+ if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
+ $fileOpHandles[] = $curFileOpHandles; // push this batch
+ $curFileOpHandles = array();
+ }
+ $curFileOpHandles[$index] = $subStatus->value; // keep index
+ } else { // error or completed
+ $statuses[$index] = $subStatus; // keep index
+ }
+ }
+ if ( count( $curFileOpHandles ) ) {
+ $fileOpHandles[] = $curFileOpHandles; // last batch
+ }
+ // Do all the async ops that can be done concurrently...
+ foreach ( $fileOpHandles as $fileHandleBatch ) {
+ $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
+ }
+ // Marshall and merge all the responses...
+ foreach ( $statuses as $index => $subStatus ) {
+ $status->merge( $subStatus );
+ if ( $subStatus->isOK() ) {
+ $status->success[$index] = true;
+ ++$status->successCount;
+ } else {
+ $status->success[$index] = false;
+ ++$status->failCount;
+ }
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $status;
+ }
+
+ /**
+ * Execute a list of FileBackendStoreOpHandle handles in parallel.
+ * The resulting Status object fields will correspond
+ * to the order in which the handles where given.
+ *
+ * @param $handles Array List of FileBackendStoreOpHandle objects
+ * @return Array Map of Status objects
+ * @throws MWException
+ */
+ final public function executeOpHandlesInternal( array $fileOpHandles ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+ foreach ( $fileOpHandles as $fileOpHandle ) {
+ if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
+ throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
+ } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
+ throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
+ }
+ }
+ $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
+ foreach ( $fileOpHandles as $fileOpHandle ) {
+ $fileOpHandle->closeResources();
+ }
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $res;
+ }
+
+ /**
+ * @see FileBackendStore::executeOpHandlesInternal()
+ * @return Array List of corresponding Status objects
+ */
+ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
+ foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
+ throw new MWException( "This backend supports no asynchronous operations." );
+ }
+ return array();
+ }
+
+ /**
+ * @see FileBackend::preloadCache()
+ */
+ final public function preloadCache( array $paths ) {
+ $fullConts = array(); // full container names
+ foreach ( $paths as $path ) {
+ list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
+ $fullConts[] = $fullCont;
+ }
+ // Load from the persistent file and container caches
+ $this->primeContainerCache( $fullConts );
+ $this->primeFileCache( $paths );
+ }
+
+ /**
+ * @see FileBackend::clearCache()
+ */
+ final public function clearCache( array $paths = null ) {
+ if ( is_array( $paths ) ) {
+ $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
+ $paths = array_filter( $paths, 'strlen' ); // remove nulls
+ }
+ if ( $paths === null ) {
+ $this->cheapCache->clear();
+ $this->expensiveCache->clear();
+ } else {
+ foreach ( $paths as $path ) {
+ $this->cheapCache->clear( $path );
+ $this->expensiveCache->clear( $path );
+ }
+ }
+ $this->doClearCache( $paths );
+ }
+
+ /**
+ * Clears any additional stat caches for storage paths
+ *
+ * @see FileBackend::clearCache()
+ *
+ * @param $paths Array Storage paths (optional)
+ * @return void
+ */
+ protected function doClearCache( array $paths = null ) {}
+
+ /**
+ * Is this a key/value store where directories are just virtual?
+ * Virtual directories exists in so much as files exists that are
+ * prefixed with the directory path followed by a forward slash.
+ *
+ * @return bool
+ */
+ abstract protected function directoriesAreVirtual();
+
+ /**
+ * Check if a container name is valid.
+ * This checks for for length and illegal characters.
+ *
+ * @param $container string
+ * @return bool
+ */
+ final protected static function isValidContainerName( $container ) {
+ // This accounts for Swift and S3 restrictions while leaving room
+ // for things like '.xxx' (hex shard chars) or '.seg' (segments).
+ // This disallows directory separators or traversal characters.
+ // Note that matching strings URL encode to the same string;
+ // in Swift, the length restriction is *after* URL encoding.
+ return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
+ }
+
+ /**
+ * Splits a storage path into an internal container name,
+ * an internal relative file name, and a container shard suffix.
+ * Any shard suffix is already appended to the internal container name.
+ * This also checks that the storage path is valid and within this backend.
+ *
+ * If the container is sharded but a suffix could not be determined,
+ * this means that the path can only refer to a directory and can only
+ * be scanned by looking in all the container shards.
+ *
+ * @param $storagePath string
+ * @return Array (container, path, container suffix) or (null, null, null) if invalid
+ */
+ final protected function resolveStoragePath( $storagePath ) {
+ list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
+ if ( $backend === $this->name ) { // must be for this backend
+ $relPath = self::normalizeContainerPath( $relPath );
+ if ( $relPath !== null ) {
+ // Get shard for the normalized path if this container is sharded
+ $cShard = $this->getContainerShard( $container, $relPath );
+ // Validate and sanitize the relative path (backend-specific)
+ $relPath = $this->resolveContainerPath( $container, $relPath );
+ if ( $relPath !== null ) {
+ // Prepend any wiki ID prefix to the container name
+ $container = $this->fullContainerName( $container );
+ if ( self::isValidContainerName( $container ) ) {
+ // Validate and sanitize the container name (backend-specific)
+ $container = $this->resolveContainerName( "{$container}{$cShard}" );
+ if ( $container !== null ) {
+ return array( $container, $relPath, $cShard );
+ }
+ }
+ }
+ }
+ }
+ return array( null, null, null );
+ }
+
+ /**
+ * Like resolveStoragePath() except null values are returned if
+ * the container is sharded and the shard could not be determined.
+ *
+ * @see FileBackendStore::resolveStoragePath()
+ *
+ * @param $storagePath string
+ * @return Array (container, path) or (null, null) if invalid
+ */
+ final protected function resolveStoragePathReal( $storagePath ) {
+ list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
+ if ( $cShard !== null ) {
+ return array( $container, $relPath );
+ }
+ return array( null, null );
+ }
+
+ /**
+ * Get the container name shard suffix for a given path.
+ * Any empty suffix means the container is not sharded.
+ *
+ * @param $container string Container name
+ * @param $relPath string Storage path relative to the container
+ * @return string|null Returns null if shard could not be determined
+ */
+ final protected function getContainerShard( $container, $relPath ) {
+ list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
+ if ( $levels == 1 || $levels == 2 ) {
+ // Hash characters are either base 16 or 36
+ $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
+ // Get a regex that represents the shard portion of paths.
+ // The concatenation of the captures gives us the shard.
+ if ( $levels === 1 ) { // 16 or 36 shards per container
+ $hashDirRegex = '(' . $char . ')';
+ } else { // 256 or 1296 shards per container
+ if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
+ $hashDirRegex = $char . '/(' . $char . '{2})';
+ } else { // short hash dir format (e.g. "a/b/c")
+ $hashDirRegex = '(' . $char . ')/(' . $char . ')';
+ }
+ }
+ // Allow certain directories to be above the hash dirs so as
+ // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
+ // They must be 2+ chars to avoid any hash directory ambiguity.
+ $m = array();
+ if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
+ return '.' . implode( '', array_slice( $m, 1 ) );
+ }
+ return null; // failed to match
+ }
+ return ''; // no sharding
+ }
+
+ /**
+ * Check if a storage path maps to a single shard.
+ * Container dirs like "a", where the container shards on "x/xy",
+ * can reside on several shards. Such paths are tricky to handle.
+ *
+ * @param $storagePath string Storage path
+ * @return bool
+ */
+ final public function isSingleShardPathInternal( $storagePath ) {
+ list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
+ return ( $shard !== null );
+ }
+
+ /**
+ * Get the sharding config for a container.
+ * If greater than 0, then all file storage paths within
+ * the container are required to be hashed accordingly.
+ *
+ * @param $container string
+ * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
+ */
+ final protected function getContainerHashLevels( $container ) {
+ if ( isset( $this->shardViaHashLevels[$container] ) ) {
+ $config = $this->shardViaHashLevels[$container];
+ $hashLevels = (int)$config['levels'];
+ if ( $hashLevels == 1 || $hashLevels == 2 ) {
+ $hashBase = (int)$config['base'];
+ if ( $hashBase == 16 || $hashBase == 36 ) {
+ return array( $hashLevels, $hashBase, $config['repeat'] );
+ }
+ }
+ }
+ return array( 0, 0, false ); // no sharding
+ }
+
+ /**
+ * Get a list of full container shard suffixes for a container
+ *
+ * @param $container string
+ * @return Array
+ */
+ final protected function getContainerSuffixes( $container ) {
+ $shards = array();
+ list( $digits, $base ) = $this->getContainerHashLevels( $container );
+ if ( $digits > 0 ) {
+ $numShards = pow( $base, $digits );
+ for ( $index = 0; $index < $numShards; $index++ ) {
+ $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
+ }
+ }
+ return $shards;
+ }
+
+ /**
+ * Get the full container name, including the wiki ID prefix
+ *
+ * @param $container string
+ * @return string
+ */
+ final protected function fullContainerName( $container ) {
+ if ( $this->wikiId != '' ) {
+ return "{$this->wikiId}-$container";
+ } else {
+ return $container;
+ }
+ }
+
+ /**
+ * Resolve a container name, checking if it's allowed by the backend.
+ * This is intended for internal use, such as encoding illegal chars.
+ * Subclasses can override this to be more restrictive.
+ *
+ * @param $container string
+ * @return string|null
+ */
+ protected function resolveContainerName( $container ) {
+ return $container;
+ }
+
+ /**
+ * Resolve a relative storage path, checking if it's allowed by the backend.
+ * This is intended for internal use, such as encoding illegal chars or perhaps
+ * getting absolute paths (e.g. FS based backends). Note that the relative path
+ * may be the empty string (e.g. the path is simply to the container).
+ *
+ * @param $container string Container name
+ * @param $relStoragePath string Storage path relative to the container
+ * @return string|null Path or null if not valid
+ */
+ protected function resolveContainerPath( $container, $relStoragePath ) {
+ return $relStoragePath;
+ }
+
+ /**
+ * Get the cache key for a container
+ *
+ * @param $container string Resolved container name
+ * @return string
+ */
+ private function containerCacheKey( $container ) {
+ return wfMemcKey( 'backend', $this->getName(), 'container', $container );
+ }
+
+ /**
+ * Set the cached info for a container
+ *
+ * @param $container string Resolved container name
+ * @param $val mixed Information to cache
+ */
+ final protected function setContainerCache( $container, $val ) {
+ $this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 );
+ }
+
+ /**
+ * Delete the cached info for a container.
+ * The cache key is salted for a while to prevent race conditions.
+ *
+ * @param $container string Resolved container name
+ */
+ final protected function deleteContainerCache( $container ) {
+ if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
+ trigger_error( "Unable to delete stat cache for container $container." );
+ }
+ }
+
+ /**
+ * Do a batch lookup from cache for container stats for all containers
+ * used in a list of container names, storage paths, or FileOp objects.
+ *
+ * @param $items Array
+ * @return void
+ */
+ final protected function primeContainerCache( array $items ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+
+ $paths = array(); // list of storage paths
+ $contNames = array(); // (cache key => resolved container name)
+ // Get all the paths/containers from the items...
+ foreach ( $items as $item ) {
+ if ( $item instanceof FileOp ) {
+ $paths = array_merge( $paths, $item->storagePathsRead() );
+ $paths = array_merge( $paths, $item->storagePathsChanged() );
+ } elseif ( self::isStoragePath( $item ) ) {
+ $paths[] = $item;
+ } elseif ( is_string( $item ) ) { // full container name
+ $contNames[$this->containerCacheKey( $item )] = $item;
+ }
+ }
+ // Get all the corresponding cache keys for paths...
+ foreach ( $paths as $path ) {
+ list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
+ if ( $fullCont !== null ) { // valid path for this backend
+ $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
+ }
+ }
+
+ $contInfo = array(); // (resolved container name => cache value)
+ // Get all cache entries for these container cache keys...
+ $values = $this->memCache->getMulti( array_keys( $contNames ) );
+ foreach ( $values as $cacheKey => $val ) {
+ $contInfo[$contNames[$cacheKey]] = $val;
+ }
+
+ // Populate the container process cache for the backend...
+ $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ }
+
+ /**
+ * Fill the backend-specific process cache given an array of
+ * resolved container names and their corresponding cached info.
+ * Only containers that actually exist should appear in the map.
+ *
+ * @param $containerInfo Array Map of resolved container names to cached info
+ * @return void
+ */
+ protected function doPrimeContainerCache( array $containerInfo ) {}
+
+ /**
+ * Get the cache key for a file path
+ *
+ * @param $path string Storage path
+ * @return string
+ */
+ private function fileCacheKey( $path ) {
+ return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
+ }
+
+ /**
+ * Set the cached stat info for a file path.
+ * Negatives (404s) are not cached. By not caching negatives, we can skip cache
+ * salting for the case when a file is created at a path were there was none before.
+ *
+ * @param $path string Storage path
+ * @param $val mixed Information to cache
+ */
+ final protected function setFileCache( $path, $val ) {
+ $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
+ }
+
+ /**
+ * Delete the cached stat info for a file path.
+ * The cache key is salted for a while to prevent race conditions.
+ *
+ * @param $path string Storage path
+ */
+ final protected function deleteFileCache( $path ) {
+ if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
+ trigger_error( "Unable to delete stat cache for file $path." );
+ }
+ }
+
+ /**
+ * Do a batch lookup from cache for file stats for all paths
+ * used in a list of storage paths or FileOp objects.
+ *
+ * @param $items Array List of storage paths or FileOps
+ * @return void
+ */
+ final protected function primeFileCache( array $items ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+
+ $paths = array(); // list of storage paths
+ $pathNames = array(); // (cache key => storage path)
+ // Get all the paths/containers from the items...
+ foreach ( $items as $item ) {
+ if ( $item instanceof FileOp ) {
+ $paths = array_merge( $paths, $item->storagePathsRead() );
+ $paths = array_merge( $paths, $item->storagePathsChanged() );
+ } elseif ( self::isStoragePath( $item ) ) {
+ $paths[] = $item;
+ }
+ }
+ // Get all the corresponding cache keys for paths...
+ foreach ( $paths as $path ) {
+ list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
+ if ( $rel !== null ) { // valid path for this backend
+ $pathNames[$this->fileCacheKey( $path )] = $path;
+ }
+ }
+ // Get all cache entries for these container cache keys...
+ $values = $this->memCache->getMulti( array_keys( $pathNames ) );
+ foreach ( $values as $cacheKey => $val ) {
+ if ( is_array( $val ) ) {
+ $path = $pathNames[$cacheKey];
+ $this->cheapCache->set( $path, 'stat', $val );
+ if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
+ $this->cheapCache->set( $path, 'sha1',
+ array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
+ }
+ }
+ }
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ }
+}
+
+/**
+ * FileBackendStore helper class for performing asynchronous file operations.
+ *
+ * For example, calling FileBackendStore::createInternal() with the "async"
+ * param flag may result in a Status that contains this object as a value.
+ * This class is largely backend-specific and is mostly just "magic" to be
+ * passed to FileBackendStore::executeOpHandlesInternal().
+ */
+abstract class FileBackendStoreOpHandle {
+ /** @var Array */
+ public $params = array(); // params to caller functions
+ /** @var FileBackendStore */
+ public $backend;
+ /** @var Array */
+ public $resourcesToClose = array();
+
+ public $call; // string; name that identifies the function called
+
+ /**
+ * Close all open file handles
+ *
+ * @return void
+ */
+ public function closeResources() {
+ array_map( 'fclose', $this->resourcesToClose );
+ }
+}
+
+/**
+ * FileBackendStore helper function to handle listings that span container shards.
+ * Do not use this class from places outside of FileBackendStore.
+ *
+ * @ingroup FileBackend
+ */
+abstract class FileBackendStoreShardListIterator implements Iterator {
+ /** @var FileBackendStore */
+ protected $backend;
+ /** @var Array */
+ protected $params;
+ /** @var Array */
+ protected $shardSuffixes;
+ protected $container; // string; full container name
+ protected $directory; // string; resolved relative path
+
+ /** @var Traversable */
+ protected $iter;
+ protected $curShard = 0; // integer
+ protected $pos = 0; // integer
+
+ /** @var Array */
+ protected $multiShardPaths = array(); // (rel path => 1)
+
+ /**
+ * @param $backend FileBackendStore
+ * @param $container string Full storage container name
+ * @param $dir string Storage directory relative to container
+ * @param $suffixes Array List of container shard suffixes
+ * @param $params Array
+ */
+ public function __construct(
+ FileBackendStore $backend, $container, $dir, array $suffixes, array $params
+ ) {
+ $this->backend = $backend;
+ $this->container = $container;
+ $this->directory = $dir;
+ $this->shardSuffixes = $suffixes;
+ $this->params = $params;
+ }
+
+ /**
+ * @see Iterator::key()
+ * @return integer
+ */
+ public function key() {
+ return $this->pos;
+ }
+
+ /**
+ * @see Iterator::valid()
+ * @return bool
+ */
+ public function valid() {
+ if ( $this->iter instanceof Iterator ) {
+ return $this->iter->valid();
+ } elseif ( is_array( $this->iter ) ) {
+ return ( current( $this->iter ) !== false ); // no paths can have this value
+ }
+ return false; // some failure?
+ }
+
+ /**
+ * @see Iterator::current()
+ * @return string|bool String or false
+ */
+ public function current() {
+ return ( $this->iter instanceof Iterator )
+ ? $this->iter->current()
+ : current( $this->iter );
+ }
+
+ /**
+ * @see Iterator::next()
+ * @return void
+ */
+ public function next() {
+ ++$this->pos;
+ ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
+ do {
+ $continue = false; // keep scanning shards?
+ $this->filterViaNext(); // filter out duplicates
+ // Find the next non-empty shard if no elements are left
+ if ( !$this->valid() ) {
+ $this->nextShardIteratorIfNotValid();
+ $continue = $this->valid(); // re-filter unless we ran out of shards
+ }
+ } while ( $continue );
+ }
+
+ /**
+ * @see Iterator::rewind()
+ * @return void
+ */
+ public function rewind() {
+ $this->pos = 0;
+ $this->curShard = 0;
+ $this->setIteratorFromCurrentShard();
+ do {
+ $continue = false; // keep scanning shards?
+ $this->filterViaNext(); // filter out duplicates
+ // Find the next non-empty shard if no elements are left
+ if ( !$this->valid() ) {
+ $this->nextShardIteratorIfNotValid();
+ $continue = $this->valid(); // re-filter unless we ran out of shards
+ }
+ } while ( $continue );
+ }
+
+ /**
+ * Filter out duplicate items by advancing to the next ones
+ */
+ protected function filterViaNext() {
+ while ( $this->valid() ) {
+ $rel = $this->iter->current(); // path relative to given directory
+ $path = $this->params['dir'] . "/{$rel}"; // full storage path
+ if ( $this->backend->isSingleShardPathInternal( $path ) ) {
+ break; // path is only on one shard; no issue with duplicates
+ } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
+ // Don't keep listing paths that are on multiple shards
+ ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
+ } else {
+ $this->multiShardPaths[$rel] = 1;
+ break;
+ }
+ }
+ }
+
+ /**
+ * If the list iterator for this container shard is out of items,
+ * then move on to the next container that has items.
+ * If there are none, then it advances to the last container.
+ */
+ protected function nextShardIteratorIfNotValid() {
+ while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) {
+ $this->setIteratorFromCurrentShard();
+ }
+ }
+
+ /**
+ * Set the list iterator to that of the current container shard
+ */
+ protected function setIteratorFromCurrentShard() {
+ $this->iter = $this->listFromShard(
+ $this->container . $this->shardSuffixes[$this->curShard],
+ $this->directory, $this->params );
+ // Start loading results so that current() works
+ if ( $this->iter ) {
+ ( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter );
+ }
+ }
+
+ /**
+ * Get the list for a given container shard
+ *
+ * @param $container string Resolved container name
+ * @param $dir string Resolved path relative to container
+ * @param $params Array
+ * @return Traversable|Array|null
+ */
+ abstract protected function listFromShard( $container, $dir, array $params );
+}
+
+/**
+ * Iterator for listing directories
+ */
+class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
+ /**
+ * @see FileBackendStoreShardListIterator::listFromShard()
+ * @return Array|null|Traversable
+ */
+ protected function listFromShard( $container, $dir, array $params ) {
+ return $this->backend->getDirectoryListInternal( $container, $dir, $params );
+ }
+}
+
+/**
+ * Iterator for listing regular files
+ */
+class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
+ /**
+ * @see FileBackendStoreShardListIterator::listFromShard()
+ * @return Array|null|Traversable
+ */
+ protected function listFromShard( $container, $dir, array $params ) {
+ return $this->backend->getFileListInternal( $container, $dir, $params );
+ }
+}