diff options
Diffstat (limited to 'includes/filebackend')
23 files changed, 2561 insertions, 1834 deletions
diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php index 8f0a1334..1659c62a 100644 --- a/includes/filebackend/FSFile.php +++ b/includes/filebackend/FSFile.php @@ -27,26 +27,25 @@ * @ingroup FileBackend */ class FSFile { - protected $path; // path to file - protected $sha1Base36; // file SHA-1 in base 36 + /** @var string Path to file */ + protected $path; + + /** @var string File SHA-1 in base 36 */ + protected $sha1Base36; /** * Sets up the file object * * @param string $path Path to temporary file on local disk - * @throws MWException */ public function __construct( $path ) { - if ( FileBackend::isStoragePath( $path ) ) { - throw new MWException( __METHOD__ . " given storage path `$path`." ); - } $this->path = $path; } /** * Returns the file system path * - * @return String + * @return string */ public function getPath() { return $this->path; @@ -82,6 +81,7 @@ class FSFile { if ( $timestamp !== false ) { $timestamp = wfTimestamp( TS_MW, $timestamp ); } + return $timestamp; } @@ -98,7 +98,7 @@ class FSFile { * Get an associative array containing information about * a file with the given storage path. * - * @param Mixed $ext: the file extension, or true to extract it from the filename. + * @param string|bool $ext The file extension, or true to extract it from the filename. * Set it to false to ignore the extension. * * @return array @@ -118,9 +118,9 @@ class FSFile { $ext = self::extensionFromPath( $this->path ); } - # mime type according to file contents + # MIME type according to file contents $info['file-mime'] = $this->getMimeType(); - # logical mime type + # logical MIME type $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext ); list( $info['major_mime'], $info['minor_mime'] ) = File::splitMime( $info['mime'] ); @@ -147,13 +147,14 @@ class FSFile { } wfProfileOut( __METHOD__ ); + return $info; } /** * Placeholder file properties to use for files that don't exist * - * @return Array + * @return array */ public static function placeholderProps() { $info = array(); @@ -165,6 +166,7 @@ class FSFile { $info['width'] = 0; $info['height'] = 0; $info['bits'] = 0; + return $info; } @@ -172,7 +174,7 @@ class FSFile { * Exract image size information * * @param array $gis - * @return Array + * @return array */ protected function extractImageSizeInfo( array $gis ) { $info = array(); @@ -184,6 +186,7 @@ class FSFile { } else { $info['bits'] = 0; } + return $info; } @@ -202,6 +205,7 @@ class FSFile { if ( $this->sha1Base36 !== null && !$recache ) { wfProfileOut( __METHOD__ ); + return $this->sha1Base36; } @@ -214,6 +218,7 @@ class FSFile { } wfProfileOut( __METHOD__ ); + return $this->sha1Base36; } @@ -225,19 +230,21 @@ class FSFile { */ public static function extensionFromPath( $path ) { $i = strrpos( $path, '.' ); + return strtolower( $i ? substr( $path, $i + 1 ) : '' ); } /** * Get an associative array containing information about a file in the local filesystem. * - * @param string $path absolute local filesystem path - * @param Mixed $ext: the file extension, or true to extract it from the filename. - * Set it to false to ignore the extension. + * @param string $path Absolute local filesystem path + * @param string|bool $ext The file extension, or true to extract it from the filename. + * Set it to false to ignore the extension. * @return array */ public static function getPropsFromPath( $path, $ext = true ) { $fsFile = new self( $path ); + return $fsFile->getProps( $ext ); } @@ -253,6 +260,7 @@ class FSFile { */ public static function getSha1Base36FromPath( $path ) { $fsFile = new self( $path ); + return $fsFile->getSha1Base36(); } } diff --git a/includes/filebackend/FSFileBackend.php b/includes/filebackend/FSFileBackend.php index 6d642162..b99ffb62 100644 --- a/includes/filebackend/FSFileBackend.php +++ b/includes/filebackend/FSFileBackend.php @@ -39,14 +39,22 @@ * @since 1.19 */ class FSFileBackend extends FileBackendStore { - protected $basePath; // string; directory holding the container directories - /** @var Array Map of container names to root paths */ - protected $containerPaths = array(); // for custom container paths - protected $fileMode; // integer; file permission mode - protected $fileOwner; // string; required OS username to own files - protected $currentUser; // string; OS username running this script - - /** @var Array */ + /** @var string Directory holding the container directories */ + protected $basePath; + + /** @var array Map of container names to root paths for custom container paths */ + protected $containerPaths = array(); + + /** @var int File permission mode */ + protected $fileMode; + + /** @var string Required OS username to own files */ + protected $fileOwner; + + /** @var string OS username running this script */ + protected $currentUser; + + /** @var array */ protected $hadWarningErrors = array(); /** @@ -82,6 +90,10 @@ class FSFileBackend extends FileBackendStore { } } + public function getFeatures() { + return !wfIsWindows() ? FileBackend::ATTR_UNICODE_PATHS : 0; + } + protected function resolveContainerPath( $container, $relStoragePath ) { // Check that container has a root directory if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) { @@ -90,6 +102,7 @@ class FSFileBackend extends FileBackendStore { return $relStoragePath; } } + return null; } @@ -125,6 +138,7 @@ class FSFileBackend extends FileBackendStore { } elseif ( isset( $this->basePath ) ) { return "{$this->basePath}/{$fullCont}"; } + return null; // no container base path defined } @@ -144,6 +158,7 @@ class FSFileBackend extends FileBackendStore { if ( $relPath != '' ) { $fsPath .= "/{$relPath}"; } + return $fsPath; } @@ -174,6 +189,7 @@ class FSFileBackend extends FileBackendStore { $dest = $this->resolveToFSPath( $params['dst'] ); if ( $dest === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; } @@ -181,6 +197,7 @@ class FSFileBackend extends FileBackendStore { $tempFile = TempFSFile::factory( 'create_', 'tmp' ); if ( !$tempFile ) { $status->fatal( 'backend-fail-create', $params['dst'] ); + return $status; } $this->trapWarnings(); @@ -188,6 +205,7 @@ class FSFileBackend extends FileBackendStore { $this->untrapWarnings(); if ( $bytes === false ) { $status->fatal( 'backend-fail-create', $params['dst'] ); + return $status; } $cmd = implode( ' ', array( @@ -195,7 +213,13 @@ class FSFileBackend extends FileBackendStore { wfEscapeShellArg( $this->cleanPathSlashes( $tempFile->getPath() ) ), wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) ) ); - $status->value = new FSFileOpHandle( $this, $params, 'Create', $cmd, $dest ); + $handler = function ( $errors, Status $status, array $params, $cmd ) { + if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { + $status->fatal( 'backend-fail-create', $params['dst'] ); + trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output + } + }; + $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest ); $tempFile->bind( $status->value ); } else { // immediate write $this->trapWarnings(); @@ -203,6 +227,7 @@ class FSFileBackend extends FileBackendStore { $this->untrapWarnings(); if ( $bytes === false ) { $status->fatal( 'backend-fail-create', $params['dst'] ); + return $status; } $this->chmod( $dest ); @@ -211,22 +236,13 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FSFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseCreate( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - } - protected function doStoreInternal( array $params ) { $status = Status::newGood(); $dest = $this->resolveToFSPath( $params['dst'] ); if ( $dest === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; } @@ -236,7 +252,13 @@ class FSFileBackend extends FileBackendStore { wfEscapeShellArg( $this->cleanPathSlashes( $params['src'] ) ), wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) ) ); - $status->value = new FSFileOpHandle( $this, $params, 'Store', $cmd, $dest ); + $handler = function ( $errors, Status $status, array $params, $cmd ) { + if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { + $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output + } + }; + $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest ); } else { // immediate write $this->trapWarnings(); $ok = copy( $params['src'], $dest ); @@ -248,6 +270,7 @@ class FSFileBackend extends FileBackendStore { trigger_error( __METHOD__ . ": copy() failed but returned true." ); } $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + return $status; } $this->chmod( $dest ); @@ -256,28 +279,20 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FSFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseStore( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - } - protected function doCopyInternal( array $params ) { $status = Status::newGood(); $source = $this->resolveToFSPath( $params['src'] ); if ( $source === null ) { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; } $dest = $this->resolveToFSPath( $params['dst'] ); if ( $dest === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; } @@ -285,6 +300,7 @@ class FSFileBackend extends FileBackendStore { if ( empty( $params['ignoreMissingSource'] ) ) { $status->fatal( 'backend-fail-copy', $params['src'] ); } + return $status; // do nothing; either OK or bad status } @@ -294,7 +310,13 @@ class FSFileBackend extends FileBackendStore { wfEscapeShellArg( $this->cleanPathSlashes( $source ) ), wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) ) ); - $status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd, $dest ); + $handler = function ( $errors, Status $status, array $params, $cmd ) { + if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output + } + }; + $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest ); } else { // immediate write $this->trapWarnings(); $ok = ( $source === $dest ) ? true : copy( $source, $dest ); @@ -308,6 +330,7 @@ class FSFileBackend extends FileBackendStore { trigger_error( __METHOD__ . ": copy() failed but returned true." ); } $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + return $status; } $this->chmod( $dest ); @@ -316,28 +339,20 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FSFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseCopy( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - } - protected function doMoveInternal( array $params ) { $status = Status::newGood(); $source = $this->resolveToFSPath( $params['src'] ); if ( $source === null ) { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; } $dest = $this->resolveToFSPath( $params['dst'] ); if ( $dest === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; } @@ -345,6 +360,7 @@ class FSFileBackend extends FileBackendStore { if ( empty( $params['ignoreMissingSource'] ) ) { $status->fatal( 'backend-fail-move', $params['src'] ); } + return $status; // do nothing; either OK or bad status } @@ -354,7 +370,13 @@ class FSFileBackend extends FileBackendStore { wfEscapeShellArg( $this->cleanPathSlashes( $source ) ), wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) ) ); - $status->value = new FSFileOpHandle( $this, $params, 'Move', $cmd ); + $handler = function ( $errors, Status $status, array $params, $cmd ) { + if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output + } + }; + $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd ); } else { // immediate write $this->trapWarnings(); $ok = ( $source === $dest ) ? true : rename( $source, $dest ); @@ -362,6 +384,7 @@ class FSFileBackend extends FileBackendStore { clearstatcache(); // file no longer at source if ( !$ok ) { $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + return $status; } } @@ -369,22 +392,13 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FSFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseMove( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - } - protected function doDeleteInternal( array $params ) { $status = Status::newGood(); $source = $this->resolveToFSPath( $params['src'] ); if ( $source === null ) { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; } @@ -392,6 +406,7 @@ class FSFileBackend extends FileBackendStore { if ( empty( $params['ignoreMissingSource'] ) ) { $status->fatal( 'backend-fail-delete', $params['src'] ); } + return $status; // do nothing; either OK or bad status } @@ -400,13 +415,20 @@ class FSFileBackend extends FileBackendStore { wfIsWindows() ? 'DEL' : 'unlink', wfEscapeShellArg( $this->cleanPathSlashes( $source ) ) ) ); - $status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd ); + $handler = function ( $errors, Status $status, array $params, $cmd ) { + if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output + } + }; + $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd ); } else { // immediate write $this->trapWarnings(); $ok = unlink( $source ); $this->untrapWarnings(); if ( !$ok ) { $status->fatal( 'backend-fail-delete', $params['src'] ); + return $status; } } @@ -415,15 +437,11 @@ class FSFileBackend extends FileBackendStore { } /** - * @see FSFileBackend::doExecuteOpHandlesInternal() + * @param string $fullCont + * @param string $dirRel + * @param array $params + * @return Status */ - protected function _getResponseDelete( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - } - protected function doPrepareInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); @@ -444,6 +462,7 @@ class FSFileBackend extends FileBackendStore { if ( is_dir( $dir ) && !$existed ) { $status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) ); } + return $status; } @@ -471,6 +490,7 @@ class FSFileBackend extends FileBackendStore { $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" ); } } + return $status; } @@ -498,6 +518,7 @@ class FSFileBackend extends FileBackendStore { } $this->untrapWarnings(); } + return $status; } @@ -511,6 +532,7 @@ class FSFileBackend extends FileBackendStore { rmdir( $dir ); // remove directory if empty } $this->untrapWarnings(); + return $status; } @@ -557,7 +579,10 @@ class FSFileBackend extends FileBackendStore { /** * @see FileBackendStore::getDirectoryListInternal() - * @return Array|null + * @param string $fullCont + * @param string $dirRel + * @param array $params + * @return array|null */ public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) { list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); @@ -566,17 +591,23 @@ class FSFileBackend extends FileBackendStore { $exists = is_dir( $dir ); if ( !$exists ) { wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" ); + return array(); // nothing under this dir } elseif ( !is_readable( $dir ) ) { wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" ); + return null; // bad permissions? } + return new FSFileBackendDirList( $dir, $params ); } /** * @see FileBackendStore::getFileListInternal() - * @return Array|FSFileBackendFileList|null + * @param string $fullCont + * @param string $dirRel + * @param array $params + * @return array|FSFileBackendFileList|null */ public function getFileListInternal( $fullCont, $dirRel, array $params ) { list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); @@ -585,11 +616,14 @@ class FSFileBackend extends FileBackendStore { $exists = is_dir( $dir ); if ( !$exists ) { wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" ); + return array(); // nothing under this dir } elseif ( !is_readable( $dir ) ) { wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" ); + return null; // bad permissions? } + return new FSFileBackendFileList( $dir, $params ); } @@ -662,8 +696,8 @@ class FSFileBackend extends FileBackendStore { foreach ( $fileOpHandles as $index => $fileOpHandle ) { $status = Status::newGood(); - $function = '_getResponse' . $fileOpHandle->call; - $this->$function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd ); + $function = $fileOpHandle->call; + $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd ); $statuses[$index] = $status; if ( $status->isOK() && $fileOpHandle->chmodPath ) { $this->chmod( $fileOpHandle->chmodPath ); @@ -718,8 +752,6 @@ class FSFileBackend extends FileBackendStore { /** * Listen for E_WARNING errors and track whether any happen - * - * @return void */ protected function trapWarnings() { $this->hadWarningErrors[] = false; // push to stack @@ -737,7 +769,7 @@ class FSFileBackend extends FileBackendStore { } /** - * @param integer $errno + * @param int $errno * @param string $errstr * @return bool * @access private @@ -745,6 +777,7 @@ class FSFileBackend extends FileBackendStore { public function handleWarning( $errno, $errstr ) { wfDebugLog( 'FSFileBackend', $errstr ); // more detailed error logging $this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true; + return true; // suppress from PHP handler } } @@ -759,9 +792,9 @@ class FSFileOpHandle extends FileBackendStoreOpHandle { /** * @param FSFileBackend $backend * @param array $params - * @param string $call + * @param callable $call * @param string $cmd - * @param integer|null $chmodPath + * @param int|null $chmodPath */ public function __construct( FSFileBackend $backend, array $params, $call, $cmd, $chmodPath = null @@ -784,13 +817,18 @@ class FSFileOpHandle extends FileBackendStoreOpHandle { abstract class FSFileBackendList implements Iterator { /** @var Iterator */ protected $iter; - protected $suffixStart; // integer - protected $pos = 0; // integer - /** @var Array */ + + /** @var int */ + protected $suffixStart; + + /** @var int */ + protected $pos = 0; + + /** @var array */ protected $params = array(); /** - * @param string $dir file system directory + * @param string $dir File system directory * @param array $params */ public function __construct( $dir, array $params ) { @@ -811,7 +849,7 @@ abstract class FSFileBackendList implements Iterator { /** * Return an appropriate iterator object to wrap * - * @param string $dir file system directory + * @param string $dir File system directory * @return Iterator */ protected function initIterator( $dir ) { @@ -823,6 +861,7 @@ abstract class FSFileBackendList implements Iterator { # RecursiveDirectoryIterator extends FilesystemIterator. # FilesystemIterator::SKIP_DOTS default is inconsistent in PHP 5.3.x. $flags = FilesystemIterator::CURRENT_AS_SELF | FilesystemIterator::SKIP_DOTS; + return new RecursiveIteratorIterator( new RecursiveDirectoryIterator( $dir, $flags ), RecursiveIteratorIterator::CHILD_FIRST // include dirs @@ -832,7 +871,7 @@ abstract class FSFileBackendList implements Iterator { /** * @see Iterator::key() - * @return integer + * @return int */ public function key() { return $this->pos; @@ -848,7 +887,7 @@ abstract class FSFileBackendList implements Iterator { /** * @see Iterator::next() - * @return void + * @throws FileBackendError */ public function next() { try { @@ -862,7 +901,7 @@ abstract class FSFileBackendList implements Iterator { /** * @see Iterator::rewind() - * @return void + * @throws FileBackendError */ public function rewind() { $this->pos = 0; @@ -885,13 +924,14 @@ abstract class FSFileBackendList implements Iterator { /** * Filter out items by advancing to the next ones */ - protected function filterViaNext() {} + protected function filterViaNext() { + } /** * Return only the relative path and normalize slashes to FileBackend-style. * Uses the "real path" since the suffix is based upon that. * - * @param string $path + * @param string $dir * @return string */ protected function getRelPath( $dir ) { @@ -899,6 +939,7 @@ abstract class FSFileBackendList implements Iterator { if ( $path === false ) { $path = $dir; } + return strtr( substr( $path, $this->suffixStart ), '\\', '/' ); } } diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php index f586578b..8c0a61a1 100644 --- a/includes/filebackend/FileBackend.php +++ b/includes/filebackend/FileBackend.php @@ -47,12 +47,35 @@ * For legacy reasons, the FSFileBackend class allows manually setting the paths of * containers to ones that do not respect the "wiki ID". * - * In key/value stores, the container is the only hierarchy (the rest is emulated). + * In key/value (object) stores, containers are the only hierarchy (the rest is emulated). * FS-based backends are somewhat more restrictive due to the existence of real * directory files; a regular file cannot have the same name as a directory. Other * backends with virtual directories may not have this limitation. Callers should * store files in such a way that no files and directories are under the same path. * + * In general, this class allows for callers to access storage through the same + * interface, without regard to the underlying storage system. However, calling code + * must follow certain patterns and be aware of certain things to ensure compatibility: + * - a) Always call prepare() on the parent directory before trying to put a file there; + * key/value stores only need the container to exist first, but filesystems need + * all the parent directories to exist first (prepare() is aware of all this) + * - b) Always call clean() on a directory when it might become empty to avoid empty + * directory buildup on filesystems; key/value stores never have empty directories, + * so doing this helps preserve consistency in both cases + * - c) Likewise, do not rely on the existence of empty directories for anything; + * calling directoryExists() on a path that prepare() was previously called on + * will return false for key/value stores if there are no files under that path + * - d) Never alter the resulting FSFile returned from getLocalReference(), as it could + * either be a copy of the source file in /tmp or the original source file itself + * - e) Use a file layout that results in never attempting to store files over directories + * or directories over files; key/value stores allow this but filesystems do not + * - f) Use ASCII file names (e.g. base32, IDs, hashes) to avoid Unicode issues in Windows + * - g) Do not assume that move operations are atomic (difficult with key/value stores) + * - h) Do not assume that file stat or read operations always have immediate consistency; + * various methods have a "latest" flag that should always be used if up-to-date + * information is required (this trades performance for correctness as needed) + * - i) Do not assume that directory listings have immediate consistency + * * Methods of subclasses should avoid throwing exceptions at all costs. * As a corollary, external dependencies should be kept to a minimum. * @@ -60,58 +83,69 @@ * @since 1.19 */ abstract class FileBackend { - protected $name; // string; unique backend name - protected $wikiId; // string; unique wiki name - protected $readOnly; // string; read-only explanation message - protected $parallelize; // string; when to do operations in parallel - protected $concurrency; // integer; how many operations can be done in parallel + /** @var string Unique backend name */ + protected $name; + + /** @var string Unique wiki name */ + protected $wikiId; + + /** @var string Read-only explanation message */ + protected $readOnly; + + /** @var string When to do operations in parallel */ + protected $parallelize; + + /** @var int How many operations can be done in parallel */ + protected $concurrency; /** @var LockManager */ protected $lockManager; + /** @var FileJournal */ protected $fileJournal; + /** Bitfield flags for supported features */ + const ATTR_HEADERS = 1; // files can be tagged with standard HTTP headers + const ATTR_METADATA = 2; // files can be stored with metadata key/values + const ATTR_UNICODE_PATHS = 4; // files can have Unicode paths (not just ASCII) + /** * Create a new backend instance from configuration. * This should only be called from within FileBackendGroup. * - * $config includes: + * @param array $config Parameters include: * - name : The unique name of this backend. * This should consist of alphanumberic, '-', and '_' characters. * This name should not be changed after use (e.g. with journaling). * Note that the name is *not* used in actual container names. * - wikiId : Prefix to container names that is unique to this backend. - * If not provided, this defaults to the current wiki ID. * It should only consist of alphanumberic, '-', and '_' characters. * This ID is what avoids collisions if multiple logical backends * use the same storage system, so this should be set carefully. - * - lockManager : Registered name of a file lock manager to use. - * - fileJournal : File journal configuration; see FileJournal::factory(). - * Journals simply log changes to files stored in the backend. + * - lockManager : LockManager object to use for any file locking. + * If not provided, then no file locking will be enforced. + * - fileJournal : FileJournal object to use for logging changes to files. + * If not provided, then change journaling will be disabled. * - readOnly : Write operations are disallowed if this is a non-empty string. * It should be an explanation for the backend being read-only. * - parallelize : When to do file operations in parallel (when possible). * Allowed values are "implicit", "explicit" and "off". * - concurrency : How many file operations can be done in parallel. - * - * @param array $config - * @throws MWException + * @throws FileBackendException */ public function __construct( array $config ) { $this->name = $config['name']; + $this->wikiId = $config['wikiId']; // e.g. "my_wiki-en_" if ( !preg_match( '!^[a-zA-Z0-9-_]{1,255}$!', $this->name ) ) { - throw new MWException( "Backend name `{$this->name}` is invalid." ); + throw new FileBackendException( "Backend name '{$this->name}' is invalid." ); + } elseif ( !is_string( $this->wikiId ) ) { + throw new FileBackendException( "Backend wiki ID not provided for '{$this->name}'." ); } - $this->wikiId = isset( $config['wikiId'] ) - ? $config['wikiId'] - : wfWikiID(); // e.g. "my_wiki-en_" - $this->lockManager = ( $config['lockManager'] instanceof LockManager ) + $this->lockManager = isset( $config['lockManager'] ) ? $config['lockManager'] - : LockManagerGroup::singleton( $this->wikiId )->get( $config['lockManager'] ); + : new NullLockManager( array() ); $this->fileJournal = isset( $config['fileJournal'] ) - ? ( ( $config['fileJournal'] instanceof FileJournal ) - ? $config['fileJournal'] - : FileJournal::factory( $config['fileJournal'], $this->name ) ) + ? $config['fileJournal'] : FileJournal::factory( array( 'class' => 'NullFileJournal' ), $this->name ); $this->readOnly = isset( $config['readOnly'] ) ? (string)$config['readOnly'] @@ -165,6 +199,27 @@ abstract class FileBackend { } /** + * Get the a bitfield of extra features supported by the backend medium + * + * @return int Bitfield of FileBackend::ATTR_* flags + * @since 1.23 + */ + public function getFeatures() { + return self::ATTR_UNICODE_PATHS; + } + + /** + * Check if the backend medium supports a field of extra features + * + * @param int $bitfield Bitfield of FileBackend::ATTR_* flags + * @return bool + * @since 1.23 + */ + final public function hasFeatures( $bitfield ) { + return ( $this->getFeatures() & $bitfield ) === $bitfield; + } + + /** * This is the main entry point into the backend for write operations. * Callers supply an ordered list of operations to perform as a transaction. * Files will be locked, the stat cache cleared, and then the operations attempted. @@ -671,8 +726,7 @@ abstract class FileBackend { * otherwise safe from modification from other processes. Normally, * the file will be a new temp file, which should be adequate. * - * @param array $params Operation parameters - * $params include: + * @param array $params Operation parameters, include: * - srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) * - dst : file system path to 0-byte temp file * - parallelize : try to do operations in parallel when possible @@ -691,8 +745,7 @@ abstract class FileBackend { * However, setting them is not guaranteed to actually do anything. * Additional server configuration may be needed to achieve the desired effect. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * - noAccess : try to deny file access (since 1.20) * - noListing : try to deny file listing (since 1.20) @@ -721,8 +774,7 @@ abstract class FileBackend { * This is not guaranteed to actually make files or listings publically hidden. * Additional server configuration may be needed to achieve the desired effect. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * - noAccess : try to deny file access * - noListing : try to deny file listing @@ -752,8 +804,7 @@ abstract class FileBackend { * This is not guaranteed to actually make files or listings publically viewable. * Additional server configuration may be needed to achieve the desired effect. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * - access : try to allow file access * - listing : try to allow file listing @@ -779,8 +830,7 @@ abstract class FileBackend { * Backends using key/value stores may do nothing unless the directory * is that of an empty container, in which case it will be deleted. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * - recursive : recursively delete empty subdirectories first (since 1.20) * - bypassReadOnly : allow writes in read-only mode (since 1.20) @@ -807,12 +857,13 @@ abstract class FileBackend { * @return ScopedCallback|null */ final protected function getScopedPHPBehaviorForOps() { - if ( php_sapi_name() != 'cli' ) { // http://bugs.php.net/bug.php?id=47540 + if ( PHP_SAPI != 'cli' ) { // http://bugs.php.net/bug.php?id=47540 $old = ignore_user_abort( true ); // avoid half-finished operations - return new ScopedCallback( function() use ( $old ) { + return new ScopedCallback( function () use ( $old ) { ignore_user_abort( $old ); } ); } + return null; } @@ -820,8 +871,7 @@ abstract class FileBackend { * Check if a file exists at a storage path in the backend. * This returns false if only a directory exists at the path. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data * @return bool|null Returns null on failure @@ -831,8 +881,7 @@ abstract class FileBackend { /** * Get the last-modified timestamp of the file at a storage path. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data * @return string|bool TS_MW timestamp or false on failure @@ -843,8 +892,7 @@ abstract class FileBackend { * Get the contents of a file at a storage path in the backend. * This should be avoided for potentially large files. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data * @return string|bool Returns false on failure @@ -863,24 +911,42 @@ abstract class FileBackend { * * @see FileBackend::getFileContents() * - * @param array $params - * $params include: + * @param array $params Parameters include: * - srcs : list of source storage paths * - latest : use the latest available data * - parallelize : try to do operations in parallel when possible - * @return Array Map of (path name => string or false on failure) + * @return array Map of (path name => string or false on failure) * @since 1.20 */ abstract public function getFileContentsMulti( array $params ); /** - * Get the size (bytes) of a file at a storage path in the backend. + * Get metadata about a file at a storage path in the backend. + * If the file does not exist, then this returns false. + * Otherwise, the result is an associative array that includes: + * - headers : map of HTTP headers used for GET/HEAD requests (name => value) + * - metadata : map of file metadata (name => value) + * Metadata keys and headers names will be returned in all lower-case. + * Additional values may be included for internal use only. + * + * Use FileBackend::hasFeatures() to check how well this is supported. * * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data - * @return integer|bool Returns false on failure + * @return array|bool Returns false on failure + * @since 1.23 + */ + abstract public function getFileXAttributes( array $params ); + + /** + * Get the size (bytes) of a file at a storage path in the backend. + * + * @param array $params Parameters include: + * - src : source storage path + * - latest : use the latest available data + * @return int|bool Returns false on failure */ abstract public function getFileSize( array $params ); @@ -892,19 +958,17 @@ abstract class FileBackend { * - size : the file size (bytes) * Additional values may be included for internal use only. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data - * @return Array|bool|null Returns null on failure + * @return array|bool|null Returns null on failure */ abstract public function getFileStat( array $params ); /** * Get a SHA-1 hash of the file at a storage path in the backend. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data * @return string|bool Hash string or false on failure @@ -915,11 +979,10 @@ abstract class FileBackend { * Get the properties of the file at a storage path in the backend. * This gives the result of FSFile::getProps() on a local copy of the file. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data - * @return Array Returns FSFile::placeholderProps() on failure + * @return array Returns FSFile::placeholderProps() on failure */ abstract public function getFileProps( array $params ); @@ -930,8 +993,7 @@ abstract class FileBackend { * will be sent if streaming began, while none will be sent otherwise. * Implementations should flush the output buffer before sending data. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - headers : list of additional HTTP headers to send on success * - latest : use the latest available data @@ -952,8 +1014,7 @@ abstract class FileBackend { * In that later case, there are copies of the file that must stay in sync. * Additionally, further calls to this function may return the same file. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data * @return FSFile|null Returns null on failure @@ -972,12 +1033,11 @@ abstract class FileBackend { * * @see FileBackend::getLocalReference() * - * @param array $params - * $params include: + * @param array $params Parameters include: * - srcs : list of source storage paths * - latest : use the latest available data * - parallelize : try to do operations in parallel when possible - * @return Array Map of (path name => FSFile or null on failure) + * @return array Map of (path name => FSFile or null on failure) * @since 1.20 */ abstract public function getLocalReferenceMulti( array $params ); @@ -987,8 +1047,7 @@ abstract class FileBackend { * The temporary copy will have the same file extension as the source. * Temporary files may be purged when the file object falls out of scope. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - latest : use the latest available data * @return TempFSFile|null Returns null on failure @@ -1007,12 +1066,11 @@ abstract class FileBackend { * * @see FileBackend::getLocalCopy() * - * @param array $params - * $params include: + * @param array $params Parameters include: * - srcs : list of source storage paths * - latest : use the latest available data * - parallelize : try to do operations in parallel when possible - * @return Array Map of (path name => TempFSFile or null on failure) + * @return array Map of (path name => TempFSFile or null on failure) * @since 1.20 */ abstract public function getLocalCopyMulti( array $params ); @@ -1027,8 +1085,7 @@ abstract class FileBackend { * Otherwise, one would need to use getLocalReference(), which involves loading * the entire file on to local disk. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - src : source storage path * - ttl : lifetime (seconds) if pre-authenticated; default is 1 day * @return string|null @@ -1043,8 +1100,7 @@ abstract class FileBackend { * * Storage backends with eventual consistency might return stale data. * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * @return bool|null Returns null on failure * @since 1.20 @@ -1063,11 +1119,10 @@ abstract class FileBackend { * * Failures during iteration can result in FileBackendError exceptions (since 1.22). * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * - topOnly : only return direct child dirs of the directory - * @return Traversable|Array|null Returns null on failure + * @return Traversable|array|null Returns null on failure * @since 1.20 */ abstract public function getDirectoryList( array $params ); @@ -1080,10 +1135,9 @@ abstract class FileBackend { * * Failures during iteration can result in FileBackendError exceptions (since 1.22). * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory - * @return Traversable|Array|null Returns null on failure + * @return Traversable|array|null Returns null on failure * @since 1.20 */ final public function getTopDirectoryList( array $params ) { @@ -1102,12 +1156,11 @@ abstract class FileBackend { * * Failures during iteration can result in FileBackendError exceptions (since 1.22). * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * - topOnly : only return direct child files of the directory (since 1.20) * - adviseStat : set to true if stat requests will be made on the files (since 1.22) - * @return Traversable|Array|null Returns null on failure + * @return Traversable|array|null Returns null on failure */ abstract public function getFileList( array $params ); @@ -1119,11 +1172,10 @@ abstract class FileBackend { * * Failures during iteration can result in FileBackendError exceptions (since 1.22). * - * @param array $params - * $params include: + * @param array $params Parameters include: * - dir : storage directory * - adviseStat : set to true if stat requests will be made on the files (since 1.22) - * @return Traversable|Array|null Returns null on failure + * @return Traversable|array|null Returns null on failure * @since 1.20 */ final public function getTopFileList( array $params ) { @@ -1131,22 +1183,38 @@ abstract class FileBackend { } /** - * Preload persistent file stat and property cache into in-process cache. + * Preload persistent file stat cache and property cache into in-process cache. * This should be used when stat calls will be made on a known list of a many files. * + * @see FileBackend::getFileStat() + * * @param array $paths Storage paths - * @return void */ - public function preloadCache( array $paths ) {} + abstract public function preloadCache( array $paths ); /** * Invalidate any in-process file stat and property cache. * If $paths is given, then only the cache for those files will be cleared. * + * @see FileBackend::getFileStat() + * * @param array $paths Storage paths (optional) - * @return void */ - public function clearCache( array $paths = null ) {} + abstract public function clearCache( array $paths = null ); + + /** + * Preload file stat information (concurrently if possible) into in-process cache. + * This should be used when stat calls will be made on a known list of a many files. + * + * @see FileBackend::getFileStat() + * + * @param array $params Parameters include: + * - srcs : list of source storage paths + * - latest : use the latest available data + * @return bool All requests proceeded without I/O errors (since 1.24) + * @since 1.23 + */ + abstract public function preloadFileStat( array $params ); /** * Lock the files at the given storage paths in the backend. @@ -1155,23 +1223,26 @@ abstract class FileBackend { * Callers should consider using getScopedFileLocks() instead. * * @param array $paths Storage paths - * @param integer $type LockManager::LOCK_* constant + * @param int $type LockManager::LOCK_* constant + * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.24) * @return Status */ - final public function lockFiles( array $paths, $type ) { + final public function lockFiles( array $paths, $type, $timeout = 0 ) { $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); - return $this->lockManager->lock( $paths, $type ); + + return $this->lockManager->lock( $paths, $type, $timeout ); } /** * Unlock the files at the given storage paths in the backend. * * @param array $paths Storage paths - * @param integer $type LockManager::LOCK_* constant + * @param int $type LockManager::LOCK_* constant * @return Status */ final public function unlockFiles( array $paths, $type ) { $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); + return $this->lockManager->unlock( $paths, $type ); } @@ -1186,11 +1257,12 @@ abstract class FileBackend { * @see ScopedLock::factory() * * @param array $paths List of storage paths or map of lock types to path lists - * @param integer|string $type LockManager::LOCK_* constant or "mixed" + * @param int|string $type LockManager::LOCK_* constant or "mixed" * @param Status $status Status to update on lock/unlock + * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.24) * @return ScopedLock|null Returns null on failure */ - final public function getScopedFileLocks( array $paths, $type, Status $status ) { + final public function getScopedFileLocks( array $paths, $type, Status $status, $timeout = 0 ) { if ( $type === 'mixed' ) { foreach ( $paths as &$typePaths ) { $typePaths = array_map( 'FileBackend::normalizeStoragePath', $typePaths ); @@ -1198,7 +1270,8 @@ abstract class FileBackend { } else { $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); } - return ScopedLock::factory( $this->lockManager, $paths, $type, $status ); + + return ScopedLock::factory( $this->lockManager, $paths, $type, $status, $timeout ); } /** @@ -1214,7 +1287,7 @@ abstract class FileBackend { * * @param array $ops List of file operations to FileBackend::doOperations() * @param Status $status Status to update on lock/unlock - * @return Array List of ScopedFileLocks or null values + * @return array List of ScopedFileLocks or null values * @since 1.20 */ abstract public function getScopedLocksForOps( array $ops, Status $status ); @@ -1267,7 +1340,7 @@ abstract class FileBackend { * This does not do any path normalization or traversal checks. * * @param string $storagePath - * @return Array (backend, container, rel object) or (null, null, null) + * @return array (backend, container, rel object) or (null, null, null) */ final public static function splitStoragePath( $storagePath ) { if ( self::isStoragePath( $storagePath ) ) { @@ -1281,6 +1354,7 @@ abstract class FileBackend { } } } + return array( null, null, null ); } @@ -1301,6 +1375,7 @@ abstract class FileBackend { : "mwstore://{$backend}/{$container}"; } } + return null; } @@ -1315,6 +1390,7 @@ abstract class FileBackend { final public static function parentStoragePath( $storagePath ) { $storagePath = dirname( $storagePath ); list( , , $rel ) = self::splitStoragePath( $storagePath ); + return ( $rel === null ) ? null : $storagePath; } @@ -1322,11 +1398,20 @@ abstract class FileBackend { * Get the final extension from a storage or FS path * * @param string $path + * @param string $case One of (rawcase, uppercase, lowercase) (since 1.24) * @return string */ - final public static function extensionFromPath( $path ) { + final public static function extensionFromPath( $path, $case = 'lowercase' ) { $i = strrpos( $path, '.' ); - return strtolower( $i ? substr( $path, $i + 1 ) : '' ); + $ext = $i ? substr( $path, $i + 1 ) : ''; + + if ( $case === 'lowercase' ) { + $ext = strtolower( $ext ); + } elseif ( $case === 'uppercase' ) { + $ext = strtoupper( $ext ); + } + + return $ext; } /** @@ -1345,7 +1430,7 @@ abstract class FileBackend { * * @param string $type One of (attachment, inline) * @param string $filename Suggested file name (should not contain slashes) - * @throws MWException + * @throws FileBackendError * @return string * @since 1.20 */ @@ -1354,7 +1439,7 @@ abstract class FileBackend { $type = strtolower( $type ); if ( !in_array( $type, array( 'inline', 'attachment' ) ) ) { - throw new MWException( "Invalid Content-Disposition type '$type'." ); + throw new FileBackendError( "Invalid Content-Disposition type '$type'." ); } $parts[] = $type; @@ -1395,12 +1480,25 @@ abstract class FileBackend { return null; } } + return $path; } } /** + * Generic file backend exception for checked and unexpected (e.g. config) exceptions + * + * @ingroup FileBackend + * @since 1.23 + */ +class FileBackendException extends MWException { +} + +/** + * File backend exception for checked exceptions (e.g. I/O errors) + * * @ingroup FileBackend * @since 1.22 */ -class FileBackendError extends MWException {} +class FileBackendError extends FileBackendException { +} diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php index be8a2076..1b88db7e 100644 --- a/includes/filebackend/FileBackendGroup.php +++ b/includes/filebackend/FileBackendGroup.php @@ -29,15 +29,14 @@ * @since 1.19 */ class FileBackendGroup { - /** - * @var FileBackendGroup - */ + /** @var FileBackendGroup */ protected static $instance = null; - /** @var Array (name => ('class' => string, 'config' => array, 'instance' => object)) */ + /** @var array (name => ('class' => string, 'config' => array, 'instance' => object)) */ protected $backends = array(); - protected function __construct() {} + protected function __construct() { + } /** * @return FileBackendGroup @@ -47,13 +46,12 @@ class FileBackendGroup { self::$instance = new self(); self::$instance->initFromGlobals(); } + return self::$instance; } /** * Destroy the singleton instance - * - * @return void */ public static function destroySingleton() { self::$instance = null; @@ -61,8 +59,6 @@ class FileBackendGroup { /** * Register file backends from the global variables - * - * @return void */ protected function initFromGlobals() { global $wgLocalFileRepo, $wgForeignFileRepos, $wgFileBackends; @@ -116,20 +112,19 @@ class FileBackendGroup { /** * Register an array of file backend configurations * - * @param Array $configs - * @return void - * @throws MWException + * @param array $configs + * @throws FileBackendException */ protected function register( array $configs ) { foreach ( $configs as $config ) { if ( !isset( $config['name'] ) ) { - throw new MWException( "Cannot register a backend with no name." ); + throw new FileBackendException( "Cannot register a backend with no name." ); } $name = $config['name']; if ( isset( $this->backends[$name] ) ) { - throw new MWException( "Backend with name `{$name}` already registered." ); + throw new FileBackendException( "Backend with name `{$name}` already registered." ); } elseif ( !isset( $config['class'] ) ) { - throw new MWException( "Cannot register backend `{$name}` with no class." ); + throw new FileBackendException( "Backend with name `{$name}` has no class." ); } $class = $config['class']; @@ -147,18 +142,27 @@ class FileBackendGroup { * * @param string $name * @return FileBackend - * @throws MWException + * @throws FileBackendException */ public function get( $name ) { if ( !isset( $this->backends[$name] ) ) { - throw new MWException( "No backend defined with the name `$name`." ); + throw new FileBackendException( "No backend defined with the name `$name`." ); } // Lazy-load the actual backend instance if ( !isset( $this->backends[$name]['instance'] ) ) { $class = $this->backends[$name]['class']; $config = $this->backends[$name]['config']; + $config['wikiId'] = isset( $config['wikiId'] ) + ? $config['wikiId'] + : wfWikiID(); // e.g. "my_wiki-en_" + $config['lockManager'] = + LockManagerGroup::singleton( $config['wikiId'] )->get( $config['lockManager'] ); + $config['fileJournal'] = isset( $config['fileJournal'] ) + ? FileJournal::factory( $config['fileJournal'], $name ) + : FileJournal::factory( array( 'class' => 'NullFileJournal' ), $name ); $this->backends[$name]['instance'] = new $class( $config ); } + return $this->backends[$name]['instance']; } @@ -166,14 +170,15 @@ class FileBackendGroup { * Get the config array for a backend object with a given name * * @param string $name - * @return Array - * @throws MWException + * @return array + * @throws FileBackendException */ public function config( $name ) { if ( !isset( $this->backends[$name] ) ) { - throw new MWException( "No backend defined with the name `$name`." ); + throw new FileBackendException( "No backend defined with the name `$name`." ); } $class = $this->backends[$name]['class']; + return array( 'class' => $class ) + $this->backends[$name]['config']; } @@ -188,6 +193,7 @@ class FileBackendGroup { if ( $backend !== null && isset( $this->backends[$backend] ) ) { return $this->get( $backend ); } + return null; } } diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php index 97584a71..bfffcc0f 100644 --- a/includes/filebackend/FileBackendMultiWrite.php +++ b/includes/filebackend/FileBackendMultiWrite.php @@ -40,15 +40,25 @@ * @since 1.19 */ class FileBackendMultiWrite extends FileBackend { - /** @var Array Prioritized list of FileBackendStore objects */ - protected $backends = array(); // array of (backend index => backends) - protected $masterIndex = -1; // integer; index of master backend - protected $syncChecks = 0; // integer; bitfield - protected $autoResync = false; // boolean + /** @var array Prioritized list of FileBackendStore objects. + * array of (backend index => backends) + */ + protected $backends = array(); + + /** @var int Index of master backend */ + protected $masterIndex = -1; + + /** @var int Bitfield */ + protected $syncChecks = 0; - /** @var Array */ + /** @var string|bool */ + protected $autoResync = false; + + /** @var array */ protected $noPushDirConts = array(); - protected $noPushQuickOps = false; // boolean + + /** @var bool */ + protected $noPushQuickOps = false; /* Possible internal backend consistency checks */ const CHECK_SIZE = 1; @@ -81,8 +91,8 @@ class FileBackendMultiWrite extends FileBackend { * - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend. * - noPushDirConts : (hack) Only apply directory functions to the master backend. * - * @param Array $config - * @throws MWException + * @param array $config + * @throws FileBackendError */ public function __construct( array $config ) { parent::__construct( $config ); @@ -109,30 +119,30 @@ class FileBackendMultiWrite extends FileBackend { } $name = $config['name']; if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates - throw new MWException( "Two or more backends defined with the name $name." ); + throw new FileBackendError( "Two or more backends defined with the name $name." ); } $namesUsed[$name] = 1; // Alter certain sub-backend settings for sanity unset( $config['readOnly'] ); // use proxy backend setting unset( $config['fileJournal'] ); // use proxy backend journal + unset( $config['lockManager'] ); // lock under proxy backend $config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID - $config['lockManager'] = 'nullLockManager'; // lock under proxy backend if ( !empty( $config['isMultiMaster'] ) ) { if ( $this->masterIndex >= 0 ) { - throw new MWException( 'More than one master backend defined.' ); + throw new FileBackendError( 'More than one master backend defined.' ); } $this->masterIndex = $index; // this is the "master" $config['fileJournal'] = $this->fileJournal; // log under proxy backend } // Create sub-backend object if ( !isset( $config['class'] ) ) { - throw new MWException( 'No class given for a backend config.' ); + throw new FileBackendError( 'No class given for a backend config.' ); } $class = $config['class']; $this->backends[$index] = new $class( $config ); } if ( $this->masterIndex < 0 ) { // need backends and must have a master - throw new MWException( 'No master backend defined.' ); + throw new FileBackendError( 'No master backend defined.' ); } } @@ -167,6 +177,7 @@ class FileBackendMultiWrite extends FileBackend { // Try to resync the clone backends to the master on the spot... if ( !$this->autoResync || !$this->resyncFiles( $relevantPaths )->isOK() ) { $status->merge( $syncStatus ); + return $status; // abort } } @@ -321,8 +332,8 @@ class FileBackendMultiWrite extends FileBackend { // already synced; nothing to do } elseif ( $mSha1 !== false ) { // file is in master if ( $this->autoResync === 'conservative' - && $cStat && $cStat['mtime'] > $mStat['mtime'] ) - { + && $cStat && $cStat['mtime'] > $mStat['mtime'] + ) { $status->fatal( 'backend-fail-synced', $path ); continue; // don't rollback data } @@ -348,7 +359,7 @@ class FileBackendMultiWrite extends FileBackend { * Get a list of file storage paths to read or write for a list of operations * * @param array $ops Same format as doOperations() - * @return Array List of storage paths to files (does not include directories) + * @return array List of storage paths to files (does not include directories) */ protected function fileStoragePathsForOps( array $ops ) { $paths = array(); @@ -357,8 +368,8 @@ class FileBackendMultiWrite extends FileBackend { // For things like copy/move/delete with "ignoreMissingSource" and there // is no source file, nothing should happen and there should be no errors. if ( empty( $op['ignoreMissingSource'] ) - || $this->fileExists( array( 'src' => $op['src'] ) ) ) - { + || $this->fileExists( array( 'src' => $op['src'] ) ) + ) { $paths[] = $op['src']; } } @@ -369,6 +380,7 @@ class FileBackendMultiWrite extends FileBackend { $paths[] = $op['dst']; } } + return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) ); } @@ -378,7 +390,7 @@ class FileBackendMultiWrite extends FileBackend { * * @param array $ops List of file operation arrays * @param FileBackendStore $backend - * @return Array + * @return array */ protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) { $newOps = array(); // operations @@ -391,6 +403,7 @@ class FileBackendMultiWrite extends FileBackend { } $newOps[] = $newOp; } + return $newOps; } @@ -399,10 +412,11 @@ class FileBackendMultiWrite extends FileBackend { * * @param array $ops File operation array * @param FileBackendStore $backend - * @return Array + * @return array */ protected function substOpPaths( array $ops, FileBackendStore $backend ) { $newOps = $this->substOpBatchPaths( array( $ops ), $backend ); + return $newOps[0]; } @@ -411,7 +425,7 @@ class FileBackendMultiWrite extends FileBackend { * * @param array|string $paths List of paths or single string path * @param FileBackendStore $backend - * @return Array|string + * @return array|string */ protected function substPaths( $paths, FileBackendStore $backend ) { return preg_replace( @@ -425,7 +439,7 @@ class FileBackendMultiWrite extends FileBackend { * Substitute the backend of internal storage paths with the proxy backend's name * * @param array|string $paths List of paths or single string path - * @return Array|string + * @return array|string */ protected function unsubstPaths( $paths ) { return preg_replace( @@ -456,6 +470,7 @@ class FileBackendMultiWrite extends FileBackend { $status->success = $masterStatus->success; $status->successCount = $masterStatus->successCount; $status->failCount = $masterStatus->failCount; + return $status; } @@ -465,6 +480,7 @@ class FileBackendMultiWrite extends FileBackend { */ protected function replicateContainerDirChanges( $path ) { list( , $shortCont, ) = self::splitStoragePath( $path ); + return !in_array( $shortCont, $this->noPushDirConts ); } @@ -477,6 +493,7 @@ class FileBackendMultiWrite extends FileBackend { $status->merge( $backend->doPrepare( $realParams ) ); } } + return $status; } @@ -489,6 +506,7 @@ class FileBackendMultiWrite extends FileBackend { $status->merge( $backend->doSecure( $realParams ) ); } } + return $status; } @@ -501,6 +519,7 @@ class FileBackendMultiWrite extends FileBackend { $status->merge( $backend->doPublish( $realParams ) ); } } + return $status; } @@ -513,35 +532,47 @@ class FileBackendMultiWrite extends FileBackend { $status->merge( $backend->doClean( $realParams ) ); } } + return $status; } public function concatenate( array $params ) { // We are writing to an FS file, so we don't need to do this per-backend $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->concatenate( $realParams ); } public function fileExists( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->fileExists( $realParams ); } public function getFileTimestamp( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams ); } public function getFileSize( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileSize( $realParams ); } public function getFileStat( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileStat( $realParams ); } + public function getFileXAttributes( array $params ) { + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + + return $this->backends[$this->masterIndex]->getFileXAttributes( $realParams ); + } + public function getFileContentsMulti( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams ); @@ -550,21 +581,25 @@ class FileBackendMultiWrite extends FileBackend { foreach ( $contentsM as $path => $data ) { $contents[$this->unsubstPaths( $path )] = $data; } + return $contents; } public function getFileSha1Base36( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams ); } public function getFileProps( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileProps( $realParams ); } public function streamFile( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->streamFile( $realParams ); } @@ -576,6 +611,7 @@ class FileBackendMultiWrite extends FileBackend { foreach ( $fsFilesM as $path => $fsFile ) { $fsFiles[$this->unsubstPaths( $path )] = $fsFile; } + return $fsFiles; } @@ -587,29 +623,38 @@ class FileBackendMultiWrite extends FileBackend { foreach ( $tempFilesM as $path => $tempFile ) { $tempFiles[$this->unsubstPaths( $path )] = $tempFile; } + return $tempFiles; } public function getFileHttpUrl( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams ); } public function directoryExists( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->directoryExists( $realParams ); } public function getDirectoryList( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getDirectoryList( $realParams ); } public function getFileList( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileList( $realParams ); } + public function getFeatures() { + return $this->backends[$this->masterIndex]->getFeatures(); + } + public function clearCache( array $paths = null ) { foreach ( $this->backends as $backend ) { $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null; @@ -617,6 +662,16 @@ class FileBackendMultiWrite extends FileBackend { } } + public function preloadCache( array $paths ) { + $realPaths = $this->substPaths( $paths, $this->backends[$this->masterIndex] ); + $this->backends[$this->masterIndex]->preloadCache( $realPaths ); + } + + public function preloadFileStat( array $params ) { + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->preloadFileStat( $realParams ); + } + public function getScopedLocksForOps( array $ops, Status $status ) { $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps ); @@ -627,6 +682,7 @@ class FileBackendMultiWrite extends FileBackend { LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ), LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] ) ); + // Actually acquire the locks return array( $this->getScopedFileLocks( $pbPaths, 'mixed', $status ) ); } diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php index 0921e99f..495ac3c0 100644 --- a/includes/filebackend/FileBackendStore.php +++ b/includes/filebackend/FileBackendStore.php @@ -43,16 +43,16 @@ abstract class FileBackendStore extends FileBackend { /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */ protected $expensiveCache; - /** @var Array Map of container names to sharding config */ + /** @var array Map of container names to sharding config */ protected $shardViaHashLevels = array(); - /** @var callback Method to get the MIME type of files */ + /** @var callable Method to get the MIME type of files */ protected $mimeCallback; protected $maxFileSize = 4294967296; // integer bytes (4GiB) const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries - const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache" + const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache" const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache" /** @@ -68,8 +68,8 @@ abstract class FileBackendStore extends FileBackend { parent::__construct( $config ); $this->mimeCallback = isset( $config['mimeCallback'] ) ? $config['mimeCallback'] - : function( $storagePath, $content, $fsPath ) { - // @TODO: handle the case of extension-less files using the contents + : function ( $storagePath, $content, $fsPath ) { + // @todo handle the case of extension-less files using the contents return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown'; }; $this->memCache = new EmptyBagOStuff(); // disabled by default @@ -82,7 +82,7 @@ abstract class FileBackendStore extends FileBackend { * medium restrictions and basic performance constraints. * Do not call this function from places outside FileBackend and FileOp. * - * @return integer Bytes + * @return int Bytes */ final public function maxFileSizeInternal() { return $this->maxFileSize; @@ -129,11 +129,13 @@ abstract class FileBackendStore extends FileBackend { $this->deleteFileCache( $params['dst'] ); // persistent cache } } + return $status; } /** * @see FileBackendStore::createInternal() + * @param array $params * @return Status */ abstract protected function doCreateInternal( array $params ); @@ -168,11 +170,13 @@ abstract class FileBackendStore extends FileBackend { $this->deleteFileCache( $params['dst'] ); // persistent cache } } + return $status; } /** * @see FileBackendStore::storeInternal() + * @param array $params * @return Status */ abstract protected function doStoreInternal( array $params ); @@ -203,11 +207,13 @@ abstract class FileBackendStore extends FileBackend { if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } + return $status; } /** * @see FileBackendStore::copyInternal() + * @param array $params * @return Status */ abstract protected function doCopyInternal( array $params ); @@ -236,6 +242,7 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::deleteInternal() + * @param array $params * @return Status */ abstract protected function doDeleteInternal( array $params ); @@ -267,11 +274,13 @@ abstract class FileBackendStore extends FileBackend { if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } + return $status; } /** * @see FileBackendStore::moveInternal() + * @param array $params * @return Status */ protected function doMoveInternal( array $params ) { @@ -285,6 +294,7 @@ abstract class FileBackendStore extends FileBackend { $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) ); $status->setResult( true, $status->value ); // ignore delete() errors } + return $status; } @@ -311,11 +321,13 @@ abstract class FileBackendStore extends FileBackend { } else { $status = Status::newGood(); // nothing to do } + return $status; } /** * @see FileBackendStore::describeInternal() + * @param array $params * @return Status */ protected function doDescribeInternal( array $params ) { @@ -345,8 +357,8 @@ abstract class FileBackendStore extends FileBackend { $status->merge( $this->doConcatenate( $params ) ); $sec = microtime( true ) - $start_time; if ( !$status->isOK() ) { - wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " . - count( $params['srcs'] ) . " file(s) [$sec sec]" ); + wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name}" . + " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" ); } } @@ -355,6 +367,7 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::concatenate() + * @param array $params * @return Status */ protected function doConcatenate( array $params ) { @@ -368,6 +381,7 @@ abstract class FileBackendStore extends FileBackend { wfRestoreWarnings(); if ( !$ok ) { // not present or not empty $status->fatal( 'backend-fail-opentemp', $tmpPath ); + return $status; } @@ -378,6 +392,7 @@ abstract class FileBackendStore extends FileBackend { $fsFile = $this->getLocalReference( array( 'src' => $path ) ); if ( !$fsFile ) { // retry failed? $status->fatal( 'backend-fail-read', $path ); + return $status; } } @@ -388,6 +403,7 @@ abstract class FileBackendStore extends FileBackend { $tmpHandle = fopen( $tmpPath, 'ab' ); if ( $tmpHandle === false ) { $status->fatal( 'backend-fail-opentemp', $tmpPath ); + return $status; } @@ -398,6 +414,7 @@ abstract class FileBackendStore extends FileBackend { if ( $sourceHandle === false ) { fclose( $tmpHandle ); $status->fatal( 'backend-fail-read', $virtualSource ); + return $status; } // Append chunk to file (pass chunk size to avoid magic quotes) @@ -405,12 +422,14 @@ abstract class FileBackendStore extends FileBackend { fclose( $sourceHandle ); fclose( $tmpHandle ); $status->fatal( 'backend-fail-writetemp', $tmpPath ); + return $status; } fclose( $sourceHandle ); } if ( !fclose( $tmpHandle ) ) { $status->fatal( 'backend-fail-closetemp', $tmpPath ); + return $status; } @@ -426,6 +445,7 @@ abstract class FileBackendStore extends FileBackend { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); + return $status; // invalid storage path } @@ -444,6 +464,9 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::doPrepare() + * @param string $container + * @param string $dir + * @param array $params * @return Status */ protected function doPrepareInternal( $container, $dir, array $params ) { @@ -457,6 +480,7 @@ abstract class FileBackendStore extends FileBackend { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); + return $status; // invalid storage path } @@ -475,6 +499,9 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::doSecure() + * @param string $container + * @param string $dir + * @param array $params * @return Status */ protected function doSecureInternal( $container, $dir, array $params ) { @@ -488,6 +515,7 @@ abstract class FileBackendStore extends FileBackend { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); + return $status; // invalid storage path } @@ -506,6 +534,9 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::doPublish() + * @param string $container + * @param string $dir + * @param array $params * @return Status */ protected function doPublishInternal( $container, $dir, array $params ) { @@ -531,6 +562,7 @@ abstract class FileBackendStore extends FileBackend { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); + return $status; // invalid storage path } @@ -558,6 +590,9 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::doClean() + * @param string $container + * @param string $dir + * @param array $params * @return Status */ protected function doCleanInternal( $container, $dir, array $params ) { @@ -567,18 +602,21 @@ abstract class FileBackendStore extends FileBackend { final public function fileExists( array $params ) { $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); + return ( $stat === null ) ? null : (bool)$stat; // null => failure } final public function getFileTimestamp( array $params ) { $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); + return $stat ? $stat['mtime'] : false; } final public function getFileSize( array $params ) { $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); + return $stat ? $stat['size'] : false; } @@ -606,27 +644,32 @@ abstract class FileBackendStore extends FileBackend { } } } - wfProfileIn( __METHOD__ . '-miss' ); wfProfileIn( __METHOD__ . '-miss-' . $this->name ); $stat = $this->doGetFileStat( $params ); wfProfileOut( __METHOD__ . '-miss-' . $this->name ); - wfProfileOut( __METHOD__ . '-miss' ); if ( is_array( $stat ) ) { // file exists - $stat['latest'] = $latest; + // Strongly consistent backends can automatically set "latest" + $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest; $this->cheapCache->set( $path, 'stat', $stat ); $this->setFileCache( $path, $stat ); // update persistent cache if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata $this->cheapCache->set( $path, 'sha1', array( 'hash' => $stat['sha1'], 'latest' => $latest ) ); } + if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata + $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); + $this->cheapCache->set( $path, 'xattr', + array( 'map' => $stat['xattr'], 'latest' => $latest ) ); + } } elseif ( $stat === false ) { // file does not exist $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); - $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too - array( 'hash' => false, 'latest' => $latest ) ); + $this->cheapCache->set( $path, 'xattr', array( 'map' => false, 'latest' => $latest ) ); + $this->cheapCache->set( $path, 'sha1', array( 'hash' => false, 'latest' => $latest ) ); wfDebug( __METHOD__ . ": File $path does not exist.\n" ); } else { // an error occurred wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); } + return $stat; } @@ -646,7 +689,8 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::getFileContentsMulti() - * @return Array + * @param array $params + * @return array */ protected function doGetFileContentsMulti( array $params ) { $contents = array(); @@ -655,9 +699,44 @@ abstract class FileBackendStore extends FileBackend { $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false; wfRestoreWarnings(); } + return $contents; } + final public function getFileXAttributes( array $params ) { + $path = self::normalizeStoragePath( $params['src'] ); + if ( $path === null ) { + return false; // invalid storage path + } + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); + $latest = !empty( $params['latest'] ); // use latest data? + if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) { + $stat = $this->cheapCache->get( $path, 'xattr' ); + // If we want the latest data, check that this cached + // value was in fact fetched with the latest available data. + if ( !$latest || $stat['latest'] ) { + return $stat['map']; + } + } + wfProfileIn( __METHOD__ . '-miss' ); + wfProfileIn( __METHOD__ . '-miss-' . $this->name ); + $fields = $this->doGetFileXAttributes( $params ); + $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false; + wfProfileOut( __METHOD__ . '-miss-' . $this->name ); + wfProfileOut( __METHOD__ . '-miss' ); + $this->cheapCache->set( $path, 'xattr', array( 'map' => $fields, 'latest' => $latest ) ); + + return $fields; + } + + /** + * @see FileBackendStore::getFileXAttributes() + * @return bool|string + */ + protected function doGetFileXAttributes( array $params ) { + return array( 'headers' => array(), 'metadata' => array() ); // not supported + } + final public function getFileSha1Base36( array $params ) { $path = self::normalizeStoragePath( $params['src'] ); if ( $path === null ) { @@ -673,17 +752,17 @@ abstract class FileBackendStore extends FileBackend { return $stat['hash']; } } - wfProfileIn( __METHOD__ . '-miss' ); wfProfileIn( __METHOD__ . '-miss-' . $this->name ); $hash = $this->doGetFileSha1Base36( $params ); wfProfileOut( __METHOD__ . '-miss-' . $this->name ); - wfProfileOut( __METHOD__ . '-miss' ); $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) ); + return $hash; } /** * @see FileBackendStore::getFileSha1Base36() + * @param array $params * @return bool|string */ protected function doGetFileSha1Base36( array $params ) { @@ -699,6 +778,7 @@ abstract class FileBackendStore extends FileBackend { $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $fsFile = $this->getLocalReference( $params ); $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); + return $props; } @@ -738,7 +818,8 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::getLocalReferenceMulti() - * @return Array + * @param array $params + * @return array */ protected function doGetLocalReferenceMulti( array $params ) { return $this->doGetLocalCopyMulti( $params ); @@ -755,12 +836,14 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::getLocalCopyMulti() - * @return Array + * @param array $params + * @return array */ abstract protected function doGetLocalCopyMulti( array $params ); /** * @see FileBackend::getFileHttpUrl() + * @param array $params * @return string|null */ public function getFileHttpUrl( array $params ) { @@ -782,11 +865,9 @@ abstract class FileBackendStore extends FileBackend { if ( $res == StreamFile::NOT_MODIFIED ) { // do nothing; client cache is up to date } elseif ( $res == StreamFile::READY_STREAM ) { - wfProfileIn( __METHOD__ . '-send' ); wfProfileIn( __METHOD__ . '-send-' . $this->name ); $status = $this->doStreamFile( $params ); wfProfileOut( __METHOD__ . '-send-' . $this->name ); - wfProfileOut( __METHOD__ . '-send' ); if ( !$status->isOK() ) { // Per bug 41113, nasty things can happen if bad cache entries get // stuck in cache. It's also possible that this error can come up @@ -804,6 +885,7 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::streamFile() + * @param array $params * @return Status */ protected function doStreamFile( array $params ) { @@ -839,6 +921,7 @@ abstract class FileBackendStore extends FileBackend { $res = null; // if we don't find anything, it is indeterminate } } + return $res; } } @@ -865,6 +948,7 @@ abstract class FileBackendStore extends FileBackend { wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); // File listing spans multiple containers/shards list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); + return new FileBackendStoreShardDirIterator( $this, $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); } @@ -878,7 +962,7 @@ abstract class FileBackendStore extends FileBackend { * @param string $container Resolved container name * @param string $dir Resolved path relative to container * @param array $params - * @return Traversable|Array|null Returns null on failure + * @return Traversable|array|null Returns null on failure */ abstract public function getDirectoryListInternal( $container, $dir, array $params ); @@ -894,6 +978,7 @@ abstract class FileBackendStore extends FileBackend { wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); // File listing spans multiple containers/shards list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); + return new FileBackendStoreShardFileIterator( $this, $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); } @@ -907,7 +992,7 @@ abstract class FileBackendStore extends FileBackend { * @param string $container Resolved container name * @param string $dir Resolved path relative to container * @param array $params - * @return Traversable|Array|null Returns null on failure + * @return Traversable|array|null Returns null on failure */ abstract public function getFileListInternal( $container, $dir, array $params ); @@ -919,8 +1004,8 @@ abstract class FileBackendStore extends FileBackend { * An exception is thrown if an unsupported operation is requested. * * @param array $ops Same format as doOperations() - * @return Array List of FileOp objects - * @throws MWException + * @return array List of FileOp objects + * @throws FileBackendError */ final public function getOperationsInternal( array $ops ) { $supportedOps = array( @@ -944,7 +1029,7 @@ abstract class FileBackendStore extends FileBackend { // Append the FileOp class $performOps[] = new $class( $this, $params ); } else { - throw new MWException( "Operation '$opName' is not supported." ); + throw new FileBackendError( "Operation '$opName' is not supported." ); } } @@ -959,7 +1044,7 @@ abstract class FileBackendStore extends FileBackend { * normalized. * * @param array $performOps List of FileOp objects - * @return Array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list) + * @return array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list) */ final public function getPathsToLockForOpsInternal( array $performOps ) { // Build up a list of files to lock... @@ -981,6 +1066,7 @@ abstract class FileBackendStore extends FileBackend { public function getScopedLocksForOps( array $ops, Status $status ) { $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); + return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) ); } @@ -1010,18 +1096,43 @@ abstract class FileBackendStore extends FileBackend { $this->clearCache(); } - // Load from the persistent file and container caches - $this->primeFileCache( $performOps ); - $this->primeContainerCache( $performOps ); + // Build the list of paths involved + $paths = array(); + foreach ( $performOps as $op ) { + $paths = array_merge( $paths, $op->storagePathsRead() ); + $paths = array_merge( $paths, $op->storagePathsChanged() ); + } - // Actually attempt the operation batch... - $opts = $this->setConcurrencyFlags( $opts ); - $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal ); + // Enlarge the cache to fit the stat entries of these files + $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) ); + + // Load from the persistent container caches + $this->primeContainerCache( $paths ); + // Get the latest stat info for all the files (having locked them) + $ok = $this->preloadFileStat( array( 'srcs' => $paths, 'latest' => true ) ); + + if ( $ok ) { + // Actually attempt the operation batch... + $opts = $this->setConcurrencyFlags( $opts ); + $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal ); + } else { + // If we could not even stat some files, then bail out... + $subStatus = Status::newFatal( 'backend-fail-internal', $this->name ); + foreach ( $ops as $i => $op ) { // mark each op as failed + $subStatus->success[$i] = false; + ++$subStatus->failCount; + } + wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name} " . + " stat failure; aborted operations: " . FormatJson::encode( $ops ) ); + } // Merge errors into status fields $status->merge( $subStatus ); $status->success = $subStatus->success; // not done in merge() + // Shrink the stat cache back to normal size + $this->cheapCache->resize( self::CACHE_CHEAP_SIZE ); + return $status; } @@ -1035,7 +1146,8 @@ abstract class FileBackendStore extends FileBackend { // Clear any file cache entries $this->clearCache(); - $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' ); + $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ); + // Parallel ops may be disabled in config due to dependencies (e.g. needing popen()) $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 ); $maxConcurrency = $this->concurrency; // throttle @@ -1045,7 +1157,7 @@ abstract class FileBackendStore extends FileBackend { // Perform the sync-only ops and build up op handles for the async ops... foreach ( $ops as $index => $params ) { if ( !in_array( $params['op'], $supportedOps ) ) { - throw new MWException( "Operation '{$params['op']}' is not supported." ); + throw new FileBackendError( "Operation '{$params['op']}' is not supported." ); } $method = $params['op'] . 'Internal'; // e.g. "storeInternal" $subStatus = $this->$method( array( 'async' => $async ) + $params ); @@ -1086,36 +1198,40 @@ abstract class FileBackendStore extends FileBackend { * The resulting Status object fields will correspond * to the order in which the handles where given. * - * @param array $handles List of FileBackendStoreOpHandle objects - * @return Array Map of Status objects - * @throws MWException + * @param array $fileOpHandles + * @throws FileBackendError + * @internal param array $handles List of FileBackendStoreOpHandle objects + * @return array Map of Status objects */ final public function executeOpHandlesInternal( array $fileOpHandles ) { $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); + foreach ( $fileOpHandles as $fileOpHandle ) { if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { - throw new MWException( "Given a non-FileBackendStoreOpHandle object." ); + throw new FileBackendError( "Given a non-FileBackendStoreOpHandle object." ); } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) { - throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." ); + throw new FileBackendError( "Given a FileBackendStoreOpHandle for the wrong backend." ); } } $res = $this->doExecuteOpHandlesInternal( $fileOpHandles ); foreach ( $fileOpHandles as $fileOpHandle ) { $fileOpHandle->closeResources(); } + return $res; } /** * @see FileBackendStore::executeOpHandlesInternal() * @param array $fileOpHandles - * @throws MWException - * @return Array List of corresponding Status objects + * @throws FileBackendError + * @return array List of corresponding Status objects */ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { - foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty - throw new MWException( "This backend supports no asynchronous operations." ); + if ( count( $fileOpHandles ) ) { + throw new FileBackendError( "This backend supports no asynchronous operations." ); } + return array(); } @@ -1126,7 +1242,7 @@ abstract class FileBackendStore extends FileBackend { * specific errors, especially in the middle of batch file operations. * * @param array $op Same format as doOperation() - * @return Array + * @return array */ protected function stripInvalidHeadersFromOp( array $op ) { static $longs = array( 'Content-Disposition' ); @@ -1141,6 +1257,7 @@ abstract class FileBackendStore extends FileBackend { } } } + return $op; } @@ -1178,9 +1295,71 @@ abstract class FileBackendStore extends FileBackend { * @see FileBackend::clearCache() * * @param array $paths Storage paths (optional) - * @return void */ - protected function doClearCache( array $paths = null ) {} + protected function doClearCache( array $paths = null ) { + } + + final public function preloadFileStat( array $params ) { + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); + $success = true; // no network errors + + $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1; + $stats = $this->doGetFileStatMulti( $params ); + if ( $stats === null ) { + return true; // not supported + } + + $latest = !empty( $params['latest'] ); // use latest data? + foreach ( $stats as $path => $stat ) { + $path = FileBackend::normalizeStoragePath( $path ); + if ( $path === null ) { + continue; // this shouldn't happen + } + if ( is_array( $stat ) ) { // file exists + // Strongly consistent backends can automatically set "latest" + $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest; + $this->cheapCache->set( $path, 'stat', $stat ); + $this->setFileCache( $path, $stat ); // update persistent cache + if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata + $this->cheapCache->set( $path, 'sha1', + array( 'hash' => $stat['sha1'], 'latest' => $latest ) ); + } + if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata + $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); + $this->cheapCache->set( $path, 'xattr', + array( 'map' => $stat['xattr'], 'latest' => $latest ) ); + } + } elseif ( $stat === false ) { // file does not exist + $this->cheapCache->set( $path, 'stat', + $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); + $this->cheapCache->set( $path, 'xattr', + array( 'map' => false, 'latest' => $latest ) ); + $this->cheapCache->set( $path, 'sha1', + array( 'hash' => false, 'latest' => $latest ) ); + wfDebug( __METHOD__ . ": File $path does not exist.\n" ); + } else { // an error occurred + $success = false; + wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); + } + } + + return $success; + } + + /** + * Get file stat information (concurrently if possible) for several files + * + * @see FileBackend::getFileStat() + * + * @param array $params Parameters include: + * - srcs : list of source storage paths + * - latest : use the latest available data + * @return array|null Map of storage paths to array|bool|null (returns null if not supported) + * @since 1.23 + */ + protected function doGetFileStatMulti( array $params ) { + return null; // not supported + } /** * Is this a key/value store where directories are just virtual? @@ -1218,7 +1397,7 @@ abstract class FileBackendStore extends FileBackend { * be scanned by looking in all the container shards. * * @param string $storagePath - * @return Array (container, path, container suffix) or (null, null, null) if invalid + * @return array (container, path, container suffix) or (null, null, null) if invalid */ final protected function resolveStoragePath( $storagePath ) { list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath ); @@ -1242,6 +1421,7 @@ abstract class FileBackendStore extends FileBackend { } } } + return array( null, null, null ); } @@ -1258,13 +1438,14 @@ abstract class FileBackendStore extends FileBackend { * @see FileBackendStore::resolveStoragePath() * * @param string $storagePath - * @return Array (container, path) or (null, null) if invalid + * @return array (container, path) or (null, null) if invalid */ final protected function resolveStoragePathReal( $storagePath ) { list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) { return array( $container, $relPath ); } + return array( null, null ); } @@ -1299,8 +1480,10 @@ abstract class FileBackendStore extends FileBackend { if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) { return '.' . implode( '', array_slice( $m, 1 ) ); } + return null; // failed to match } + return ''; // no sharding } @@ -1314,6 +1497,7 @@ abstract class FileBackendStore extends FileBackend { */ final public function isSingleShardPathInternal( $storagePath ) { list( , , $shard ) = $this->resolveStoragePath( $storagePath ); + return ( $shard !== null ); } @@ -1323,7 +1507,7 @@ abstract class FileBackendStore extends FileBackend { * the container are required to be hashed accordingly. * * @param string $container - * @return Array (integer levels, integer base, repeat flag) or (0, 0, false) + * @return array (integer levels, integer base, repeat flag) or (0, 0, false) */ final protected function getContainerHashLevels( $container ) { if ( isset( $this->shardViaHashLevels[$container] ) ) { @@ -1336,6 +1520,7 @@ abstract class FileBackendStore extends FileBackend { } } } + return array( 0, 0, false ); // no sharding } @@ -1343,7 +1528,7 @@ abstract class FileBackendStore extends FileBackend { * Get a list of full container shard suffixes for a container * * @param string $container - * @return Array + * @return array */ final protected function getContainerSuffixes( $container ) { $shards = array(); @@ -1354,6 +1539,7 @@ abstract class FileBackendStore extends FileBackend { $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits ); } } + return $shards; } @@ -1404,7 +1590,7 @@ abstract class FileBackendStore extends FileBackend { * @return string */ private function containerCacheKey( $container ) { - return wfMemcKey( 'backend', $this->getName(), 'container', $container ); + return "filebackend:{$this->name}:{$this->wikiId}:container:{$container}"; } /** @@ -1412,7 +1598,6 @@ abstract class FileBackendStore extends FileBackend { * * @param string $container Resolved container name * @param array $val Information to cache - * @return void */ final protected function setContainerCache( $container, array $val ) { $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 ); @@ -1423,7 +1608,6 @@ abstract class FileBackendStore extends FileBackend { * The cache key is salted for a while to prevent race conditions. * * @param string $container Resolved container name - * @return void */ final protected function deleteContainerCache( $container ) { if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) { @@ -1433,11 +1617,10 @@ abstract class FileBackendStore extends FileBackend { /** * Do a batch lookup from cache for container stats for all containers - * used in a list of container names, storage paths, or FileOp objects. + * used in a list of container names or storage paths objects. * This loads the persistent cache values into the process cache. * - * @param Array $items - * @return void + * @param array $items */ final protected function primeContainerCache( array $items ) { $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); @@ -1446,10 +1629,7 @@ abstract class FileBackendStore extends FileBackend { $contNames = array(); // (cache key => resolved container name) // Get all the paths/containers from the items... foreach ( $items as $item ) { - if ( $item instanceof FileOp ) { - $paths = array_merge( $paths, $item->storagePathsRead() ); - $paths = array_merge( $paths, $item->storagePathsChanged() ); - } elseif ( self::isStoragePath( $item ) ) { + if ( self::isStoragePath( $item ) ) { $paths[] = $item; } elseif ( is_string( $item ) ) { // full container name $contNames[$this->containerCacheKey( $item )] = $item; @@ -1480,9 +1660,9 @@ abstract class FileBackendStore extends FileBackend { * Only containers that actually exist should appear in the map. * * @param array $containerInfo Map of resolved container names to cached info - * @return void */ - protected function doPrimeContainerCache( array $containerInfo ) {} + protected function doPrimeContainerCache( array $containerInfo ) { + } /** * Get the cache key for a file path @@ -1491,7 +1671,7 @@ abstract class FileBackendStore extends FileBackend { * @return string */ private function fileCacheKey( $path ) { - return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) ); + return "filebackend:{$this->name}:{$this->wikiId}:file:" . sha1( $path ); } /** @@ -1501,7 +1681,6 @@ abstract class FileBackendStore extends FileBackend { * * @param string $path Storage path * @param array $val Stat information to cache - * @return void */ final protected function setFileCache( $path, array $val ) { $path = FileBackend::normalizeStoragePath( $path ); @@ -1510,7 +1689,22 @@ abstract class FileBackendStore extends FileBackend { } $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] ); $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) ); - $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl ); + $key = $this->fileCacheKey( $path ); + // Set the cache unless it is currently salted with the value "PURGED". + // Using add() handles this except it also is a no-op in that case where + // the current value is not "latest" but $val is, so use CAS in that case. + if ( !$this->memCache->add( $key, $val, $ttl ) && !empty( $val['latest'] ) ) { + $this->memCache->merge( + $key, + function ( BagOStuff $cache, $key, $cValue ) use ( $val ) { + return ( is_array( $cValue ) && empty( $cValue['latest'] ) ) + ? $val // update the stat cache with the lastest info + : false; // do nothing (cache is salted or some error happened) + }, + $ttl, + 1 + ); + } } /** @@ -1520,7 +1714,6 @@ abstract class FileBackendStore extends FileBackend { * a file is created at a path were there was none before. * * @param string $path Storage path - * @return void */ final protected function deleteFileCache( $path ) { $path = FileBackend::normalizeStoragePath( $path ); @@ -1537,8 +1730,7 @@ abstract class FileBackendStore extends FileBackend { * used in a list of storage paths or FileOp objects. * This loads the persistent cache values into the process cache. * - * @param array $items List of storage paths or FileOps - * @return void + * @param array $items List of storage paths */ final protected function primeFileCache( array $items ) { $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); @@ -1547,10 +1739,7 @@ abstract class FileBackendStore extends FileBackend { $pathNames = array(); // (cache key => storage path) // Get all the paths/containers from the items... foreach ( $items as $item ) { - if ( $item instanceof FileOp ) { - $paths = array_merge( $paths, $item->storagePathsRead() ); - $paths = array_merge( $paths, $item->storagePathsChanged() ); - } elseif ( self::isStoragePath( $item ) ) { + if ( self::isStoragePath( $item ) ) { $paths[] = FileBackend::normalizeStoragePath( $item ); } } @@ -1573,15 +1762,41 @@ abstract class FileBackendStore extends FileBackend { $this->cheapCache->set( $path, 'sha1', array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) ); } + if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata + $val['xattr'] = self::normalizeXAttributes( $val['xattr'] ); + $this->cheapCache->set( $path, 'xattr', + array( 'map' => $val['xattr'], 'latest' => $val['latest'] ) ); + } } } } /** + * Normalize file headers/metadata to the FileBackend::getFileXAttributes() format + * + * @param array $xattr + * @return array + * @since 1.22 + */ + final protected static function normalizeXAttributes( array $xattr ) { + $newXAttr = array( 'headers' => array(), 'metadata' => array() ); + + foreach ( $xattr['headers'] as $name => $value ) { + $newXAttr['headers'][strtolower( $name )] = $value; + } + + foreach ( $xattr['metadata'] as $name => $value ) { + $newXAttr['metadata'][strtolower( $name )] = $value; + } + + return $newXAttr; + } + + /** * Set the 'concurrency' option from a list of operation options * * @param array $opts Map of operation options - * @return Array + * @return array */ final protected function setConcurrencyFlags( array $opts ) { $opts['concurrency'] = 1; // off @@ -1594,6 +1809,7 @@ abstract class FileBackendStore extends FileBackend { $opts['concurrency'] = $this->concurrency; } } + return $opts; } @@ -1603,7 +1819,7 @@ abstract class FileBackendStore extends FileBackend { * @param string $storagePath * @param string|null $content File data * @param string|null $fsPath File system path - * @return MIME type + * @return string MIME type */ protected function getContentType( $storagePath, $content, $fsPath ) { return call_user_func_array( $this->mimeCallback, func_get_args() ); @@ -1619,19 +1835,17 @@ abstract class FileBackendStore extends FileBackend { * passed to FileBackendStore::executeOpHandlesInternal(). */ abstract class FileBackendStoreOpHandle { - /** @var Array */ + /** @var array */ public $params = array(); // params to caller functions /** @var FileBackendStore */ public $backend; - /** @var Array */ + /** @var array */ public $resourcesToClose = array(); public $call; // string; name that identifies the function called /** * Close all open file handles - * - * @return void */ public function closeResources() { array_map( 'fclose', $this->resourcesToClose ); @@ -1647,13 +1861,17 @@ abstract class FileBackendStoreOpHandle { abstract class FileBackendStoreShardListIterator extends FilterIterator { /** @var FileBackendStore */ protected $backend; - /** @var Array */ + + /** @var array */ protected $params; - protected $container; // string; full container name - protected $directory; // string; resolved relative path + /** @var string Full container name */ + protected $container; - /** @var Array */ + /** @var string Resolved relative path */ + protected $directory; + + /** @var array */ protected $multiShardPaths = array(); // (rel path => 1) /** @@ -1689,6 +1907,7 @@ abstract class FileBackendStoreShardListIterator extends FilterIterator { return false; } else { $this->multiShardPaths[$rel] = 1; + return true; } } diff --git a/includes/filebackend/FileOp.php b/includes/filebackend/FileOp.php index fe833084..66d87943 100644 --- a/includes/filebackend/FileOp.php +++ b/includes/filebackend/FileOp.php @@ -34,20 +34,35 @@ * @since 1.19 */ abstract class FileOp { - /** @var Array */ + /** @var array */ protected $params = array(); + /** @var FileBackendStore */ protected $backend; - protected $state = self::STATE_NEW; // integer - protected $failed = false; // boolean - protected $async = false; // boolean - protected $batchId; // string + /** @var int */ + protected $state = self::STATE_NEW; + + /** @var bool */ + protected $failed = false; + + /** @var bool */ + protected $async = false; + + /** @var string */ + protected $batchId; - protected $doOperation = true; // boolean; operation is not a no-op - protected $sourceSha1; // string - protected $overwriteSameCase; // boolean - protected $destExists; // boolean + /** @var bool Operation is not a no-op */ + protected $doOperation = true; + + /** @var string */ + protected $sourceSha1; + + /** @var bool */ + protected $overwriteSameCase; + + /** @var bool */ + protected $destExists; /* Object life-cycle */ const STATE_NEW = 1; @@ -58,47 +73,29 @@ abstract class FileOp { * Build a new batch file operation transaction * * @param FileBackendStore $backend - * @param Array $params - * @throws MWException + * @param array $params + * @throws FileBackendError */ final public function __construct( FileBackendStore $backend, array $params ) { $this->backend = $backend; - list( $required, $optional ) = $this->allowedParams(); - // @todo normalizeAnyStoragePaths() calls are overzealous, use a parameter list + list( $required, $optional, $paths ) = $this->allowedParams(); foreach ( $required as $name ) { if ( isset( $params[$name] ) ) { - // Normalize paths so the paths to the same file have the same string - $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] ); + $this->params[$name] = $params[$name]; } else { - throw new MWException( "File operation missing parameter '$name'." ); + throw new FileBackendError( "File operation missing parameter '$name'." ); } } foreach ( $optional as $name ) { if ( isset( $params[$name] ) ) { - // Normalize paths so the paths to the same file have the same string - $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] ); + $this->params[$name] = $params[$name]; } } - $this->params = $params; - } - - /** - * Normalize $item or anything in $item that is a valid storage path - * - * @param string $item|array - * @return string|Array - */ - protected function normalizeAnyStoragePaths( $item ) { - if ( is_array( $item ) ) { - $res = array(); - foreach ( $item as $k => $v ) { - $k = self::normalizeIfValidStoragePath( $k ); - $v = self::normalizeIfValidStoragePath( $v ); - $res[$k] = $v; + foreach ( $paths as $name ) { + if ( isset( $this->params[$name] ) ) { + // Normalize paths so the paths to the same file have the same string + $this->params[$name] = self::normalizeIfValidStoragePath( $this->params[$name] ); } - return $res; - } else { - return self::normalizeIfValidStoragePath( $item ); } } @@ -111,8 +108,10 @@ abstract class FileOp { protected static function normalizeIfValidStoragePath( $path ) { if ( FileBackend::isStoragePath( $path ) ) { $res = FileBackend::normalizeStoragePath( $path ); + return ( $res !== null ) ? $res : $path; } + return $path; } @@ -120,7 +119,6 @@ abstract class FileOp { * Set the batch UUID this operation belongs to * * @param string $batchId - * @return void */ final public function setBatchId( $batchId ) { $this->batchId = $batchId; @@ -148,7 +146,7 @@ abstract class FileOp { /** * Get a new empty predicates array for precheck() * - * @return Array + * @return array */ final public static function newPredicates() { return array( 'exists' => array(), 'sha1' => array() ); @@ -157,7 +155,7 @@ abstract class FileOp { /** * Get a new empty dependency tracking array for paths read/written to * - * @return Array + * @return array */ final public static function newDependencies() { return array( 'read' => array(), 'write' => array() ); @@ -167,19 +165,20 @@ abstract class FileOp { * Update a dependency tracking array to account for this operation * * @param array $deps Prior path reads/writes; format of FileOp::newPredicates() - * @return Array + * @return array */ final public function applyDependencies( array $deps ) { $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 ); $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 ); + return $deps; } /** * Check if this operation changes files listed in $paths * - * @param array $paths Prior path reads/writes; format of FileOp::newPredicates() - * @return boolean + * @param array $deps Prior path reads/writes; format of FileOp::newPredicates() + * @return bool */ final public function dependsOn( array $deps ) { foreach ( $this->storagePathsChanged() as $path ) { @@ -192,6 +191,7 @@ abstract class FileOp { return true; // "flow" dependency } } + return false; } @@ -200,7 +200,7 @@ abstract class FileOp { * * @param array $oPredicates Pre-op info about files (format of FileOp::newPredicates) * @param array $nPredicates Post-op info about files (format of FileOp::newPredicates) - * @return Array + * @return array */ final public function getJournalEntries( array $oPredicates, array $nPredicates ) { if ( !$this->doOperation ) { @@ -232,6 +232,7 @@ abstract class FileOp { ); } } + return array_merge( $nullEntries, $updateEntries, $deleteEntries ); } @@ -240,7 +241,7 @@ abstract class FileOp { * This must update $predicates for each path that the op can change * except when a failing status object is returned. * - * @param Array $predicates + * @param array $predicates * @return Status */ final public function precheck( array &$predicates ) { @@ -252,10 +253,12 @@ abstract class FileOp { if ( !$status->isOK() ) { $this->failed = true; } + return $status; } /** + * @param array $predicates * @return Status */ protected function doPrecheck( array &$predicates ) { @@ -283,6 +286,7 @@ abstract class FileOp { } else { // no-op $status = Status::newGood(); } + return $status; } @@ -302,23 +306,24 @@ abstract class FileOp { $this->async = true; $result = $this->attempt(); $this->async = false; + return $result; } /** * Get the file operation parameters * - * @return Array (required params list, optional params list) + * @return array (required params list, optional params list, list of params that are paths) */ protected function allowedParams() { - return array( array(), array() ); + return array( array(), array(), array() ); } /** * Adjust params to FileBackendStore internal file calls * - * @param Array $params - * @return Array (required params list, optional params list) + * @param array $params + * @return array (required params list, optional params list) */ protected function setFlags( array $params ) { return array( 'async' => $this->async ) + $params; @@ -327,7 +332,7 @@ abstract class FileOp { /** * Get a list of storage paths read from for this operation * - * @return Array + * @return array */ public function storagePathsRead() { return array(); @@ -336,7 +341,7 @@ abstract class FileOp { /** * Get a list of storage paths written to for this operation * - * @return Array + * @return array */ public function storagePathsChanged() { return array(); @@ -347,7 +352,7 @@ abstract class FileOp { * Also set the destExists, overwriteSameCase and sourceSha1 member variables. * A bad status will be returned if there is no chance it can be overwritten. * - * @param Array $predicates + * @param array $predicates * @return Status */ protected function precheckDestExistence( array $predicates ) { @@ -373,12 +378,15 @@ abstract class FileOp { } else { $this->overwriteSameCase = true; // OK } + return $status; // do nothing; either OK or bad status } else { $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] ); + return $status; } } + return $status; } @@ -396,7 +404,7 @@ abstract class FileOp { * Check if a file will exist in storage when this operation is attempted * * @param string $source Storage path - * @param Array $predicates + * @param array $predicates * @return bool */ final protected function fileExists( $source, array $predicates ) { @@ -404,6 +412,7 @@ abstract class FileOp { return $predicates['exists'][$source]; // previous op assures this } else { $params = array( 'src' => $source, 'latest' => true ); + return $this->backend->fileExists( $params ); } } @@ -412,7 +421,7 @@ abstract class FileOp { * Get the SHA-1 of a file in storage when this operation is attempted * * @param string $source Storage path - * @param Array $predicates + * @param array $predicates * @return string|bool False on failure */ final protected function fileSha1( $source, array $predicates ) { @@ -422,6 +431,7 @@ abstract class FileOp { return false; // previous op assures this } else { $params = array( 'src' => $source, 'latest' => true ); + return $this->backend->getFileSha1Base36( $params ); } } @@ -439,7 +449,6 @@ abstract class FileOp { * Log a file operation failure and preserve any temp files * * @param string $action - * @return void */ final public function logFailure( $action ) { $params = $this->params; @@ -459,8 +468,11 @@ abstract class FileOp { */ class CreateFileOp extends FileOp { protected function allowedParams() { - return array( array( 'content', 'dst' ), - array( 'overwrite', 'overwriteSame', 'headers' ) ); + return array( + array( 'content', 'dst' ), + array( 'overwrite', 'overwriteSame', 'headers' ), + array( 'dst' ) + ); } protected function doPrecheck( array &$predicates ) { @@ -470,11 +482,13 @@ class CreateFileOp extends FileOp { $status->fatal( 'backend-fail-maxsize', $this->params['dst'], $this->backend->maxFileSizeInternal() ); $status->fatal( 'backend-fail-create', $this->params['dst'] ); + return $status; // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); $status->fatal( 'backend-fail-create', $this->params['dst'] ); + return $status; } // Check if destination file exists @@ -485,6 +499,7 @@ class CreateFileOp extends FileOp { $predicates['exists'][$this->params['dst']] = true; $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; } + return $status; // safe to call attempt() } @@ -493,6 +508,7 @@ class CreateFileOp extends FileOp { // Create the file at the destination return $this->backend->createInternal( $this->setFlags( $this->params ) ); } + return Status::newGood(); } @@ -511,8 +527,11 @@ class CreateFileOp extends FileOp { */ class StoreFileOp extends FileOp { protected function allowedParams() { - return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'headers' ) ); + return array( + array( 'src', 'dst' ), + array( 'overwrite', 'overwriteSame', 'headers' ), + array( 'src', 'dst' ) + ); } protected function doPrecheck( array &$predicates ) { @@ -520,17 +539,20 @@ class StoreFileOp extends FileOp { // Check if the source file exists on the file system if ( !is_file( $this->params['src'] ) ) { $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; // Check if the source file is too big } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) { $status->fatal( 'backend-fail-maxsize', $this->params['dst'], $this->backend->maxFileSizeInternal() ); $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); + return $status; // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); + return $status; } // Check if destination file exists @@ -541,6 +563,7 @@ class StoreFileOp extends FileOp { $predicates['exists'][$this->params['dst']] = true; $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; } + return $status; // safe to call attempt() } @@ -549,6 +572,7 @@ class StoreFileOp extends FileOp { // Store the file at the destination return $this->backend->storeInternal( $this->setFlags( $this->params ) ); } + return Status::newGood(); } @@ -559,6 +583,7 @@ class StoreFileOp extends FileOp { if ( $hash !== false ) { $hash = wfBaseConvert( $hash, 16, 36, 31 ); } + return $hash; } @@ -573,8 +598,11 @@ class StoreFileOp extends FileOp { */ class CopyFileOp extends FileOp { protected function allowedParams() { - return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) ); + return array( + array( 'src', 'dst' ), + array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ), + array( 'src', 'dst' ) + ); } protected function doPrecheck( array &$predicates ) { @@ -586,15 +614,18 @@ class CopyFileOp extends FileOp { // Update file existence predicates (cache 404s) $predicates['exists'][$this->params['src']] = false; $predicates['sha1'][$this->params['src']] = false; + return $status; // nothing to do } else { $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; } - // Check if a file can be placed/changed at the destination + // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] ); + return $status; } // Check if destination file exists @@ -605,6 +636,7 @@ class CopyFileOp extends FileOp { $predicates['exists'][$this->params['dst']] = true; $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; } + return $status; // safe to call attempt() } @@ -621,6 +653,7 @@ class CopyFileOp extends FileOp { // Copy the file to the destination $status = $this->backend->copyInternal( $this->setFlags( $this->params ) ); } + return $status; } @@ -639,8 +672,11 @@ class CopyFileOp extends FileOp { */ class MoveFileOp extends FileOp { protected function allowedParams() { - return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) ); + return array( + array( 'src', 'dst' ), + array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ), + array( 'src', 'dst' ) + ); } protected function doPrecheck( array &$predicates ) { @@ -652,15 +688,18 @@ class MoveFileOp extends FileOp { // Update file existence predicates (cache 404s) $predicates['exists'][$this->params['src']] = false; $predicates['sha1'][$this->params['src']] = false; + return $status; // nothing to do } else { $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; } // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] ); + return $status; } // Check if destination file exists @@ -673,6 +712,7 @@ class MoveFileOp extends FileOp { $predicates['exists'][$this->params['dst']] = true; $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; } + return $status; // safe to call attempt() } @@ -697,6 +737,7 @@ class MoveFileOp extends FileOp { // Move the file to the destination $status = $this->backend->moveInternal( $this->setFlags( $this->params ) ); } + return $status; } @@ -715,7 +756,7 @@ class MoveFileOp extends FileOp { */ class DeleteFileOp extends FileOp { protected function allowedParams() { - return array( array( 'src' ), array( 'ignoreMissingSource' ) ); + return array( array( 'src' ), array( 'ignoreMissingSource' ), array( 'src' ) ); } protected function doPrecheck( array &$predicates ) { @@ -727,20 +768,24 @@ class DeleteFileOp extends FileOp { // Update file existence predicates (cache 404s) $predicates['exists'][$this->params['src']] = false; $predicates['sha1'][$this->params['src']] = false; + return $status; // nothing to do } else { $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; } // Check if a file can be placed/changed at the source } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['src'] ); $status->fatal( 'backend-fail-delete', $this->params['src'] ); + return $status; } // Update file existence predicates $predicates['exists'][$this->params['src']] = false; $predicates['sha1'][$this->params['src']] = false; + return $status; // safe to call attempt() } @@ -760,7 +805,7 @@ class DeleteFileOp extends FileOp { */ class DescribeFileOp extends FileOp { protected function allowedParams() { - return array( array( 'src' ), array( 'headers' ) ); + return array( array( 'src' ), array( 'headers' ), array( 'src' ) ); } protected function doPrecheck( array &$predicates ) { @@ -768,11 +813,13 @@ class DescribeFileOp extends FileOp { // Check if the source file exists if ( !$this->fileExists( $this->params['src'], $predicates ) ) { $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; // Check if a file can be placed/changed at the source } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['src'] ); $status->fatal( 'backend-fail-describe', $this->params['src'] ); + return $status; } // Update file existence predicates @@ -780,6 +827,7 @@ class DescribeFileOp extends FileOp { $this->fileExists( $this->params['src'], $predicates ); $predicates['sha1'][$this->params['src']] = $this->fileSha1( $this->params['src'], $predicates ); + return $status; // safe to call attempt() } @@ -796,4 +844,5 @@ class DescribeFileOp extends FileOp { /** * Placeholder operation that has no params and does nothing */ -class NullFileOp extends FileOp {} +class NullFileOp extends FileOp { +} diff --git a/includes/filebackend/FileOpBatch.php b/includes/filebackend/FileOpBatch.php index 785c0bc9..b0d83e01 100644 --- a/includes/filebackend/FileOpBatch.php +++ b/includes/filebackend/FileOpBatch.php @@ -55,13 +55,13 @@ class FileOpBatch { * @return Status */ public static function attempt( array $performOps, array $opts, FileJournal $journal ) { - wfProfileIn( __METHOD__ ); + $section = new ProfileSection( __METHOD__ ); $status = Status::newGood(); $n = count( $performOps ); if ( $n > self::MAX_BATCH_SIZE ) { $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE ); - wfProfileOut( __METHOD__ ); + return $status; } @@ -107,7 +107,6 @@ class FileOpBatch { $status->success[$index] = false; ++$status->failCount; if ( !$ignoreErrors ) { - wfProfileOut( __METHOD__ ); return $status; // abort } } @@ -121,7 +120,6 @@ class FileOpBatch { if ( count( $entries ) ) { $subStatus = $journal->logChangeBatch( $entries, $batchId ); if ( !$subStatus->isOK() ) { - wfProfileOut( __METHOD__ ); return $subStatus; // abort } } @@ -133,7 +131,6 @@ class FileOpBatch { // Attempt each operation (in parallel if allowed and possible)... self::runParallelBatches( $pPerformOps, $status ); - wfProfileOut( __METHOD__ ); return $status; } @@ -145,9 +142,8 @@ class FileOpBatch { * within any given sub-batch do not depend on each other. * This will abort remaining ops on failure. * - * @param Array $pPerformOps + * @param array $pPerformOps Batches of file ops (batches use original indexes) * @param Status $status - * @return bool Success */ protected static function runParallelBatches( array $pPerformOps, Status $status ) { $aborted = false; // set to true on unexpected errors @@ -156,6 +152,8 @@ class FileOpBatch { // We can't continue (even with $ignoreErrors) as $predicates is wrong. // Log the remaining ops as failed for recovery... foreach ( $performOpsBatch as $i => $fileOp ) { + $status->success[$i] = false; + ++$status->failCount; $performOpsBatch[$i]->logFailure( 'attempt_aborted' ); } continue; @@ -168,9 +166,9 @@ class FileOpBatch { // If attemptAsync() returns a Status, it was either due to an error // or the backend does not support async ops and did it synchronously. foreach ( $performOpsBatch as $i => $fileOp ) { - if ( !$fileOp->failed() ) { // failed => already has Status - // If the batch is just one operation, it's faster to avoid - // pipelining as that can involve creating new TCP connections. + if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck() + // Parallel ops may be disabled in config due to missing dependencies, + // (e.g. needing popen()). When they are, $performOpsBatch has size 1. $subStatus = ( count( $performOpsBatch ) > 1 ) ? $fileOp->attemptAsync() : $fileOp->attempt(); @@ -185,7 +183,7 @@ class FileOpBatch { $statuses = $statuses + $backend->executeOpHandlesInternal( $opHandles ); // Marshall and merge all the responses (blocking)... foreach ( $performOpsBatch as $i => $fileOp ) { - if ( !$fileOp->failed() ) { // failed => already has Status + if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck() $subStatus = $statuses[$i]; $status->merge( $subStatus ); if ( $subStatus->isOK() ) { @@ -199,6 +197,5 @@ class FileOpBatch { } } } - return $status; } } diff --git a/includes/filebackend/MemoryFileBackend.php b/includes/filebackend/MemoryFileBackend.php new file mode 100644 index 00000000..7c2f8256 --- /dev/null +++ b/includes/filebackend/MemoryFileBackend.php @@ -0,0 +1,274 @@ +<?php +/** + * Simulation of a backend storage in memory. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup FileBackend + * @author Aaron Schulz + */ + +/** + * Simulation of a backend storage in memory. + * + * All data in the backend is automatically deleted at the end of PHP execution. + * Since the data stored here is volatile, this is only useful for staging or testing. + * + * @ingroup FileBackend + * @since 1.23 + */ +class MemoryFileBackend extends FileBackendStore { + /** @var array Map of (file path => (data,mtime) */ + protected $files = array(); + + public function isPathUsableInternal( $storagePath ) { + return true; + } + + protected function doCreateInternal( array $params ) { + $status = Status::newGood(); + + $dst = $this->resolveHashKey( $params['dst'] ); + if ( $dst === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + + return $status; + } + + $this->files[$dst] = array( + 'data' => $params['content'], + 'mtime' => wfTimestamp( TS_MW, time() ) + ); + + return $status; + } + + protected function doStoreInternal( array $params ) { + $status = Status::newGood(); + + $dst = $this->resolveHashKey( $params['dst'] ); + if ( $dst === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + + return $status; + } + + wfSuppressWarnings(); + $data = file_get_contents( $params['src'] ); + wfRestoreWarnings(); + if ( $data === false ) { // source doesn't exist? + $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + + return $status; + } + + $this->files[$dst] = array( + 'data' => $data, + 'mtime' => wfTimestamp( TS_MW, time() ) + ); + + return $status; + } + + protected function doCopyInternal( array $params ) { + $status = Status::newGood(); + + $src = $this->resolveHashKey( $params['src'] ); + if ( $src === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + + return $status; + } + + $dst = $this->resolveHashKey( $params['dst'] ); + if ( $dst === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + + return $status; + } + + if ( !isset( $this->files[$src] ) ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } + + return $status; + } + + $this->files[$dst] = array( + 'data' => $this->files[$src]['data'], + 'mtime' => wfTimestamp( TS_MW, time() ) + ); + + return $status; + } + + protected function doDeleteInternal( array $params ) { + $status = Status::newGood(); + + $src = $this->resolveHashKey( $params['src'] ); + if ( $src === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + + return $status; + } + + if ( !isset( $this->files[$src] ) ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } + + return $status; + } + + unset( $this->files[$src] ); + + return $status; + } + + protected function doGetFileStat( array $params ) { + $src = $this->resolveHashKey( $params['src'] ); + if ( $src === null ) { + return null; + } + + if ( isset( $this->files[$src] ) ) { + return array( + 'mtime' => $this->files[$src]['mtime'], + 'size' => strlen( $this->files[$src]['data'] ), + ); + } + + return false; + } + + protected function doGetLocalCopyMulti( array $params ) { + $tmpFiles = array(); // (path => TempFSFile) + foreach ( $params['srcs'] as $srcPath ) { + $src = $this->resolveHashKey( $srcPath ); + if ( $src === null || !isset( $this->files[$src] ) ) { + $fsFile = null; + } else { + // Create a new temporary file with the same extension... + $ext = FileBackend::extensionFromPath( $src ); + $fsFile = TempFSFile::factory( 'localcopy_', $ext ); + if ( $fsFile ) { + $bytes = file_put_contents( $fsFile->getPath(), $this->files[$src]['data'] ); + if ( $bytes !== strlen( $this->files[$src]['data'] ) ) { + $fsFile = null; + } + } + } + $tmpFiles[$srcPath] = $fsFile; + } + + return $tmpFiles; + } + + protected function doStreamFile( array $params ) { + $status = Status::newGood(); + + $src = $this->resolveHashKey( $params['src'] ); + if ( $src === null || !isset( $this->files[$src] ) ) { + $status->fatal( 'backend-fail-stream', $params['src'] ); + + return $status; + } + + print $this->files[$src]['data']; + + return $status; + } + + protected function doDirectoryExists( $container, $dir, array $params ) { + $prefix = rtrim( "$container/$dir", '/' ) . '/'; + foreach ( $this->files as $path => $data ) { + if ( strpos( $path, $prefix ) === 0 ) { + return true; + } + } + + return false; + } + + public function getDirectoryListInternal( $container, $dir, array $params ) { + $dirs = array(); + $prefix = rtrim( "$container/$dir", '/' ) . '/'; + $prefixLen = strlen( $prefix ); + foreach ( $this->files as $path => $data ) { + if ( strpos( $path, $prefix ) === 0 ) { + $relPath = substr( $path, $prefixLen ); + if ( $relPath === false ) { + continue; + } elseif ( strpos( $relPath, '/' ) === false ) { + continue; // just a file + } + $parts = array_slice( explode( '/', $relPath ), 0, -1 ); // last part is file name + if ( !empty( $params['topOnly'] ) ) { + $dirs[$parts[0]] = 1; // top directory + } else { + $current = ''; + foreach ( $parts as $part ) { // all directories + $dir = ( $current === '' ) ? $part : "$current/$part"; + $dirs[$dir] = 1; + $current = $dir; + } + } + } + } + + return array_keys( $dirs ); + } + + public function getFileListInternal( $container, $dir, array $params ) { + $files = array(); + $prefix = rtrim( "$container/$dir", '/' ) . '/'; + $prefixLen = strlen( $prefix ); + foreach ( $this->files as $path => $data ) { + if ( strpos( $path, $prefix ) === 0 ) { + $relPath = substr( $path, $prefixLen ); + if ( $relPath === false ) { + continue; + } elseif ( !empty( $params['topOnly'] ) && strpos( $relPath, '/' ) !== false ) { + continue; + } + $files[] = $relPath; + } + } + + return $files; + } + + protected function directoriesAreVirtual() { + return true; + } + + /** + * Get the absolute file system path for a storage path + * + * @param string $storagePath Storage path + * @return string|null + */ + protected function resolveHashKey( $storagePath ) { + list( $fullCont, $relPath ) = $this->resolveStoragePathReal( $storagePath ); + if ( $relPath === null ) { + return null; // invalid + } + + return ( $relPath !== '' ) ? "$fullCont/$relPath" : $fullCont; + } +} diff --git a/includes/filebackend/README b/includes/filebackend/README index 569f3376..c06f6fc7 100644 --- a/includes/filebackend/README +++ b/includes/filebackend/README @@ -51,7 +51,7 @@ On files: * read a file into a string or several files into a map of path names to strings * download a file or set of files to a temporary file (on a mounted file system) * get the SHA1 hash of a file -* get various properties of a file (stat information, content time, mime information, ...) +* get various properties of a file (stat information, content time, MIME information, ...) On directories: * get a list of files directly under a directory diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php index db090a98..f40ec46e 100644 --- a/includes/filebackend/SwiftFileBackend.php +++ b/includes/filebackend/SwiftFileBackend.php @@ -26,10 +26,6 @@ /** * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend. * - * This requires the SwiftCloudFiles MediaWiki extension, which includes - * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles). - * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions. - * * Status messages should avoid mentioning the Swift account name. * Likewise, error suppression should be used to avoid path disclosure. * @@ -37,32 +33,47 @@ * @since 1.19 */ class SwiftFileBackend extends FileBackendStore { - /** @var CF_Authentication */ - protected $auth; // Swift authentication handler - protected $authTTL; // integer seconds - protected $swiftTempUrlKey; // string; shared secret value for making temp urls - protected $swiftAnonUser; // string; username to handle unauthenticated requests - protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled - protected $swiftCDNExpiry; // integer; how long to cache things in the CDN - protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled - - // Rados Gateway specific options - protected $rgwS3AccessKey; // string; S3 access key - protected $rgwS3SecretKey; // string; S3 authentication key - - /** @var CF_Connection */ - protected $conn; // Swift connection handle - protected $sessionStarted = 0; // integer UNIX timestamp - - /** @var CloudFilesException */ - protected $connException; - protected $connErrorTime = 0; // UNIX timestamp + /** @var MultiHttpClient */ + protected $http; + + /** @var int TTL in seconds */ + protected $authTTL; + + /** @var string Authentication base URL (without version) */ + protected $swiftAuthUrl; + + /** @var string Swift user (account:user) to authenticate as */ + protected $swiftUser; + + /** @var string Secret key for user */ + protected $swiftKey; + + /** @var string Shared secret value for making temp URLs */ + protected $swiftTempUrlKey; + + /** @var string S3 access key (RADOS Gateway) */ + protected $rgwS3AccessKey; + + /** @var string S3 authentication key (RADOS Gateway) */ + protected $rgwS3SecretKey; /** @var BagOStuff */ protected $srvCache; - /** @var ProcessCacheLRU */ - protected $connContainerCache; // container object cache + /** @var ProcessCacheLRU Container stat cache */ + protected $containerStatCache; + + /** @var array */ + protected $authCreds; + + /** @var int UNIX timestamp */ + protected $authSessionTimestamp = 0; + + /** @var int UNIX timestamp */ + protected $authErrorTimestamp = null; + + /** @var bool Whether the server is an Ceph RGW */ + protected $isRGW = false; /** * @see FileBackendStore::__construct() @@ -73,16 +84,6 @@ class SwiftFileBackend extends FileBackendStore { * - swiftAuthTTL : Swift authentication TTL (seconds) * - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account. * Do not set this until it has been set in the backend. - * - swiftAnonUser : Swift user used for end-user requests (account:username). - * If set, then views of public containers are assumed to go - * through this user. If not set, then public containers are - * accessible to unauthenticated requests via ".r:*" in the ACL. - * - swiftUseCDN : Whether a Cloud Files Content Delivery Network is set up - * - swiftCDNExpiry : How long (in seconds) to store content in the CDN. - * If files may likely change, this should probably not exceed - * a few days. For example, deletions may take this long to apply. - * If object purging is enabled, however, this is not an issue. - * - swiftCDNPurgable : Whether object purge requests are allowed by the CDN. * - shardViaHashLevels : Map of container names to sharding config with: * - base : base of hash characters, 16 or 36 * - levels : the number of hash levels (and digits) @@ -91,12 +92,12 @@ class SwiftFileBackend extends FileBackendStore { * - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect. * If those are not available, then the main cache will be used. * This is probably insecure in shared hosting environments. - * - rgwS3AccessKey : Ragos Gateway S3 "access key" value on the account. + * - rgwS3AccessKey : Rados Gateway S3 "access key" value on the account. * Do not set this until it has been set in the backend. * This is used for generating expiring pre-authenticated URLs. * Only use this when using rgw and to work around * http://tracker.newdream.net/issues/3454. - * - rgwS3SecretKey : Ragos Gateway S3 "secret key" value on the account. + * - rgwS3SecretKey : Rados Gateway S3 "secret key" value on the account. * Do not set this until it has been set in the backend. * This is used for generating expiring pre-authenticated URLs. * Only use this when using rgw and to work around @@ -104,48 +105,32 @@ class SwiftFileBackend extends FileBackendStore { */ public function __construct( array $config ) { parent::__construct( $config ); - if ( !class_exists( 'CF_Constants' ) ) { - throw new MWException( 'SwiftCloudFiles extension not installed.' ); - } // Required settings - $this->auth = new CF_Authentication( - $config['swiftUser'], - $config['swiftKey'], - null, // account; unused - $config['swiftAuthUrl'] - ); + $this->swiftAuthUrl = $config['swiftAuthUrl']; + $this->swiftUser = $config['swiftUser']; + $this->swiftKey = $config['swiftKey']; // Optional settings $this->authTTL = isset( $config['swiftAuthTTL'] ) ? $config['swiftAuthTTL'] : 5 * 60; // some sane number - $this->swiftAnonUser = isset( $config['swiftAnonUser'] ) - ? $config['swiftAnonUser'] - : ''; $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] ) ? $config['swiftTempUrlKey'] : ''; $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) ? $config['shardViaHashLevels'] : ''; - $this->swiftUseCDN = isset( $config['swiftUseCDN'] ) - ? $config['swiftUseCDN'] - : false; - $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] ) - ? $config['swiftCDNExpiry'] - : 12 * 3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) - $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] ) - ? $config['swiftCDNPurgable'] - : true; $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] ) ? $config['rgwS3AccessKey'] : ''; $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] ) ? $config['rgwS3SecretKey'] : ''; + // HTTP helper client + $this->http = new MultiHttpClient( array() ); // Cache container information to mask latency $this->memCache = wfGetMainCache(); // Process cache for container info - $this->connContainerCache = new ProcessCacheLRU( 300 ); + $this->containerStatCache = new ProcessCacheLRU( 300 ); // Cache auth token information to avoid RTTs if ( !empty( $config['cacheAuthInfo'] ) ) { if ( PHP_SAPI === 'cli' ) { @@ -153,22 +138,25 @@ class SwiftFileBackend extends FileBackendStore { } else { try { // look for APC, XCache, WinCache, ect... $this->srvCache = ObjectCache::newAccelerator( array() ); - } catch ( Exception $e ) {} + } catch ( Exception $e ) { + } } } - $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff(); + $this->srvCache = $this->srvCache ?: new EmptyBagOStuff(); + } + + public function getFeatures() { + return ( FileBackend::ATTR_UNICODE_PATHS | + FileBackend::ATTR_HEADERS | FileBackend::ATTR_METADATA ); } - /** - * @see FileBackendStore::resolveContainerPath() - * @return null - */ protected function resolveContainerPath( $container, $relStoragePath ) { if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF return null; // not UTF-8, makes it hard to use CF and the swift HTTP API } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { return null; // too long for Swift } + return $relStoragePath; } @@ -178,45 +166,48 @@ class SwiftFileBackend extends FileBackendStore { return false; // invalid } - try { - $this->getContainer( $container ); - return true; // container exists - } catch ( NoSuchContainerException $e ) { - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) ); - } - - return false; + return is_array( $this->getContainerStat( $container ) ); } /** - * @param array $headers - * @return array + * Sanitize and filter the custom headers from a $params array. + * We only allow certain Content- and X-Content- headers. + * + * @param array $params + * @return array Sanitized value of 'headers' field in $params */ - protected function sanitizeHdrs( array $headers ) { - // By default, Swift has annoyingly low maximum header value limits - if ( isset( $headers['Content-Disposition'] ) ) { - $headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] ); + protected function sanitizeHdrs( array $params ) { + $headers = array(); + + // Normalize casing, and strip out illegal headers + if ( isset( $params['headers'] ) ) { + foreach ( $params['headers'] as $name => $value ) { + $name = strtolower( $name ); + if ( preg_match( '/^content-(type|length)$/', $name ) ) { + continue; // blacklisted + } elseif ( preg_match( '/^(x-)?content-/', $name ) ) { + $headers[$name] = $value; // allowed + } elseif ( preg_match( '/^content-(disposition)/', $name ) ) { + $headers[$name] = $value; // allowed + } + } } - return $headers; - } - - /** - * @param string $disposition Content-Disposition header value - * @return string Truncated Content-Disposition header value to meet Swift limits - */ - protected function truncDisp( $disposition ) { - $res = ''; - foreach ( explode( ';', $disposition ) as $part ) { - $part = trim( $part ); - $new = ( $res === '' ) ? $part : "{$res};{$part}"; - if ( strlen( $new ) <= 255 ) { - $res = $new; - } else { - break; // too long; sigh + // By default, Swift has annoyingly low maximum header value limits + if ( isset( $headers['content-disposition'] ) ) { + $disposition = ''; + foreach ( explode( ';', $headers['content-disposition'] ) as $part ) { + $part = trim( $part ); + $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}"; + if ( strlen( $new ) <= 255 ) { + $disposition = $new; + } else { + break; // too long; sigh + } } + $headers['content-disposition'] = $disposition; } - return $res; + + return $headers; } protected function doCreateInternal( array $params ) { @@ -225,152 +216,109 @@ class SwiftFileBackend extends FileBackendStore { list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); if ( $dstRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - return $status; - } - // (a) Check the destination container and object - try { - $dContObj = $this->getContainer( $dstCont ); - } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - return $status; - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); return $status; } - // (b) Get a SHA-1 hash of the object $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 ); - - // (c) Actually create the object - try { - // Create a fresh CF_Object with no fields preloaded. - // We don't want to preserve headers, metadata, and such. - $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); - // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59). - // The MD5 here will be checked within Swift against its own MD5. - $obj->set_etag( md5( $params['content'] ) ); - // Use the same content type as StreamFile for security - $obj->content_type = $this->getContentType( $params['dst'], $params['content'], null ); - // Set any other custom headers if requested - if ( isset( $params['headers'] ) ) { - $obj->headers += $this->sanitizeHdrs( $params['headers'] ); - } - if ( !empty( $params['async'] ) ) { // deferred - $op = $obj->write_async( $params['content'] ); - $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op ); - $status->value->affectedObjects[] = $obj; - } else { // actually write the object in Swift - $obj->write( $params['content'] ); - $this->purgeCDNCache( array( $obj ) ); + $contentType = $this->getContentType( $params['dst'], $params['content'], null ); + + $reqs = array( array( + 'method' => 'PUT', + 'url' => array( $dstCont, $dstRel ), + 'headers' => array( + 'content-length' => strlen( $params['content'] ), + 'etag' => md5( $params['content'] ), + 'content-type' => $contentType, + 'x-object-meta-sha1base36' => $sha1Hash + ) + $this->sanitizeHdrs( $params ), + 'body' => $params['content'] + ) ); + + $be = $this; + $method = __METHOD__; + $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; + if ( $rcode === 201 ) { + // good + } elseif ( $rcode === 412 ) { + $status->fatal( 'backend-fail-contenttype', $params['dst'] ); + } else { + $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( BadContentTypeException $e ) { - $status->fatal( 'backend-fail-contenttype', $params['dst'] ); - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + }; + + $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); + if ( !empty( $params['async'] ) ) { // deferred + $status->value = $opHandle; + } else { // actually write the object in Swift + $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); } return $status; } - /** - * @see SwiftFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) { - try { - $cfOp->getLastResponse(); - } catch ( BadContentTypeException $e ) { - $status->fatal( 'backend-fail-contenttype', $params['dst'] ); - } - } - protected function doStoreInternal( array $params ) { $status = Status::newGood(); list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); if ( $dstRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - return $status; - } - // (a) Check the destination container and object - try { - $dContObj = $this->getContainer( $dstCont ); - } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - return $status; - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); return $status; } - // (b) Get a SHA-1 hash of the object wfSuppressWarnings(); $sha1Hash = sha1_file( $params['src'] ); wfRestoreWarnings(); if ( $sha1Hash === false ) { // source doesn't exist? - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + return $status; } $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 ); + $contentType = $this->getContentType( $params['dst'], null, $params['src'] ); - // (c) Actually store the object - try { - // Create a fresh CF_Object with no fields preloaded. - // We don't want to preserve headers, metadata, and such. - $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); - // The MD5 here will be checked within Swift against its own MD5. - $obj->set_etag( md5_file( $params['src'] ) ); - // Use the same content type as StreamFile for security - $obj->content_type = $this->getContentType( $params['dst'], null, $params['src'] ); - // Set any other custom headers if requested - if ( isset( $params['headers'] ) ) { - $obj->headers += $this->sanitizeHdrs( $params['headers'] ); - } - if ( !empty( $params['async'] ) ) { // deferred - wfSuppressWarnings(); - $fp = fopen( $params['src'], 'rb' ); - wfRestoreWarnings(); - if ( !$fp ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } else { - $op = $obj->write_async( $fp, filesize( $params['src'] ), true ); - $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op ); - $status->value->resourcesToClose[] = $fp; - $status->value->affectedObjects[] = $obj; - } - } else { // actually write the object in Swift - $obj->load_from_filename( $params['src'], true ); // calls $obj->write() - $this->purgeCDNCache( array( $obj ) ); - } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( BadContentTypeException $e ) { - $status->fatal( 'backend-fail-contenttype', $params['dst'] ); - } catch ( IOException $e ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + $handle = fopen( $params['src'], 'rb' ); + if ( $handle === false ) { // source doesn't exist? + $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + + return $status; } - return $status; - } + $reqs = array( array( + 'method' => 'PUT', + 'url' => array( $dstCont, $dstRel ), + 'headers' => array( + 'content-length' => filesize( $params['src'] ), + 'etag' => md5_file( $params['src'] ), + 'content-type' => $contentType, + 'x-object-meta-sha1base36' => $sha1Hash + ) + $this->sanitizeHdrs( $params ), + 'body' => $handle // resource + ) ); + + $be = $this; + $method = __METHOD__; + $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; + if ( $rcode === 201 ) { + // good + } elseif ( $rcode === 412 ) { + $status->fatal( 'backend-fail-contenttype', $params['dst'] ); + } else { + $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); + } + }; - /** - * @see SwiftFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) { - try { - $cfOp->getLastResponse(); - } catch ( BadContentTypeException $e ) { - $status->fatal( 'backend-fail-contenttype', $params['dst'] ); - } catch ( IOException $e ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); + if ( !empty( $params['async'] ) ) { // deferred + $status->value = $opHandle; + } else { // actually write the object in Swift + $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); } + + return $status; } protected function doCopyInternal( array $params ) { @@ -379,221 +327,202 @@ class SwiftFileBackend extends FileBackendStore { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; } list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); if ( $dstRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - return $status; - } - // (a) Check the source/destination containers and destination object - try { - $sContObj = $this->getContainer( $srcCont ); - $dContObj = $this->getContainer( $dstCont ); - } catch ( NoSuchContainerException $e ) { - if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } - return $status; - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); return $status; } - // (b) Actually copy the file to the destination - try { - $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - $hdrs = array(); // source file headers to override with new values - // Set any other custom headers if requested - if ( isset( $params['headers'] ) ) { - $hdrs += $this->sanitizeHdrs( $params['headers'] ); - } - if ( !empty( $params['async'] ) ) { // deferred - $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); - $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op ); - $status->value->affectedObjects[] = $dstObj; - } else { // actually write the object in Swift - $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); - $this->purgeCDNCache( array( $dstObj ) ); - } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( NoSuchObjectException $e ) { // source object does not exist - if ( empty( $params['ignoreMissingSource'] ) ) { + $reqs = array( array( + 'method' => 'PUT', + 'url' => array( $dstCont, $dstRel ), + 'headers' => array( + 'x-copy-from' => '/' . rawurlencode( $srcCont ) . + '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) ) + ) + $this->sanitizeHdrs( $params ), // extra headers merged into object + ) ); + + $be = $this; + $method = __METHOD__; + $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; + if ( $rcode === 201 ) { + // good + } elseif ( $rcode === 404 ) { $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } else { + $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); } - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + }; + + $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); + if ( !empty( $params['async'] ) ) { // deferred + $status->value = $opHandle; + } else { // actually write the object in Swift + $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); } return $status; } - /** - * @see SwiftFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) { - try { - $cfOp->getLastResponse(); - } catch ( NoSuchObjectException $e ) { // source object does not exist - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } - } - protected function doMoveInternal( array $params ) { $status = Status::newGood(); list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; } list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); if ( $dstRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; } - // (a) Check the source/destination containers and destination object - try { - $sContObj = $this->getContainer( $srcCont ); - $dContObj = $this->getContainer( $dstCont ); - } catch ( NoSuchContainerException $e ) { - if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); - } - return $status; - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); - return $status; + $reqs = array( + array( + 'method' => 'PUT', + 'url' => array( $dstCont, $dstRel ), + 'headers' => array( + 'x-copy-from' => '/' . rawurlencode( $srcCont ) . + '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) ) + ) + $this->sanitizeHdrs( $params ) // extra headers merged into object + ) + ); + if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) { + $reqs[] = array( + 'method' => 'DELETE', + 'url' => array( $srcCont, $srcRel ), + 'headers' => array() + ); } - // (b) Actually move the file to the destination - try { - $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - $hdrs = array(); // source file headers to override with new values - // Set any other custom headers if requested - if ( isset( $params['headers'] ) ) { - $hdrs += $this->sanitizeHdrs( $params['headers'] ); - } - if ( !empty( $params['async'] ) ) { // deferred - $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); - $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op ); - $status->value->affectedObjects[] = $srcObj; - $status->value->affectedObjects[] = $dstObj; - } else { // actually write the object in Swift - $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); - $this->purgeCDNCache( array( $srcObj ) ); - $this->purgeCDNCache( array( $dstObj ) ); - } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( NoSuchObjectException $e ) { // source object does not exist - if ( empty( $params['ignoreMissingSource'] ) ) { + $be = $this; + $method = __METHOD__; + $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; + if ( $request['method'] === 'PUT' && $rcode === 201 ) { + // good + } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) { + // good + } elseif ( $rcode === 404 ) { $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + } else { + $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); } - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + }; + + $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); + if ( !empty( $params['async'] ) ) { // deferred + $status->value = $opHandle; + } else { // actually move the object in Swift + $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); } return $status; } - /** - * @see SwiftFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) { - try { - $cfOp->getLastResponse(); - } catch ( NoSuchObjectException $e ) { // source object does not exist - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); - } - } - protected function doDeleteInternal( array $params ) { $status = Status::newGood(); list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; } - try { - $sContObj = $this->getContainer( $srcCont ); - $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - if ( !empty( $params['async'] ) ) { // deferred - $op = $sContObj->delete_object_async( $srcRel ); - $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op ); - $status->value->affectedObjects[] = $srcObj; - } else { // actually write the object in Swift - $sContObj->delete_object( $srcRel ); - $this->purgeCDNCache( array( $srcObj ) ); - } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( NoSuchContainerException $e ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - } - } catch ( NoSuchObjectException $e ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); + $reqs = array( array( + 'method' => 'DELETE', + 'url' => array( $srcCont, $srcRel ), + 'headers' => array() + ) ); + + $be = $this; + $method = __METHOD__; + $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; + if ( $rcode === 204 ) { + // good + } elseif ( $rcode === 404 ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } + } else { + $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); } - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + }; + + $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); + if ( !empty( $params['async'] ) ) { // deferred + $status->value = $opHandle; + } else { // actually delete the object in Swift + $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); } return $status; } - /** - * @see SwiftFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) { - try { - $cfOp->getLastResponse(); - } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - } catch ( NoSuchObjectException $e ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - } - } - } - protected function doDescribeInternal( array $params ) { $status = Status::newGood(); list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; } - try { - $sContObj = $this->getContainer( $srcCont ); - // Get the latest version of the current metadata - $srcObj = $sContObj->get_object( $srcRel, - $this->headersFromParams( array( 'latest' => true ) ) ); - // Merge in the metadata updates... - if ( isset( $params['headers'] ) ) { - $srcObj->headers = $this->sanitizeHdrs( $params['headers'] ) + $srcObj->headers; - } - $srcObj->sync_metadata(); // save to Swift - $this->purgeCDNCache( array( $srcObj ) ); - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-describe', $params['src'] ); - } catch ( NoSuchObjectException $e ) { + // Fetch the old object headers/metadata...this should be in stat cache by now + $stat = $this->getFileStat( array( 'src' => $params['src'], 'latest' => 1 ) ); + if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry + $stat = $this->doGetFileStat( array( 'src' => $params['src'], 'latest' => 1 ) ); + } + if ( !$stat ) { $status->fatal( 'backend-fail-describe', $params['src'] ); - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + + return $status; + } + + // POST clears prior headers, so we need to merge the changes in to the old ones + $metaHdrs = array(); + foreach ( $stat['xattr']['metadata'] as $name => $value ) { + $metaHdrs["x-object-meta-$name"] = $value; + } + $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers']; + + $reqs = array( array( + 'method' => 'POST', + 'url' => array( $srcCont, $srcRel ), + 'headers' => $metaHdrs + $customHdrs + ) ); + + $be = $this; + $method = __METHOD__; + $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; + if ( $rcode === 202 ) { + // good + } elseif ( $rcode === 404 ) { + $status->fatal( 'backend-fail-describe', $params['src'] ); + } else { + $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); + } + }; + + $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); + if ( !empty( $params['async'] ) ) { // deferred + $status->value = $opHandle; + } else { // actually change the object in Swift + $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) ); } return $status; @@ -603,110 +532,62 @@ class SwiftFileBackend extends FileBackendStore { $status = Status::newGood(); // (a) Check if container already exists - try { - $this->getContainer( $fullCont ); - // NoSuchContainerException not thrown: container must exist - return $status; // already exists - } catch ( NoSuchContainerException $e ) { - // NoSuchContainerException thrown: container does not exist - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + $stat = $this->getContainerStat( $fullCont ); + if ( is_array( $stat ) ) { + return $status; // already there + } elseif ( $stat === null ) { + $status->fatal( 'backend-fail-internal', $this->name ); + return $status; } - // (b) Create container as needed - try { - $contObj = $this->createContainer( $fullCont ); - if ( !empty( $params['noAccess'] ) ) { - // Make container private to end-users... - $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); - } else { - // Make container public to end-users... - $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); - } - if ( $this->swiftUseCDN ) { // Rackspace style CDN - $contObj->make_public( $this->swiftCDNExpiry ); - } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); - return $status; + // (b) Create container as needed with proper ACLs + if ( $stat === false ) { + $params['op'] = 'prepare'; + $status->merge( $this->createContainer( $fullCont, $params ) ); } return $status; } - /** - * @see FileBackendStore::doSecureInternal() - * @return Status - */ protected function doSecureInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); if ( empty( $params['noAccess'] ) ) { return $status; // nothing to do } - // Restrict container from end-users... - try { - // doPrepareInternal() should have been called, - // so the Swift container should already exist... - $contObj = $this->getContainer( $fullCont ); // normally a cache hit - // NoSuchContainerException not thrown: container must exist - + $stat = $this->getContainerStat( $fullCont ); + if ( is_array( $stat ) ) { // Make container private to end-users... $status->merge( $this->setContainerAccess( - $contObj, - array( $this->auth->username ), // read - array( $this->auth->username ) // write + $fullCont, + array( $this->swiftUser ), // read + array( $this->swiftUser ) // write ) ); - if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN - $contObj->make_private(); - } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + } elseif ( $stat === false ) { + $status->fatal( 'backend-fail-usable', $params['dir'] ); + } else { + $status->fatal( 'backend-fail-internal', $this->name ); } return $status; } - /** - * @see FileBackendStore::doPublishInternal() - * @return Status - */ protected function doPublishInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); - // Unrestrict container from end-users... - try { - // doPrepareInternal() should have been called, - // so the Swift container should already exist... - $contObj = $this->getContainer( $fullCont ); // normally a cache hit - // NoSuchContainerException not thrown: container must exist - + $stat = $this->getContainerStat( $fullCont ); + if ( is_array( $stat ) ) { // Make container public to end-users... - if ( $this->swiftAnonUser != '' ) { - $status->merge( $this->setContainerAccess( - $contObj, - array( $this->auth->username, $this->swiftAnonUser ), // read - array( $this->auth->username, $this->swiftAnonUser ) // write - ) ); - } else { - $status->merge( $this->setContainerAccess( - $contObj, - array( $this->auth->username, '.r:*' ), // read - array( $this->auth->username ) // write - ) ); - } - if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN - $contObj->make_public(); - } - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + $status->merge( $this->setContainerAccess( + $fullCont, + array( $this->swiftUser, '.r:*' ), // read + array( $this->swiftUser ) // write + ) ); + } elseif ( $stat === false ) { + $status->fatal( 'backend-fail-usable', $params['dir'] ); + } else { + $status->fatal( 'backend-fail-internal', $this->name ); } return $status; @@ -721,73 +602,74 @@ class SwiftFileBackend extends FileBackendStore { } // (a) Check the container - try { - $contObj = $this->getContainer( $fullCont, true ); - } catch ( NoSuchContainerException $e ) { + $stat = $this->getContainerStat( $fullCont, true ); + if ( $stat === false ) { return $status; // ok, nothing to do - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + } elseif ( !is_array( $stat ) ) { + $status->fatal( 'backend-fail-internal', $this->name ); + return $status; } // (b) Delete the container if empty - if ( $contObj->object_count == 0 ) { - try { - $this->deleteContainer( $fullCont ); - } catch ( NoSuchContainerException $e ) { - return $status; // race? - } catch ( NonEmptyContainerException $e ) { - return $status; // race? consistency delay? - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); - return $status; - } + if ( $stat['count'] == 0 ) { + $params['op'] = 'clean'; + $status->merge( $this->deleteContainer( $fullCont, $params ) ); } return $status; } protected function doGetFileStat( array $params ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - return false; // invalid storage path - } + $params = array( 'srcs' => array( $params['src'] ), 'concurrency' => 1 ) + $params; + unset( $params['src'] ); + $stats = $this->doGetFileStatMulti( $params ); + + return reset( $stats ); + } - $stat = false; + /** + * Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". + * Dates might also come in like "2013-05-11T07:37:27.678360" from Swift listings, + * missing the timezone suffix (though Ceph RGW does not appear to have this bug). + * + * @param string $ts + * @param int $format Output format (TS_* constant) + * @return string + * @throws FileBackendError + */ + protected function convertSwiftDate( $ts, $format = TS_MW ) { try { - $contObj = $this->getContainer( $srcCont ); - $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) ); - $this->addMissingMetadata( $srcObj, $params['src'] ); - $stat = array( - // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW - 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ), - 'size' => (int)$srcObj->content_length, - 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) - ); - } catch ( NoSuchContainerException $e ) { - } catch ( NoSuchObjectException $e ) { - } catch ( CloudFilesException $e ) { // some other exception? - $stat = null; - $this->handleException( $e, null, __METHOD__, $params ); - } + $timestamp = new MWTimestamp( $ts ); - return $stat; + return $timestamp->getTimestamp( $format ); + } catch ( MWException $e ) { + throw new FileBackendError( $e->getMessage() ); + } } /** * Fill in any missing object metadata and save it to Swift * - * @param CF_Object $obj + * @param array $objHdrs Object response headers * @param string $path Storage path to object - * @return bool Success - * @throws Exception cloudfiles exceptions + * @return array New headers */ - protected function addMissingMetadata( CF_Object $obj, $path ) { - if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) { - return true; // nothing to do + protected function addMissingMetadata( array $objHdrs, $path ) { + if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) { + return $objHdrs; // nothing to do } - wfProfileIn( __METHOD__ ); + + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING ); + + $auth = $this->getAuthentication(); + if ( !$auth ) { + $objHdrs['x-object-meta-sha1base36'] = false; + + return $objHdrs; // failed + } + $status = Status::newGood(); $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status ); if ( $status->isOK() ) { @@ -795,101 +677,79 @@ class SwiftFileBackend extends FileBackendStore { if ( $tmpFile ) { $hash = $tmpFile->getSha1Base36(); if ( $hash !== false ) { - $obj->setMetadataValues( array( 'Sha1base36' => $hash ) ); - $obj->sync_metadata(); // save to Swift - wfProfileOut( __METHOD__ ); - return true; // success + $objHdrs['x-object-meta-sha1base36'] = $hash; + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'POST', + 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), + 'headers' => $this->authTokenHeaders( $auth ) + $objHdrs + ) ); + if ( $rcode >= 200 && $rcode <= 299 ) { + return $objHdrs; // success + } } } } trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING ); - $obj->setMetadataValues( array( 'Sha1base36' => false ) ); - wfProfileOut( __METHOD__ ); - return false; // failed + $objHdrs['x-object-meta-sha1base36'] = false; + + return $objHdrs; // failed } protected function doGetFileContentsMulti( array $params ) { $contents = array(); + $auth = $this->getAuthentication(); + $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging // Blindly create tmp files and stream to them, catching any exception if the file does // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata(). - foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { - $cfOps = array(); // (path => CF_Async_Op) - - foreach ( $pathBatch as $path ) { // each path in this concurrent batch - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); - if ( $srcRel === null ) { - $contents[$path] = false; - continue; - } - $data = false; - try { - $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - // Create a new temporary memory file... - $handle = fopen( 'php://temp', 'wb' ); - if ( $handle ) { - $headers = $this->headersFromParams( $params ); - if ( count( $pathBatch ) > 1 ) { - $cfOps[$path] = $obj->stream_async( $handle, $headers ); - $cfOps[$path]->_file_handle = $handle; // close this later - } else { - $obj->stream( $handle, $headers ); - rewind( $handle ); // start from the beginning - $data = stream_get_contents( $handle ); - fclose( $handle ); - } - } else { - $data = false; - } - } catch ( NoSuchContainerException $e ) { - $data = false; - } catch ( NoSuchObjectException $e ) { - $data = false; - } catch ( CloudFilesException $e ) { // some other exception? - $data = false; - $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); - } - $contents[$path] = $data; - } + $reqs = array(); // (path => op) - $batch = new CF_Async_Op_Batch( $cfOps ); - $cfOps = $batch->execute(); - foreach ( $cfOps as $path => $cfOp ) { - try { - $cfOp->getLastResponse(); - rewind( $cfOp->_file_handle ); // start from the beginning - $contents[$path] = stream_get_contents( $cfOp->_file_handle ); - } catch ( NoSuchContainerException $e ) { - $contents[$path] = false; - } catch ( NoSuchObjectException $e ) { - $contents[$path] = false; - } catch ( CloudFilesException $e ) { // some other exception? - $contents[$path] = false; - $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); - } - fclose( $cfOp->_file_handle ); // close open handle + foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null || !$auth ) { + $contents[$path] = false; + continue; } + // Create a new temporary memory file... + $handle = fopen( 'php://temp', 'wb' ); + if ( $handle ) { + $reqs[$path] = array( + 'method' => 'GET', + 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), + 'headers' => $this->authTokenHeaders( $auth ) + + $this->headersFromParams( $params ), + 'stream' => $handle, + ); + } + $contents[$path] = false; + } + + $opts = array( 'maxConnsPerHost' => $params['concurrency'] ); + $reqs = $this->http->runMulti( $reqs, $opts ); + foreach ( $reqs as $path => $op ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response']; + if ( $rcode >= 200 && $rcode <= 299 ) { + rewind( $op['stream'] ); // start from the beginning + $contents[$path] = stream_get_contents( $op['stream'] ); + } elseif ( $rcode === 404 ) { + $contents[$path] = false; + } else { + $this->onError( null, __METHOD__, + array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc ); + } + fclose( $op['stream'] ); // close open handle } return $contents; } - /** - * @see FileBackendStore::doDirectoryExists() - * @return bool|null - */ protected function doDirectoryExists( $fullCont, $dir, array $params ) { - try { - $container = $this->getContainer( $fullCont ); - $prefix = ( $dir == '' ) ? null : "{$dir}/"; - return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 ); - } catch ( NoSuchContainerException $e ) { - return false; - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, - array( 'cont' => $fullCont, 'dir' => $dir ) ); + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix ); + if ( $status->isOk() ) { + return ( count( $status->value ) ) > 0; } return null; // error @@ -897,6 +757,9 @@ class SwiftFileBackend extends FileBackendStore { /** * @see FileBackendStore::getDirectoryListInternal() + * @param string $fullCont + * @param string $dir + * @param array $params * @return SwiftFileBackendDirList */ public function getDirectoryListInternal( $fullCont, $dir, array $params ) { @@ -905,6 +768,9 @@ class SwiftFileBackend extends FileBackendStore { /** * @see FileBackendStore::getFileListInternal() + * @param string $fullCont + * @param string $dir + * @param array $params * @return SwiftFileBackendFileList */ public function getFileListInternal( $fullCont, $dir, array $params ) { @@ -916,10 +782,10 @@ class SwiftFileBackend extends FileBackendStore { * * @param string $fullCont Resolved container name * @param string $dir Resolved storage directory with no trailing slash - * @param string|null $after Storage path of file to list items after - * @param integer $limit Max number of items to list + * @param string|null $after Resolved container relative path to list items after + * @param int $limit Max number of items to list * @param array $params Parameters for getDirectoryList() - * @return Array List of resolved paths of directories directly under $dir + * @return array List of container relative resolved paths of directories directly under $dir * @throws FileBackendError */ public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { @@ -929,156 +795,181 @@ class SwiftFileBackend extends FileBackendStore { } $section = new ProfileSection( __METHOD__ . '-' . $this->name ); - try { - $container = $this->getContainer( $fullCont ); - $prefix = ( $dir == '' ) ? null : "{$dir}/"; - // Non-recursive: only list dirs right under $dir - if ( !empty( $params['topOnly'] ) ) { - $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { // files and directories - if ( substr( $object, -1 ) === '/' ) { - $dirs[] = $object; // directories end in '/' - } + + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + // Non-recursive: only list dirs right under $dir + if ( !empty( $params['topOnly'] ) ) { + $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); + if ( !$status->isOk() ) { + return $dirs; // error + } + $objects = $status->value; + foreach ( $objects as $object ) { // files and directories + if ( substr( $object, -1 ) === '/' ) { + $dirs[] = $object; // directories end in '/' } + } + } else { // Recursive: list all dirs under $dir and its subdirs - } else { - // Get directory from last item of prior page - $lastDir = $this->getParentDir( $after ); // must be first page - $objects = $container->list_objects( $limit, $after, $prefix ); - foreach ( $objects as $object ) { // files - $objectDir = $this->getParentDir( $object ); // directory of object - if ( $objectDir !== false && $objectDir !== $dir ) { - // Swift stores paths in UTF-8, using binary sorting. - // See function "create_container_table" in common/db.py. - // If a directory is not "greater" than the last one, - // then it was already listed by the calling iterator. - if ( strcmp( $objectDir, $lastDir ) > 0 ) { - $pDir = $objectDir; - do { // add dir and all its parent dirs - $dirs[] = "{$pDir}/"; - $pDir = $this->getParentDir( $pDir ); - } while ( $pDir !== false // sanity - && strcmp( $pDir, $lastDir ) > 0 // not done already - && strlen( $pDir ) > strlen( $dir ) // within $dir - ); - } - $lastDir = $objectDir; + $getParentDir = function ( $path ) { + return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; + }; + + // Get directory from last item of prior page + $lastDir = $getParentDir( $after ); // must be first page + $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); + + if ( !$status->isOk() ) { + return $dirs; // error + } + + $objects = $status->value; + + foreach ( $objects as $object ) { // files + $objectDir = $getParentDir( $object ); // directory of object + + if ( $objectDir !== false && $objectDir !== $dir ) { + // Swift stores paths in UTF-8, using binary sorting. + // See function "create_container_table" in common/db.py. + // If a directory is not "greater" than the last one, + // then it was already listed by the calling iterator. + if ( strcmp( $objectDir, $lastDir ) > 0 ) { + $pDir = $objectDir; + do { // add dir and all its parent dirs + $dirs[] = "{$pDir}/"; + $pDir = $getParentDir( $pDir ); + } while ( $pDir !== false // sanity + && strcmp( $pDir, $lastDir ) > 0 // not done already + && strlen( $pDir ) > strlen( $dir ) // within $dir + ); } + $lastDir = $objectDir; } } - // Page on the unfiltered directory listing (what is returned may be filtered) - if ( count( $objects ) < $limit ) { - $after = INF; // avoid a second RTT - } else { - $after = end( $objects ); // update last item - } - } catch ( NoSuchContainerException $e ) { - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, - array( 'cont' => $fullCont, 'dir' => $dir ) ); - throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); + } + // Page on the unfiltered directory listing (what is returned may be filtered) + if ( count( $objects ) < $limit ) { + $after = INF; // avoid a second RTT + } else { + $after = end( $objects ); // update last item } return $dirs; } - protected function getParentDir( $path ) { - return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; - } - /** * Do not call this function outside of SwiftFileBackendFileList * * @param string $fullCont Resolved container name * @param string $dir Resolved storage directory with no trailing slash - * @param string|null $after Storage path of file to list items after - * @param integer $limit Max number of items to list + * @param string|null $after Resolved container relative path of file to list items after + * @param int $limit Max number of items to list * @param array $params Parameters for getDirectoryList() - * @return Array List of resolved paths of files under $dir + * @return array List of resolved container relative paths of files under $dir * @throws FileBackendError */ public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { - $files = array(); + $files = array(); // list of (path, stat array or null) entries if ( $after === INF ) { return $files; // nothing more } $section = new ProfileSection( __METHOD__ . '-' . $this->name ); - try { - $container = $this->getContainer( $fullCont ); - $prefix = ( $dir == '' ) ? null : "{$dir}/"; - // Non-recursive: only list files right under $dir - if ( !empty( $params['topOnly'] ) ) { // files and dirs - if ( !empty( $params['adviseStat'] ) ) { - $limit = min( $limit, self::CACHE_CHEAP_SIZE ); - // Note: get_objects() does not include directories - $objects = $this->loadObjectListing( $params, $dir, - $container->get_objects( $limit, $after, $prefix, null, '/' ) ); - $files = $objects; - } else { - $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { // files and directories - if ( substr( $object, -1 ) !== '/' ) { - $files[] = $object; // directories end in '/' - } - } - } - // Recursive: list all files under $dir and its subdirs - } else { // files - if ( !empty( $params['adviseStat'] ) ) { - $limit = min( $limit, self::CACHE_CHEAP_SIZE ); - $objects = $this->loadObjectListing( $params, $dir, - $container->get_objects( $limit, $after, $prefix ) ); - } else { - $objects = $container->list_objects( $limit, $after, $prefix ); - } - $files = $objects; + + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + // $objects will contain a list of unfiltered names or CF_Object items + // Non-recursive: only list files right under $dir + if ( !empty( $params['topOnly'] ) ) { + if ( !empty( $params['adviseStat'] ) ) { + $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' ); + } else { + $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); } - // Page on the unfiltered object listing (what is returned may be filtered) - if ( count( $objects ) < $limit ) { - $after = INF; // avoid a second RTT + } else { + // Recursive: list all files under $dir and its subdirs + if ( !empty( $params['adviseStat'] ) ) { + $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix ); } else { - $after = end( $objects ); // update last item + $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); } - } catch ( NoSuchContainerException $e ) { - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, - array( 'cont' => $fullCont, 'dir' => $dir ) ); - throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); + } + + // Reformat this list into a list of (name, stat array or null) entries + if ( !$status->isOk() ) { + return $files; // error + } + + $objects = $status->value; + $files = $this->buildFileObjectListing( $params, $dir, $objects ); + + // Page on the unfiltered object listing (what is returned may be filtered) + if ( count( $objects ) < $limit ) { + $after = INF; // avoid a second RTT + } else { + $after = end( $objects ); // update last item + $after = is_object( $after ) ? $after->name : $after; } return $files; } /** - * Load a list of objects that belong under $dir into stat cache - * and return a list of the names of the objects in the same order. + * Build a list of file objects, filtering out any directories + * and extracting any stat info if provided in $objects (for CF_Objects) * * @param array $params Parameters for getDirectoryList() * @param string $dir Resolved container directory path - * @param array $cfObjects List of CF_Object items - * @return array List of object names + * @param array $objects List of CF_Object items or object names + * @return array List of (names,stat array or null) entries */ - private function loadObjectListing( array $params, $dir, array $cfObjects ) { + private function buildFileObjectListing( array $params, $dir, array $objects ) { $names = array(); - $storageDir = rtrim( $params['dir'], '/' ); - $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/" - // Iterate over the list *backwards* as this primes the stat cache, which is LRU. - // If this fills the cache and the caller stats an uncached file before stating - // the ones on the listing, there would be zero cache hits if this went forwards. - for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) { - $object = current( $cfObjects ); - $path = "{$storageDir}/" . substr( $object->name, $suffixStart ); - $val = array( - // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW - 'mtime' => wfTimestamp( TS_MW, $object->last_modified ), - 'size' => (int)$object->content_length, - 'latest' => false // eventually consistent - ); - $this->cheapCache->set( $path, 'stat', $val ); - $names[] = $object->name; + foreach ( $objects as $object ) { + if ( is_object( $object ) ) { + if ( isset( $object->subdir ) || !isset( $object->name ) ) { + continue; // virtual directory entry; ignore + } + $stat = array( + // Convert various random Swift dates to TS_MW + 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ), + 'size' => (int)$object->bytes, + // Note: manifiest ETags are not an MD5 of the file + 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null, + 'latest' => false // eventually consistent + ); + $names[] = array( $object->name, $stat ); + } elseif ( substr( $object, -1 ) !== '/' ) { + // Omit directories, which end in '/' in listings + $names[] = array( $object, null ); + } + } + + return $names; + } + + /** + * Do not call this function outside of SwiftFileBackendFileList + * + * @param string $path Storage path + * @param array $val Stat value + */ + public function loadListingStatInternal( $path, array $val ) { + $this->cheapCache->set( $path, 'stat', $val ); + } + + protected function doGetFileXAttributes( array $params ) { + $stat = $this->getFileStat( $params ); + if ( $stat ) { + if ( !isset( $stat['xattr'] ) ) { + // Stat entries filled by file listings don't include metadata/headers + $this->clearCache( array( $params['src'] ) ); + $stat = $this->getFileStat( $params ); + } + + return $stat['xattr']; + } else { + return false; } - return array_reverse( $names ); // keep the paths in original order } protected function doGetFileSha1base36( array $params ) { @@ -1089,6 +980,7 @@ class SwiftFileBackend extends FileBackendStore { $this->clearCache( array( $params['src'] ) ); $stat = $this->getFileStat( $params ); } + return $stat['sha1']; } else { return false; @@ -1103,24 +995,29 @@ class SwiftFileBackend extends FileBackendStore { $status->fatal( 'backend-fail-invalidpath', $params['src'] ); } - try { - $cont = $this->getContainer( $srcCont ); - } catch ( NoSuchContainerException $e ) { + $auth = $this->getAuthentication(); + if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) { $status->fatal( 'backend-fail-stream', $params['src'] ); - return $status; - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + return $status; } - try { - $output = fopen( 'php://output', 'wb' ); - $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD - $obj->stream( $output, $this->headersFromParams( $params ) ); - } catch ( NoSuchObjectException $e ) { + $handle = fopen( 'php://output', 'wb' ); + + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'GET', + 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), + 'headers' => $this->authTokenHeaders( $auth ) + + $this->headersFromParams( $params ), + 'stream' => $handle, + ) ); + + if ( $rcode >= 200 && $rcode <= 299 ) { + // good + } elseif ( $rcode === 404 ) { $status->fatal( 'backend-fail-stream', $params['src'] ); - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + } else { + $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); } return $status; @@ -1129,66 +1026,60 @@ class SwiftFileBackend extends FileBackendStore { protected function doGetLocalCopyMulti( array $params ) { $tmpFiles = array(); + $auth = $this->getAuthentication(); + $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging // Blindly create tmp files and stream to them, catching any exception if the file does // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). - foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { - $cfOps = array(); // (path => CF_Async_Op) + $reqs = array(); // (path => op) - foreach ( $pathBatch as $path ) { // each path in this concurrent batch - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); - if ( $srcRel === null ) { - $tmpFiles[$path] = null; - continue; - } - $tmpFile = null; - try { - $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - // Get source file extension - $ext = FileBackend::extensionFromPath( $path ); - // Create a new temporary file... - $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); - if ( $tmpFile ) { - $handle = fopen( $tmpFile->getPath(), 'wb' ); - if ( $handle ) { - $headers = $this->headersFromParams( $params ); - if ( count( $pathBatch ) > 1 ) { - $cfOps[$path] = $obj->stream_async( $handle, $headers ); - $cfOps[$path]->_file_handle = $handle; // close this later - } else { - $obj->stream( $handle, $headers ); - fclose( $handle ); - } - } else { - $tmpFile = null; - } - } - } catch ( NoSuchContainerException $e ) { - $tmpFile = null; - } catch ( NoSuchObjectException $e ) { - $tmpFile = null; - } catch ( CloudFilesException $e ) { // some other exception? + foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null || !$auth ) { + $tmpFiles[$path] = null; + continue; + } + // Get source file extension + $ext = FileBackend::extensionFromPath( $path ); + // Create a new temporary file... + $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); + if ( $tmpFile ) { + $handle = fopen( $tmpFile->getPath(), 'wb' ); + if ( $handle ) { + $reqs[$path] = array( + 'method' => 'GET', + 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), + 'headers' => $this->authTokenHeaders( $auth ) + + $this->headersFromParams( $params ), + 'stream' => $handle, + ); + } else { $tmpFile = null; - $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); } - $tmpFiles[$path] = $tmpFile; } - - $batch = new CF_Async_Op_Batch( $cfOps ); - $cfOps = $batch->execute(); - foreach ( $cfOps as $path => $cfOp ) { - try { - $cfOp->getLastResponse(); - } catch ( NoSuchContainerException $e ) { + $tmpFiles[$path] = $tmpFile; + } + + $opts = array( 'maxConnsPerHost' => $params['concurrency'] ); + $reqs = $this->http->runMulti( $reqs, $opts ); + foreach ( $reqs as $path => $op ) { + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response']; + fclose( $op['stream'] ); // close open handle + if ( $rcode >= 200 && $rcode <= 299 ) { + $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0; + // Double check that the disk is not full/broken + if ( $size != $rhdrs['content-length'] ) { $tmpFiles[$path] = null; - } catch ( NoSuchObjectException $e ) { - $tmpFiles[$path] = null; - } catch ( CloudFilesException $e ) { // some other exception? - $tmpFiles[$path] = null; - $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + $rerr = "Got {$size}/{$rhdrs['content-length']} bytes"; + $this->onError( null, __METHOD__, + array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc ); } - fclose( $cfOp->_file_handle ); // close open handle + } elseif ( $rcode === 404 ) { + $tmpFiles[$path] = false; + } else { + $tmpFiles[$path] = null; + $this->onError( null, __METHOD__, + array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc ); } } @@ -1197,46 +1088,55 @@ class SwiftFileBackend extends FileBackendStore { public function getFileHttpUrl( array $params ) { if ( $this->swiftTempUrlKey != '' || - ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) ) - { + ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) + ) { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { return null; // invalid path } - try { - $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400; - $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - if ( $this->swiftTempUrlKey != '' ) { - return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" ); - } else { // give S3 API URL for rgw - $expires = time() + $ttl; - // Path for signature starts with the bucket - $spath = '/' . rawurlencode( $srcCont ) . '/' . - str_replace( '%2F', '/', rawurlencode( $srcRel ) ); - // Calculate the hash - $signature = base64_encode( hash_hmac( - 'sha1', - "GET\n\n\n{$expires}\n{$spath}", - $this->rgwS3SecretKey, - true // raw - ) ); - // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. - // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. - return wfAppendQuery( - str_replace( '/swift/v1', '', // S3 API is the rgw default - $sContObj->cfs_http->getStorageUrl() . $spath ), - array( - 'Signature' => $signature, - 'Expires' => $expires, - 'AWSAccessKeyId' => $this->rgwS3AccessKey ) - ); - } - } catch ( NoSuchContainerException $e ) { - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, $params ); + + $auth = $this->getAuthentication(); + if ( !$auth ) { + return null; + } + + $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400; + $expires = time() + $ttl; + + if ( $this->swiftTempUrlKey != '' ) { + $url = $this->storageUrl( $auth, $srcCont, $srcRel ); + // Swift wants the signature based on the unencoded object name + $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH ); + $signature = hash_hmac( 'sha1', + "GET\n{$expires}\n{$contPath}/{$srcRel}", + $this->swiftTempUrlKey + ); + + return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}"; + } else { // give S3 API URL for rgw + // Path for signature starts with the bucket + $spath = '/' . rawurlencode( $srcCont ) . '/' . + str_replace( '%2F', '/', rawurlencode( $srcRel ) ); + // Calculate the hash + $signature = base64_encode( hash_hmac( + 'sha1', + "GET\n\n\n{$expires}\n{$spath}", + $this->rgwS3SecretKey, + true // raw + ) ); + // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. + // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. + return wfAppendQuery( + str_replace( '/swift/v1', '', // S3 API is the rgw default + $this->storageUrl( $auth ) . $spath ), + array( + 'Signature' => $signature, + 'Expires' => $expires, + 'AWSAccessKeyId' => $this->rgwS3AccessKey ) + ); } } + return null; } @@ -1250,37 +1150,61 @@ class SwiftFileBackend extends FileBackendStore { * $params is currently only checked for a 'latest' flag. * * @param array $params - * @return Array + * @return array */ protected function headersFromParams( array $params ) { $hdrs = array(); if ( !empty( $params['latest'] ) ) { - $hdrs[] = 'X-Newest: true'; + $hdrs['x-newest'] = 'true'; } + return $hdrs; } protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { $statuses = array(); - $cfOps = array(); // list of CF_Async_Op objects + $auth = $this->getAuthentication(); + if ( !$auth ) { + foreach ( $fileOpHandles as $index => $fileOpHandle ) { + $statuses[$index] = Status::newFatal( 'backend-fail-connect', $this->name ); + } + + return $statuses; + } + + // Split the HTTP requests into stages that can be done concurrently + $httpReqsByStage = array(); // map of (stage => index => HTTP request) foreach ( $fileOpHandles as $index => $fileOpHandle ) { - $cfOps[$index] = $fileOpHandle->cfOp; - } - $batch = new CF_Async_Op_Batch( $cfOps ); - - $cfOps = $batch->execute(); - foreach ( $cfOps as $index => $cfOp ) { - $status = Status::newGood(); - $function = '_getResponse' . $fileOpHandles[$index]->call; - try { // catch exceptions; update status - $this->$function( $cfOp, $status, $fileOpHandles[$index]->params ); - $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects ); - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, - __CLASS__ . ":$function", $fileOpHandles[$index]->params ); + $reqs = $fileOpHandle->httpOp; + // Convert the 'url' parameter to an actual URL using $auth + foreach ( $reqs as $stage => &$req ) { + list( $container, $relPath ) = $req['url']; + $req['url'] = $this->storageUrl( $auth, $container, $relPath ); + $req['headers'] = isset( $req['headers'] ) ? $req['headers'] : array(); + $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers']; + $httpReqsByStage[$stage][$index] = $req; + } + $statuses[$index] = Status::newGood(); + } + + // Run all requests for the first stage, then the next, and so on + $reqCount = count( $httpReqsByStage ); + for ( $stage = 0; $stage < $reqCount; ++$stage ) { + $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] ); + foreach ( $httpReqs as $index => $httpReq ) { + // Run the callback for each request of this operation + $callback = $fileOpHandles[$index]->callback; + call_user_func_array( $callback, array( $httpReq, $statuses[$index] ) ); + // On failure, abort all remaining requests for this operation + // (e.g. abort the DELETE request if the COPY request fails for a move) + if ( !$statuses[$index]->isOK() ) { + $stages = count( $fileOpHandles[$index]->httpOp ); + for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) { + unset( $httpReqsByStage[$s][$index] ); + } + } } - $statuses[$index] = $status; } return $statuses; @@ -1289,7 +1213,13 @@ class SwiftFileBackend extends FileBackendStore { /** * Set read/write permissions for a Swift container. * - * $readGrps is a list of the possible criteria for a request to have + * @see http://swift.openstack.org/misc.html#acls + * + * In general, we don't allow listings to end-users. It's not useful, isn't well-defined + * (lists are truncated to 10000 item with no way to page), and is just a performance risk. + * + * @param string $container Resolved Swift container + * @param array $readGrps List of the possible criteria for a request to have * access to read a container. Each item is one of the following formats: * - account:user : Grants access if the request is by the given user * - ".r:<regex>" : Grants access if the request is from a referrer host that @@ -1297,228 +1227,438 @@ class SwiftFileBackend extends FileBackendStore { * Setting this to '*' effectively makes a container public. * -".rlistings:<regex>" : Grants access if the request is from a referrer host that * matches the expression and the request is for a listing. - * - * $writeGrps is a list of the possible criteria for a request to have + * @param array $writeGrps A list of the possible criteria for a request to have * access to write to a container. Each item is of the following format: * - account:user : Grants access if the request is by the given user - * - * @see http://swift.openstack.org/misc.html#acls - * - * In general, we don't allow listings to end-users. It's not useful, isn't well-defined - * (lists are truncated to 10000 item with no way to page), and is just a performance risk. - * - * @param CF_Container $contObj Swift container - * @param array $readGrps List of read access routes - * @param array $writeGrps List of write access routes * @return Status */ - protected function setContainerAccess( - CF_Container $contObj, array $readGrps, array $writeGrps - ) { - $creds = $contObj->cfs_auth->export_credentials(); + protected function setContainerAccess( $container, array $readGrps, array $writeGrps ) { + $status = Status::newGood(); + $auth = $this->getAuthentication(); - $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name ); + if ( !$auth ) { + $status->fatal( 'backend-fail-connect', $this->name ); - // Note: 10 second timeout consistent with php-cloudfiles - $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) ); - $req->setHeader( 'X-Auth-Token', $creds['auth_token'] ); - $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) ); - $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) ); + return $status; + } + + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'POST', + 'url' => $this->storageUrl( $auth, $container ), + 'headers' => $this->authTokenHeaders( $auth ) + array( + 'x-container-read' => implode( ',', $readGrps ), + 'x-container-write' => implode( ',', $writeGrps ) + ) + ) ); + + if ( $rcode != 204 && $rcode !== 202 ) { + $status->fatal( 'backend-fail-internal', $this->name ); + } - return $req->execute(); // should return 204 + return $status; } /** - * Purge the CDN cache of affected objects if CDN caching is enabled. - * This is for Rackspace/Akamai CDNs. + * Get a Swift container stat array, possibly from process cache. + * Use $reCache if the file count or byte count is needed. * - * @param array $objects List of CF_Object items - * @return void + * @param string $container Container name + * @param bool $bypassCache Bypass all caches and load from Swift + * @return array|bool|null False on 404, null on failure */ - public function purgeCDNCache( array $objects ) { - if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) { - foreach ( $objects as $object ) { - try { - $object->purge_from_cdn(); - } catch ( CDNNotEnabledException $e ) { - // CDN not enabled; nothing to see here - } catch ( CloudFilesException $e ) { - $this->handleException( $e, null, __METHOD__, - array( 'cont' => $object->container->name, 'obj' => $object->name ) ); + protected function getContainerStat( $container, $bypassCache = false ) { + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); + + if ( $bypassCache ) { // purge cache + $this->containerStatCache->clear( $container ); + } elseif ( !$this->containerStatCache->has( $container, 'stat' ) ) { + $this->primeContainerCache( array( $container ) ); // check persistent cache + } + if ( !$this->containerStatCache->has( $container, 'stat' ) ) { + $auth = $this->getAuthentication(); + if ( !$auth ) { + return null; + } + + wfProfileIn( __METHOD__ . "-{$this->name}-miss" ); + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'HEAD', + 'url' => $this->storageUrl( $auth, $container ), + 'headers' => $this->authTokenHeaders( $auth ) + ) ); + wfProfileOut( __METHOD__ . "-{$this->name}-miss" ); + + if ( $rcode === 204 ) { + $stat = array( + 'count' => $rhdrs['x-container-object-count'], + 'bytes' => $rhdrs['x-container-bytes-used'] + ); + if ( $bypassCache ) { + return $stat; + } else { + $this->containerStatCache->set( $container, 'stat', $stat ); // cache it + $this->setContainerCache( $container, $stat ); // update persistent cache } + } elseif ( $rcode === 404 ) { + return false; + } else { + $this->onError( null, __METHOD__, + array( 'cont' => $container ), $rerr, $rcode, $rdesc ); + + return null; } } + + return $this->containerStatCache->get( $container, 'stat' ); } /** - * Get an authenticated connection handle to the Swift proxy + * Create a Swift container * - * @throws CloudFilesException - * @throws CloudFilesException|Exception - * @return CF_Connection|bool False on failure + * @param string $container Container name + * @param array $params + * @return Status */ - protected function getConnection() { - if ( $this->connException instanceof CloudFilesException ) { - if ( ( time() - $this->connErrorTime ) < 60 ) { - throw $this->connException; // failed last attempt; don't bother - } else { // actually retry this time - $this->connException = null; - $this->connErrorTime = 0; - } + protected function createContainer( $container, array $params ) { + $status = Status::newGood(); + + $auth = $this->getAuthentication(); + if ( !$auth ) { + $status->fatal( 'backend-fail-connect', $this->name ); + + return $status; } - // Session keys expire after a while, so we renew them periodically - $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL ); - // Authenticate with proxy and get a session key... - if ( !$this->conn || $reAuth ) { - $this->sessionStarted = 0; - $this->connContainerCache->clear(); - $cacheKey = $this->getCredsCacheKey( $this->auth->username ); - $creds = $this->srvCache->get( $cacheKey ); // credentials - if ( is_array( $creds ) ) { // cache hit - $this->auth->load_cached_credentials( - $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] ); - $this->sessionStarted = time() - ceil( $this->authTTL / 2 ); // skew for worst case - } else { // cache miss - try { - $this->auth->authenticate(); - $creds = $this->auth->export_credentials(); - $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL / 2 ) ); // cache - $this->sessionStarted = time(); - } catch ( CloudFilesException $e ) { - $this->connException = $e; // don't keep re-trying - $this->connErrorTime = time(); - throw $e; // throw it back - } - } - if ( $this->conn ) { // re-authorizing? - $this->conn->close(); // close active cURL handles in CF_Http object - } - $this->conn = new CF_Connection( $this->auth ); + + // @see SwiftFileBackend::setContainerAccess() + if ( empty( $params['noAccess'] ) ) { + $readGrps = array( '.r:*', $this->swiftUser ); // public + } else { + $readGrps = array( $this->swiftUser ); // private + } + $writeGrps = array( $this->swiftUser ); // sanity + + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'PUT', + 'url' => $this->storageUrl( $auth, $container ), + 'headers' => $this->authTokenHeaders( $auth ) + array( + 'x-container-read' => implode( ',', $readGrps ), + 'x-container-write' => implode( ',', $writeGrps ) + ) + ) ); + + if ( $rcode === 201 ) { // new + // good + } elseif ( $rcode === 202 ) { // already there + // this shouldn't really happen, but is OK + } else { + $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); } - return $this->conn; + + return $status; } /** - * Close the connection to the Swift proxy + * Delete a Swift container * - * @return void + * @param string $container Container name + * @param array $params + * @return Status */ - protected function closeConnection() { - if ( $this->conn ) { - $this->conn->close(); // close active cURL handles in CF_Http object - $this->conn = null; - $this->sessionStarted = 0; - $this->connContainerCache->clear(); + protected function deleteContainer( $container, array $params ) { + $status = Status::newGood(); + + $auth = $this->getAuthentication(); + if ( !$auth ) { + $status->fatal( 'backend-fail-connect', $this->name ); + + return $status; + } + + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'DELETE', + 'url' => $this->storageUrl( $auth, $container ), + 'headers' => $this->authTokenHeaders( $auth ) + ) ); + + if ( $rcode >= 200 && $rcode <= 299 ) { // deleted + $this->containerStatCache->clear( $container ); // purge + } elseif ( $rcode === 404 ) { // not there + // this shouldn't really happen, but is OK + } elseif ( $rcode === 409 ) { // not empty + $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race? + } else { + $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); } + + return $status; } /** - * Get the cache key for a container + * Get a list of objects under a container. + * Either just the names or a list of stdClass objects with details can be returned. * - * @param string $username - * @return string + * @param string $fullCont + * @param string $type ('info' for a list of object detail maps, 'names' for names only) + * @param int $limit + * @param string|null $after + * @param string|null $prefix + * @param string|null $delim + * @return Status With the list as value */ - private function getCredsCacheKey( $username ) { - return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username ); + private function objectListing( + $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null + ) { + $status = Status::newGood(); + + $auth = $this->getAuthentication(); + if ( !$auth ) { + $status->fatal( 'backend-fail-connect', $this->name ); + + return $status; + } + + $query = array( 'limit' => $limit ); + if ( $type === 'info' ) { + $query['format'] = 'json'; + } + if ( $after !== null ) { + $query['marker'] = $after; + } + if ( $prefix !== null ) { + $query['prefix'] = $prefix; + } + if ( $delim !== null ) { + $query['delimiter'] = $delim; + } + + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'GET', + 'url' => $this->storageUrl( $auth, $fullCont ), + 'query' => $query, + 'headers' => $this->authTokenHeaders( $auth ) + ) ); + + $params = array( 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ); + if ( $rcode === 200 ) { // good + if ( $type === 'info' ) { + $status->value = FormatJson::decode( trim( $rbody ) ); + } else { + $status->value = explode( "\n", trim( $rbody ) ); + } + } elseif ( $rcode === 204 ) { + $status->value = array(); // empty container + } elseif ( $rcode === 404 ) { + $status->value = array(); // no container + } else { + $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); + } + + return $status; + } + + protected function doPrimeContainerCache( array $containerInfo ) { + foreach ( $containerInfo as $container => $info ) { + $this->containerStatCache->set( $container, 'stat', $info ); + } + } + + protected function doGetFileStatMulti( array $params ) { + $stats = array(); + + $auth = $this->getAuthentication(); + + $reqs = array(); + foreach ( $params['srcs'] as $path ) { + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null ) { + $stats[$path] = false; + continue; // invalid storage path + } elseif ( !$auth ) { + $stats[$path] = null; + continue; + } + + // (a) Check the container + $cstat = $this->getContainerStat( $srcCont ); + if ( $cstat === false ) { + $stats[$path] = false; + continue; // ok, nothing to do + } elseif ( !is_array( $cstat ) ) { + $stats[$path] = null; + continue; + } + + $reqs[$path] = array( + 'method' => 'HEAD', + 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), + 'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params ) + ); + } + + $opts = array( 'maxConnsPerHost' => $params['concurrency'] ); + $reqs = $this->http->runMulti( $reqs, $opts ); + + foreach ( $params['srcs'] as $path ) { + if ( array_key_exists( $path, $stats ) ) { + continue; // some sort of failure above + } + // (b) Check the file + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response']; + if ( $rcode === 200 || $rcode === 204 ) { + // Update the object if it is missing some headers + $rhdrs = $this->addMissingMetadata( $rhdrs, $path ); + // Fetch all of the custom metadata headers + $metadata = array(); + foreach ( $rhdrs as $name => $value ) { + if ( strpos( $name, 'x-object-meta-' ) === 0 ) { + $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value; + } + } + // Fetch all of the custom raw HTTP headers + $headers = $this->sanitizeHdrs( array( 'headers' => $rhdrs ) ); + $stat = array( + // Convert various random Swift dates to TS_MW + 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ), + // Empty objects actually return no content-length header in Ceph + 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0, + 'sha1' => $rhdrs['x-object-meta-sha1base36'], + // Note: manifiest ETags are not an MD5 of the file + 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null, + 'xattr' => array( 'metadata' => $metadata, 'headers' => $headers ) + ); + if ( $this->isRGW ) { + $stat['latest'] = true; // strong consistency + } + } elseif ( $rcode === 404 ) { + $stat = false; + } else { + $stat = null; + $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc ); + } + $stats[$path] = $stat; + } + + return $stats; } /** - * Get a Swift container object, possibly from process cache. - * Use $reCache if the file count or byte count is needed. - * - * @param string $container Container name - * @param bool $bypassCache Bypass all caches and load from Swift - * @return CF_Container - * @throws CloudFilesException + * @return array|null Credential map */ - protected function getContainer( $container, $bypassCache = false ) { - $conn = $this->getConnection(); // Swift proxy connection - if ( $bypassCache ) { // purge cache - $this->connContainerCache->clear( $container ); - } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) { - $this->primeContainerCache( array( $container ) ); // check persistent cache + protected function getAuthentication() { + if ( $this->authErrorTimestamp !== null ) { + if ( ( time() - $this->authErrorTimestamp ) < 60 ) { + return null; // failed last attempt; don't bother + } else { // actually retry this time + $this->authErrorTimestamp = null; + } } - if ( !$this->connContainerCache->has( $container, 'obj' ) ) { - $contObj = $conn->get_container( $container ); - // NoSuchContainerException not thrown: container must exist - $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it - if ( !$bypassCache ) { - $this->setContainerCache( $container, // update persistent cache - array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count ) - ); + // Session keys expire after a while, so we renew them periodically + $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL ); + // Authenticate with proxy and get a session key... + if ( !$this->authCreds || $reAuth ) { + $this->authSessionTimestamp = 0; + $cacheKey = $this->getCredsCacheKey( $this->swiftUser ); + $creds = $this->srvCache->get( $cacheKey ); // credentials + // Try to use the credential cache + if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) { + $this->authCreds = $creds; + // Skew the timestamp for worst case to avoid using stale credentials + $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 ); + } else { // cache miss + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( + 'method' => 'GET', + 'url' => "{$this->swiftAuthUrl}/v1.0", + 'headers' => array( + 'x-auth-user' => $this->swiftUser, + 'x-auth-key' => $this->swiftKey + ) + ) ); + + if ( $rcode >= 200 && $rcode <= 299 ) { // OK + $this->authCreds = array( + 'auth_token' => $rhdrs['x-auth-token'], + 'storage_url' => $rhdrs['x-storage-url'] + ); + $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) ); + $this->authSessionTimestamp = time(); + } elseif ( $rcode === 401 ) { + $this->onError( null, __METHOD__, array(), "Authentication failed.", $rcode ); + $this->authErrorTimestamp = time(); + + return null; + } else { + $this->onError( null, __METHOD__, array(), "HTTP return code: $rcode", $rcode ); + $this->authErrorTimestamp = time(); + + return null; + } + } + // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>") + if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) { + $this->isRGW = true; // take advantage of strong consistency } } - return $this->connContainerCache->get( $container, 'obj' ); + + return $this->authCreds; } /** - * Create a Swift container - * - * @param string $container Container name - * @return CF_Container - * @throws CloudFilesException + * @param array $creds From getAuthentication() + * @param string $container + * @param string $object + * @return array */ - protected function createContainer( $container ) { - $conn = $this->getConnection(); // Swift proxy connection - $contObj = $conn->create_container( $container ); - $this->connContainerCache->set( $container, 'obj', $contObj ); // cache - return $contObj; + protected function storageUrl( array $creds, $container = null, $object = null ) { + $parts = array( $creds['storage_url'] ); + if ( strlen( $container ) ) { + $parts[] = rawurlencode( $container ); + } + if ( strlen( $object ) ) { + $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) ); + } + + return implode( '/', $parts ); } /** - * Delete a Swift container - * - * @param string $container Container name - * @return void - * @throws CloudFilesException + * @param array $creds From getAuthentication() + * @return array */ - protected function deleteContainer( $container ) { - $conn = $this->getConnection(); // Swift proxy connection - $this->connContainerCache->clear( $container ); // purge - $conn->delete_container( $container ); + protected function authTokenHeaders( array $creds ) { + return array( 'x-auth-token' => $creds['auth_token'] ); } - protected function doPrimeContainerCache( array $containerInfo ) { - try { - $conn = $this->getConnection(); // Swift proxy connection - foreach ( $containerInfo as $container => $info ) { - $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http, - $container, $info['count'], $info['bytes'] ); - $this->connContainerCache->set( $container, 'obj', $contObj ); - } - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, array() ); - } + /** + * Get the cache key for a container + * + * @param string $username + * @return string + */ + private function getCredsCacheKey( $username ) { + return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl ); } /** * Log an unexpected exception for this backend. * This also sets the Status object to have a fatal error. * - * @param Exception $e - * @param Status $status|null + * @param Status|null $status * @param string $func * @param array $params - * @return void + * @param string $err Error string + * @param int $code HTTP status + * @param string $desc HTTP status description */ - protected function handleException( Exception $e, $status, $func, array $params ) { + public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) { if ( $status instanceof Status ) { - if ( $e instanceof AuthenticationException ) { - $status->fatal( 'backend-fail-connect', $this->name ); - } else { - $status->fatal( 'backend-fail-internal', $this->name ); - } - } - if ( $e->getMessage() ) { - trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING ); + $status->fatal( 'backend-fail-internal', $this->name ); } - if ( $e instanceof InvalidResponseException ) { // possibly a stale token - $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) ); - $this->closeConnection(); // force a re-connect and re-auth next time + if ( $code == 401 ) { // possibly a stale token + $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) ); } wfDebugLog( 'SwiftBackend', - get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . - ( $e->getMessage() ? ": {$e->getMessage()}" : "" ) + "HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . + ( $err ? ": $err" : "" ) ); } } @@ -1527,24 +1667,20 @@ class SwiftFileBackend extends FileBackendStore { * @see FileBackendStoreOpHandle */ class SwiftFileOpHandle extends FileBackendStoreOpHandle { - /** @var CF_Async_Op */ - public $cfOp; - /** @var Array */ - public $affectedObjects = array(); + /** @var array List of Requests for MultiHttpClient */ + public $httpOp; + /** @var Closure */ + public $callback; /** * @param SwiftFileBackend $backend - * @param array $params - * @param string $call - * @param CF_Async_Op $cfOp + * @param Closure $callback Function that takes (HTTP request array, status) + * @param array $httpOp MultiHttpClient op */ - public function __construct( - SwiftFileBackend $backend, array $params, $call, CF_Async_Op $cfOp - ) { + public function __construct( SwiftFileBackend $backend, Closure $callback, array $httpOp ) { $this->backend = $backend; - $this->params = $params; - $this->call = $call; - $this->cfOp = $cfOp; + $this->callback = $callback; + $this->httpOp = $httpOp; } } @@ -1556,18 +1692,29 @@ class SwiftFileOpHandle extends FileBackendStoreOpHandle { * @ingroup FileBackend */ abstract class SwiftFileBackendList implements Iterator { - /** @var Array */ + /** @var array List of path or (path,stat array) entries */ protected $bufferIter = array(); - protected $bufferAfter = null; // string; list items *after* this path - protected $pos = 0; // integer - /** @var Array */ + + /** @var string List items *after* this path */ + protected $bufferAfter = null; + + /** @var int */ + protected $pos = 0; + + /** @var array */ protected $params = array(); /** @var SwiftFileBackend */ protected $backend; - protected $container; // string; container name - protected $dir; // string; storage directory - protected $suffixStart; // integer + + /** @var string Container name */ + protected $container; + + /** @var string Storage directory */ + protected $dir; + + /** @var int */ + protected $suffixStart; const PAGE_SIZE = 9000; // file listing buffer size @@ -1594,7 +1741,7 @@ abstract class SwiftFileBackendList implements Iterator { /** * @see Iterator::key() - * @return integer + * @return int */ public function key() { return $this->pos; @@ -1602,7 +1749,6 @@ abstract class SwiftFileBackendList implements Iterator { /** * @see Iterator::next() - * @return void */ public function next() { // Advance to the next file in the page @@ -1619,7 +1765,6 @@ abstract class SwiftFileBackendList implements Iterator { /** * @see Iterator::rewind() - * @return void */ public function rewind() { $this->pos = 0; @@ -1646,10 +1791,10 @@ abstract class SwiftFileBackendList implements Iterator { * * @param string $container Resolved container name * @param string $dir Resolved path relative to container - * @param string $after|null - * @param integer $limit + * @param string $after + * @param int $limit * @param array $params - * @return Traversable|Array + * @return Traversable|array */ abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); } @@ -1666,10 +1811,6 @@ class SwiftFileBackendDirList extends SwiftFileBackendList { return substr( current( $this->bufferIter ), $this->suffixStart, -1 ); } - /** - * @see SwiftFileBackendList::pageFromList() - * @return Array - */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); } @@ -1684,13 +1825,16 @@ class SwiftFileBackendFileList extends SwiftFileBackendList { * @return string|bool String (relative path) or false */ public function current() { - return substr( current( $this->bufferIter ), $this->suffixStart ); + list( $path, $stat ) = current( $this->bufferIter ); + $relPath = substr( $path, $this->suffixStart ); + if ( is_array( $stat ) ) { + $storageDir = rtrim( $this->params['dir'], '/' ); + $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat ); + } + + return $relPath; } - /** - * @see SwiftFileBackendList::pageFromList() - * @return Array - */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); } diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php index 8266e420..1b68130f 100644 --- a/includes/filebackend/TempFSFile.php +++ b/includes/filebackend/TempFSFile.php @@ -28,11 +28,24 @@ * @ingroup FileBackend */ class TempFSFile extends FSFile { - protected $canDelete = false; // bool; garbage collect the temp file + /** @var bool Garbage collect the temp file */ + protected $canDelete = false; - /** @var Array of active temp files to purge on shutdown */ + /** @var array Active temp files to purge on shutdown */ protected static $instances = array(); + /** @var array Map of (path => 1) for paths to delete on shutdown */ + protected static $pathsCollect = null; + + public function __construct( $path ) { + parent::__construct( $path ); + + if ( self::$pathsCollect === null ) { + self::$pathsCollect = array(); + register_shutdown_function( array( __CLASS__, 'purgeAllOnShutdown' ) ); + } + } + /** * Make a new temporary file on the file system. * Temporary files may be purged when the file object falls out of scope. @@ -56,12 +69,14 @@ class TempFSFile extends FSFile { } if ( $attempt >= 5 ) { wfProfileOut( __METHOD__ ); + return null; // give up } } $tmpFile = new self( $path ); - $tmpFile->canDelete = true; // safely instantiated + $tmpFile->autocollect(); // safely instantiated wfProfileOut( __METHOD__ ); + return $tmpFile; } @@ -75,13 +90,16 @@ class TempFSFile extends FSFile { wfSuppressWarnings(); $ok = unlink( $this->path ); wfRestoreWarnings(); + + unset( self::$pathsCollect[$this->path] ); + return $ok; } /** * Clean up the temporary file only after an object goes out of scope * - * @param Object $object + * @param stdClass $object * @return TempFSFile This object */ public function bind( $object ) { @@ -92,6 +110,7 @@ class TempFSFile extends FSFile { } $object->tempFSFileReferences[] = $this; } + return $this; } @@ -102,6 +121,9 @@ class TempFSFile extends FSFile { */ public function preserve() { $this->canDelete = false; + + unset( self::$pathsCollect[$this->path] ); + return $this; } @@ -112,17 +134,31 @@ class TempFSFile extends FSFile { */ public function autocollect() { $this->canDelete = true; + + self::$pathsCollect[$this->path] = 1; + return $this; } /** + * Try to make sure that all files are purged on error + * + * This method should only be called internally + */ + public static function purgeAllOnShutdown() { + foreach ( self::$pathsCollect as $path ) { + wfSuppressWarnings(); + unlink( $path ); + wfRestoreWarnings(); + } + } + + /** * Cleans up after the temporary file by deleting it */ function __destruct() { if ( $this->canDelete ) { - wfSuppressWarnings(); - unlink( $this->path ); - wfRestoreWarnings(); + $this->purge(); } } } diff --git a/includes/filebackend/filejournal/DBFileJournal.php b/includes/filebackend/filejournal/DBFileJournal.php index 9250aa5e..4f64f022 100644 --- a/includes/filebackend/filejournal/DBFileJournal.php +++ b/includes/filebackend/filejournal/DBFileJournal.php @@ -34,10 +34,9 @@ class DBFileJournal extends FileJournal { /** * Construct a new instance from configuration. - * $config includes: - * 'wiki' : wiki name to use for LoadBalancer * - * @param $config Array + * @param array $config Includes: + * 'wiki' : wiki name to use for LoadBalancer */ protected function __construct( array $config ) { parent::__construct( $config ); @@ -47,6 +46,8 @@ class DBFileJournal extends FileJournal { /** * @see FileJournal::logChangeBatch() + * @param array $entries + * @param string $batchId * @return Status */ protected function doLogChangeBatch( array $entries, $batchId ) { @@ -56,6 +57,7 @@ class DBFileJournal extends FileJournal { $dbw = $this->getMasterDB(); } catch ( DBError $e ) { $status->fatal( 'filejournal-fail-dbconnect', $this->backend ); + return $status; } @@ -80,6 +82,7 @@ class DBFileJournal extends FileJournal { } } catch ( DBError $e ) { $status->fatal( 'filejournal-fail-dbquery', $this->backend ); + return $status; } @@ -88,7 +91,7 @@ class DBFileJournal extends FileJournal { /** * @see FileJournal::doGetCurrentPosition() - * @return integer|false + * @return bool|mixed The value from the field, or false on failure. */ protected function doGetCurrentPosition() { $dbw = $this->getMasterDB(); @@ -101,13 +104,14 @@ class DBFileJournal extends FileJournal { /** * @see FileJournal::doGetPositionAtTime() - * @param $time integer|string timestamp - * @return integer|false + * @param int|string $time Timestamp + * @return bool|mixed The value from the field, or false on failure. */ protected function doGetPositionAtTime( $time ) { $dbw = $this->getMasterDB(); $encTimestamp = $dbw->addQuotes( $dbw->timestamp( $time ) ); + return $dbw->selectField( 'filejournal', 'fj_id', array( 'fj_backend' => $this->backend, "fj_timestamp <= $encTimestamp" ), __METHOD__, @@ -117,8 +121,9 @@ class DBFileJournal extends FileJournal { /** * @see FileJournal::doGetChangeEntries() - * @return Array - * @throws DBError + * @param int $start + * @param int $limit + * @return array */ protected function doGetChangeEntries( $start, $limit ) { $dbw = $this->getMasterDB(); @@ -179,6 +184,7 @@ class DBFileJournal extends FileJournal { $this->dbw = $lb->getConnection( DB_MASTER, array(), $this->wiki ); $this->dbw->clearFlag( DBO_TRX ); } + return $this->dbw; } } diff --git a/includes/filebackend/filejournal/FileJournal.php b/includes/filebackend/filejournal/FileJournal.php index a1b7a459..c0651485 100644 --- a/includes/filebackend/filejournal/FileJournal.php +++ b/includes/filebackend/filejournal/FileJournal.php @@ -36,15 +36,17 @@ * @since 1.20 */ abstract class FileJournal { - protected $backend; // string - protected $ttlDays; // integer + /** @var string */ + protected $backend; + + /** @var int */ + protected $ttlDays; /** * Construct a new instance from configuration. - * $config includes: - * 'ttlDays' : days to keep log entries around (false means "forever") * - * @param $config Array + * @param array $config Includes: + * 'ttlDays' : days to keep log entries around (false means "forever") */ protected function __construct( array $config ) { $this->ttlDays = isset( $config['ttlDays'] ) ? $config['ttlDays'] : false; @@ -53,7 +55,7 @@ abstract class FileJournal { /** * Create an appropriate FileJournal object from config * - * @param $config Array + * @param array $config * @param string $backend A registered file backend name * @throws MWException * @return FileJournal @@ -65,6 +67,7 @@ abstract class FileJournal { throw new MWException( "Class given is not an instance of FileJournal." ); } $jrn->backend = $backend; + return $jrn; } @@ -79,18 +82,18 @@ abstract class FileJournal { $s .= mt_rand( 0, 2147483647 ); } $s = wfBaseConvert( sha1( $s ), 16, 36, 31 ); + return substr( wfBaseConvert( wfTimestamp( TS_MW ), 10, 36, 9 ) . $s, 0, 31 ); } /** * Log changes made by a batch file operation. - * $entries is an array of log entries, each of which contains: + * + * @param array $entries List of file operations (each an array of parameters) which contain: * op : Basic operation name (create, update, delete) * path : The storage path of the file * newSha1 : The final base 36 SHA-1 of the file - * Note that 'false' should be used as the SHA-1 for non-existing files. - * - * @param array $entries List of file operations (each an array of parameters) + * Note that 'false' should be used as the SHA-1 for non-existing files. * @param string $batchId UUID string that identifies the operation batch * @return Status */ @@ -98,6 +101,7 @@ abstract class FileJournal { if ( !count( $entries ) ) { return Status::newGood(); } + return $this->doLogChangeBatch( $entries, $batchId ); } @@ -113,7 +117,7 @@ abstract class FileJournal { /** * Get the position ID of the latest journal entry * - * @return integer|false + * @return int|bool */ final public function getCurrentPosition() { return $this->doGetCurrentPosition(); @@ -121,15 +125,15 @@ abstract class FileJournal { /** * @see FileJournal::getCurrentPosition() - * @return integer|false + * @return int|bool */ abstract protected function doGetCurrentPosition(); /** * Get the position ID of the latest journal entry at some point in time * - * @param $time integer|string timestamp - * @return integer|false + * @param int|string $time Timestamp + * @return int|bool */ final public function getPositionAtTime( $time ) { return $this->doGetPositionAtTime( $time ); @@ -137,8 +141,8 @@ abstract class FileJournal { /** * @see FileJournal::getPositionAtTime() - * @param $time integer|string timestamp - * @return integer|false + * @param int|string $time Timestamp + * @return int|bool */ abstract protected function doGetPositionAtTime( $time ); @@ -146,7 +150,10 @@ abstract class FileJournal { * Get an array of file change log entries. * A starting change ID and/or limit can be specified. * - * The result as a list of associative arrays, each having: + * @param int $start Starting change ID or null + * @param int $limit Maximum number of items to return + * @param string &$next Updated to the ID of the next entry. + * @return array List of associative arrays, each having: * id : unique, monotonic, ID for this change * batch_uuid : UUID for an operation batch * backend : the backend name @@ -154,13 +161,7 @@ abstract class FileJournal { * path : affected storage path * new_sha1 : base 36 sha1 of the new file had the operation succeeded * timestamp : TS_MW timestamp of the batch change - - * Also, $next is updated to the ID of the next entry. - * - * @param $start integer Starting change ID or null - * @param $limit integer Maximum number of items to return - * @param &$next string - * @return Array + * Also, $next is updated to the ID of the next entry. */ final public function getChangeEntries( $start = null, $limit = 0, &$next = null ) { $entries = $this->doGetChangeEntries( $start, $limit ? $limit + 1 : 0 ); @@ -170,12 +171,15 @@ abstract class FileJournal { } else { $next = null; // end of list } + return $entries; } /** * @see FileJournal::getChangeEntries() - * @return Array + * @param int $start + * @param int $limit + * @return array */ abstract protected function doGetChangeEntries( $start, $limit ); @@ -202,8 +206,8 @@ abstract class FileJournal { class NullFileJournal extends FileJournal { /** * @see FileJournal::doLogChangeBatch() - * @param $entries array - * @param $batchId string + * @param array $entries + * @param string $batchId * @return Status */ protected function doLogChangeBatch( array $entries, $batchId ) { @@ -212,7 +216,7 @@ class NullFileJournal extends FileJournal { /** * @see FileJournal::doGetCurrentPosition() - * @return integer|false + * @return int|bool */ protected function doGetCurrentPosition() { return false; @@ -220,8 +224,8 @@ class NullFileJournal extends FileJournal { /** * @see FileJournal::doGetPositionAtTime() - * @param $time integer|string timestamp - * @return integer|false + * @param int|string $time Timestamp + * @return int|bool */ protected function doGetPositionAtTime( $time ) { return false; @@ -229,7 +233,9 @@ class NullFileJournal extends FileJournal { /** * @see FileJournal::doGetChangeEntries() - * @return Array + * @param int $start + * @param int $limit + * @return array */ protected function doGetChangeEntries( $start, $limit ) { return array(); diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php index 3e934ba5..450ccc82 100644 --- a/includes/filebackend/lockmanager/DBLockManager.php +++ b/includes/filebackend/lockmanager/DBLockManager.php @@ -37,7 +37,7 @@ * @since 1.19 */ abstract class DBLockManager extends QuorumLockManager { - /** @var Array Map of DB names to server config */ + /** @var array Map of DB names to server config */ protected $dbServers; // (DB name => server config array) /** @var BagOStuff */ protected $statusCache; @@ -46,13 +46,13 @@ abstract class DBLockManager extends QuorumLockManager { protected $safeDelay; // integer number of seconds protected $session = 0; // random integer - /** @var Array Map Database connections (DB name => Database) */ + /** @var array Map Database connections (DB name => Database) */ protected $conns = array(); /** * Construct a new instance from configuration. * - * $config paramaters include: + * @param array $config Paramaters include: * - dbServers : Associative array of DB names to server configuration. * Configuration is an associative array that includes: * - host : DB server name @@ -70,8 +70,6 @@ abstract class DBLockManager extends QuorumLockManager { * - lockExpiry : Lock timeout (seconds) for dropped connections. [optional] * This tells the DB server how long to wait before assuming * connection failure and releasing all the locks for a session. - * - * @param array $config */ public function __construct( array $config ) { parent::__construct( $config ); @@ -110,12 +108,13 @@ abstract class DBLockManager extends QuorumLockManager { $this->session = wfRandomString( 31 ); } - // @TODO: change this code to work in one batch + // @todo change this code to work in one batch protected function getLocksOnServer( $lockSrv, array $pathsByType ) { $status = Status::newGood(); foreach ( $pathsByType as $type => $paths ) { $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); } + return $status; } @@ -125,6 +124,7 @@ abstract class DBLockManager extends QuorumLockManager { /** * @see QuorumLockManager::isServerUp() + * @param string $lockSrv * @return bool */ protected function isServerUp( $lockSrv ) { @@ -135,15 +135,17 @@ abstract class DBLockManager extends QuorumLockManager { $this->getConnection( $lockSrv ); } catch ( DBError $e ) { $this->cacheRecordFailure( $lockSrv ); + return false; // failed to connect } + return true; } /** * Get (or reuse) a connection to a lock DB * - * @param $lockDb string + * @param string $lockDb * @return DatabaseBase * @throws DBError */ @@ -175,24 +177,25 @@ abstract class DBLockManager extends QuorumLockManager { if ( !$this->conns[$lockDb]->trxLevel() ) { $this->conns[$lockDb]->begin( __METHOD__ ); // start transaction } + return $this->conns[$lockDb]; } /** * Do additional initialization for new lock DB connection * - * @param $lockDb string - * @param $db DatabaseBase - * @return void + * @param string $lockDb + * @param DatabaseBase $db * @throws DBError */ - protected function initConnection( $lockDb, DatabaseBase $db ) {} + protected function initConnection( $lockDb, DatabaseBase $db ) { + } /** * Checks if the DB has not recently had connection/query errors. * This just avoids wasting time on doomed connection attempts. * - * @param $lockDb string + * @param string $lockDb * @return bool */ protected function cacheCheckFailures( $lockDb ) { @@ -204,7 +207,7 @@ abstract class DBLockManager extends QuorumLockManager { /** * Log a lock request failure to the cache * - * @param $lockDb string + * @param string $lockDb * @return bool Success */ protected function cacheRecordFailure( $lockDb ) { @@ -216,7 +219,7 @@ abstract class DBLockManager extends QuorumLockManager { /** * Get a cache key for recent query misses for a DB * - * @param $lockDb string + * @param string $lockDb * @return string */ protected function getMissKey( $lockDb ) { @@ -242,7 +245,7 @@ abstract class DBLockManager extends QuorumLockManager { * @ingroup LockManager */ class MySqlLockManager extends DBLockManager { - /** @var Array Mapping of lock types to the type actually used */ + /** @var array Mapping of lock types to the type actually used */ protected $lockTypeMap = array( self::LOCK_SH => self::LOCK_SH, self::LOCK_UW => self::LOCK_SH, @@ -250,8 +253,8 @@ class MySqlLockManager extends DBLockManager { ); /** - * @param $lockDb string - * @param $db DatabaseBase + * @param string $lockDb + * @param DatabaseBase $db */ protected function initConnection( $lockDb, DatabaseBase $db ) { # Let this transaction see lock rows from other transactions @@ -263,6 +266,9 @@ class MySqlLockManager extends DBLockManager { * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118. * * @see DBLockManager::getLocksOnServer() + * @param string $lockSrv + * @param array $paths + * @param string $type * @return Status */ protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { @@ -359,7 +365,7 @@ class MySqlLockManager extends DBLockManager { * @ingroup LockManager */ class PostgreSqlLockManager extends DBLockManager { - /** @var Array Mapping of lock types to the type actually used */ + /** @var array Mapping of lock types to the type actually used */ protected $lockTypeMap = array( self::LOCK_SH => self::LOCK_SH, self::LOCK_UW => self::LOCK_SH, @@ -374,7 +380,7 @@ class PostgreSqlLockManager extends DBLockManager { $db = $this->getConnection( $lockSrv ); // checked in isServerUp() $bigints = array_unique( array_map( - function( $key ) { + function ( $key ) { return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 ); }, array_map( array( $this, 'sha1Base16Absolute' ), $paths ) diff --git a/includes/filebackend/lockmanager/FSLockManager.php b/includes/filebackend/lockmanager/FSLockManager.php index eacba704..bce6b34c 100644 --- a/includes/filebackend/lockmanager/FSLockManager.php +++ b/includes/filebackend/lockmanager/FSLockManager.php @@ -34,7 +34,7 @@ * @since 1.19 */ class FSLockManager extends LockManager { - /** @var Array Mapping of lock types to the type actually used */ + /** @var array Mapping of lock types to the type actually used */ protected $lockTypeMap = array( self::LOCK_SH => self::LOCK_SH, self::LOCK_UW => self::LOCK_SH, @@ -43,16 +43,14 @@ class FSLockManager extends LockManager { protected $lockDir; // global dir for all servers - /** @var Array Map of (locked key => lock file handle) */ + /** @var array Map of (locked key => lock file handle) */ protected $handles = array(); /** * Construct a new instance from configuration. * - * $config includes: + * @param array $config Includes: * - lockDirectory : Directory containing the lock files - * - * @param array $config */ function __construct( array $config ) { parent::__construct( $config ); @@ -62,8 +60,8 @@ class FSLockManager extends LockManager { /** * @see LockManager::doLock() - * @param $paths array - * @param $type int + * @param array $paths + * @param int $type * @return Status */ protected function doLock( array $paths, $type ) { @@ -77,6 +75,7 @@ class FSLockManager extends LockManager { } else { // Abort and unlock everything $status->merge( $this->doUnlock( $lockedPaths, $type ) ); + return $status; } } @@ -86,8 +85,8 @@ class FSLockManager extends LockManager { /** * @see LockManager::doUnlock() - * @param $paths array - * @param $type int + * @param array $paths + * @param int $type * @return Status */ protected function doUnlock( array $paths, $type ) { @@ -103,8 +102,8 @@ class FSLockManager extends LockManager { /** * Lock a single resource key * - * @param $path string - * @param $type integer + * @param string $path + * @param int $type * @return Status */ protected function doSingleLock( $path, $type ) { @@ -148,8 +147,8 @@ class FSLockManager extends LockManager { /** * Unlock a single resource key * - * @param $path string - * @param $type integer + * @param string $path + * @param int $type * @return Status */ protected function doSingleUnlock( $path, $type ) { @@ -191,8 +190,8 @@ class FSLockManager extends LockManager { } /** - * @param $path string - * @param $handlesToClose array + * @param string $path + * @param array $handlesToClose * @return Status */ private function closeLockHandles( $path, array $handlesToClose ) { @@ -205,11 +204,12 @@ class FSLockManager extends LockManager { $status->warning( 'lockmanager-fail-closelock', $path ); } } + return $status; } /** - * @param $path string + * @param string $path * @return Status */ private function pruneKeyLockFiles( $path ) { @@ -221,12 +221,13 @@ class FSLockManager extends LockManager { } unset( $this->handles[$path] ); } + return $status; } /** * Get the path to the lock file for a key - * @param $path string + * @param string $path * @return string */ protected function getLockPath( $path ) { diff --git a/includes/filebackend/lockmanager/LSLockManager.php b/includes/filebackend/lockmanager/LSLockManager.php deleted file mode 100644 index 97de8dca..00000000 --- a/includes/filebackend/lockmanager/LSLockManager.php +++ /dev/null @@ -1,218 +0,0 @@ -<?php -/** - * Version of LockManager based on using lock daemon servers. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * http://www.gnu.org/copyleft/gpl.html - * - * @file - * @ingroup LockManager - */ - -/** - * Manage locks using a lock daemon server. - * - * Version of LockManager based on using lock daemon servers. - * This is meant for multi-wiki systems that may share files. - * All locks are non-blocking, which avoids deadlocks. - * - * All lock requests for a resource, identified by a hash string, will map - * to one bucket. Each bucket maps to one or several peer servers, each - * running LockServerDaemon.php, listening on a designated TCP port. - * A majority of peers must agree for a lock to be acquired. - * - * @ingroup LockManager - * @since 1.19 - */ -class LSLockManager extends QuorumLockManager { - /** @var Array Mapping of lock types to the type actually used */ - protected $lockTypeMap = array( - self::LOCK_SH => self::LOCK_SH, - self::LOCK_UW => self::LOCK_SH, - self::LOCK_EX => self::LOCK_EX - ); - - /** @var Array Map of server names to server config */ - protected $lockServers; // (server name => server config array) - - /** @var Array Map Server connections (server name => resource) */ - protected $conns = array(); - - protected $connTimeout; // float number of seconds - protected $session = ''; // random SHA-1 string - - /** - * Construct a new instance from configuration. - * - * $config paramaters include: - * - lockServers : Associative array of server names to configuration. - * Configuration is an associative array that includes: - * - host : IP address/hostname - * - port : TCP port - * - authKey : Secret string the lock server uses - * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, - * each having an odd-numbered list of server names (peers) as values. - * - connTimeout : Lock server connection attempt timeout. [optional] - * - * @param array $config - */ - public function __construct( array $config ) { - parent::__construct( $config ); - - $this->lockServers = $config['lockServers']; - // Sanitize srvsByBucket config to prevent PHP errors - $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); - $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive - - if ( isset( $config['connTimeout'] ) ) { - $this->connTimeout = $config['connTimeout']; - } else { - $this->connTimeout = 3; // use some sane amount - } - - $this->session = wfRandomString( 32 ); // 128 bits - } - - /** - * @see QuorumLockManager::getLocksOnServer() - * @return Status - */ - protected function getLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); - - // Send out the command and get the response... - $type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX'; - $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) ); - $response = $this->sendCommand( $lockSrv, 'ACQUIRE', $type, $keys ); - - if ( $response !== 'ACQUIRED' ) { - foreach ( $paths as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } - - return $status; - } - - /** - * @see QuorumLockManager::freeLocksOnServer() - * @return Status - */ - protected function freeLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); - - // Send out the command and get the response... - $type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX'; - $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) ); - $response = $this->sendCommand( $lockSrv, 'RELEASE', $type, $keys ); - - if ( $response !== 'RELEASED' ) { - foreach ( $paths as $path ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - } - - return $status; - } - - /** - * @see QuorumLockManager::releaseAllLocks() - * @return Status - */ - protected function releaseAllLocks() { - $status = Status::newGood(); - - foreach ( $this->conns as $lockSrv => $conn ) { - $response = $this->sendCommand( $lockSrv, 'RELEASE_ALL', '', array() ); - if ( $response !== 'RELEASED_ALL' ) { - $status->fatal( 'lockmanager-fail-svr-release', $lockSrv ); - } - } - - return $status; - } - - /** - * @see QuorumLockManager::isServerUp() - * @return bool - */ - protected function isServerUp( $lockSrv ) { - return (bool)$this->getConnection( $lockSrv ); - } - - /** - * Send a command and get back the response - * - * @param $lockSrv string - * @param $action string - * @param $type string - * @param $values Array - * @return string|bool - */ - protected function sendCommand( $lockSrv, $action, $type, $values ) { - $conn = $this->getConnection( $lockSrv ); - if ( !$conn ) { - return false; // no connection - } - $authKey = $this->lockServers[$lockSrv]['authKey']; - // Build of the command as a flat string... - $values = implode( '|', $values ); - $key = hash_hmac( 'sha1', "{$this->session}\n{$action}\n{$type}\n{$values}", $authKey ); - // Send out the command... - if ( fwrite( $conn, "{$this->session}:$key:$action:$type:$values\n" ) === false ) { - return false; - } - // Get the response... - $response = fgets( $conn ); - if ( $response === false ) { - return false; - } - return trim( $response ); - } - - /** - * Get (or reuse) a connection to a lock server - * - * @param $lockSrv string - * @return resource - */ - protected function getConnection( $lockSrv ) { - if ( !isset( $this->conns[$lockSrv] ) ) { - $cfg = $this->lockServers[$lockSrv]; - wfSuppressWarnings(); - $errno = $errstr = ''; - $conn = fsockopen( $cfg['host'], $cfg['port'], $errno, $errstr, $this->connTimeout ); - wfRestoreWarnings(); - if ( $conn === false ) { - return null; - } - $sec = floor( $this->connTimeout ); - $usec = floor( ( $this->connTimeout - floor( $this->connTimeout ) ) * 1e6 ); - stream_set_timeout( $conn, $sec, $usec ); - $this->conns[$lockSrv] = $conn; - } - return $this->conns[$lockSrv]; - } - - /** - * Make sure remaining locks get cleared for sanity - */ - function __destruct() { - $this->releaseAllLocks(); - foreach ( $this->conns as $conn ) { - fclose( $conn ); - } - } -} diff --git a/includes/filebackend/lockmanager/LockManager.php b/includes/filebackend/lockmanager/LockManager.php index dad8a624..df8d2d4f 100644 --- a/includes/filebackend/lockmanager/LockManager.php +++ b/includes/filebackend/lockmanager/LockManager.php @@ -43,14 +43,14 @@ * @since 1.19 */ abstract class LockManager { - /** @var Array Mapping of lock types to the type actually used */ + /** @var array Mapping of lock types to the type actually used */ protected $lockTypeMap = array( self::LOCK_SH => self::LOCK_SH, self::LOCK_UW => self::LOCK_EX, // subclasses may use self::LOCK_SH self::LOCK_EX => self::LOCK_EX ); - /** @var Array Map of (resource path => lock type => count) */ + /** @var array Map of (resource path => lock type => count) */ protected $locksHeld = array(); protected $domain; // string; domain (usually wiki ID) @@ -64,12 +64,10 @@ abstract class LockManager { /** * Construct a new instance from configuration * - * $config paramaters include: + * @param array $config Paramaters include: * - domain : Domain (usually wiki ID) that all resources are relative to [optional] * - lockTTL : Age (in seconds) at which resource locks should expire. * This only applies if locks are not tied to a connection/process. - * - * @param $config Array */ public function __construct( array $config ) { $this->domain = isset( $config['domain'] ) ? $config['domain'] : wfWikiID(); @@ -87,8 +85,8 @@ abstract class LockManager { * Lock the resources at the given abstract paths * * @param array $paths List of resource names - * @param $type integer LockManager::LOCK_* constant - * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21) + * @param int $type LockManager::LOCK_* constant + * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.21) * @return Status */ final public function lock( array $paths, $type = self::LOCK_EX, $timeout = 0 ) { @@ -99,7 +97,7 @@ abstract class LockManager { * Lock the resources at the given abstract paths * * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21) + * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.21) * @return Status * @since 1.22 */ @@ -119,6 +117,7 @@ abstract class LockManager { $elapsed = microtime( true ) - $start; } while ( $elapsed < $timeout && $elapsed >= 0 ); wfProfileOut( __METHOD__ ); + return $status; } @@ -126,7 +125,7 @@ abstract class LockManager { * Unlock the resources at the given abstract paths * * @param array $paths List of paths - * @param $type integer LockManager::LOCK_* constant + * @param int $type LockManager::LOCK_* constant * @return Status */ final public function unlock( array $paths, $type = self::LOCK_EX ) { @@ -145,6 +144,7 @@ abstract class LockManager { $pathsByType = $this->normalizePathsByType( $pathsByType ); $status = $this->doUnlockByType( $pathsByType ); wfProfileOut( __METHOD__ ); + return $status; } @@ -153,7 +153,7 @@ abstract class LockManager { * Before hashing, the path will be prefixed with the domain ID. * This should be used interally for lock key or file names. * - * @param $path string + * @param string $path * @return string */ final protected function sha1Base36Absolute( $path ) { @@ -165,7 +165,7 @@ abstract class LockManager { * Before hashing, the path will be prefixed with the domain ID. * This should be used interally for lock key or file names. * - * @param $path string + * @param string $path * @return string */ final protected function sha1Base16Absolute( $path ) { @@ -176,8 +176,8 @@ abstract class LockManager { * Normalize the $paths array by converting LOCK_UW locks into the * appropriate type and removing any duplicated paths for each lock type. * - * @param array $paths Map of LockManager::LOCK_* constants to lists of paths - * @return Array + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths + * @return array * @since 1.22 */ final protected function normalizePathsByType( array $pathsByType ) { @@ -185,12 +185,13 @@ abstract class LockManager { foreach ( $pathsByType as $type => $paths ) { $res[$this->lockTypeMap[$type]] = array_unique( $paths ); } + return $res; } /** * @see LockManager::lockByType() - * @param array $paths Map of LockManager::LOCK_* constants to lists of paths + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status * @since 1.22 */ @@ -203,12 +204,13 @@ abstract class LockManager { $lockedByType[$type] = $paths; } else { // Release the subset of locks that were acquired - foreach ( $lockedByType as $type => $paths ) { - $status->merge( $this->doUnlock( $paths, $type ) ); + foreach ( $lockedByType as $lType => $lPaths ) { + $status->merge( $this->doUnlock( $lPaths, $lType ) ); } break; } } + return $status; } @@ -216,14 +218,14 @@ abstract class LockManager { * Lock resources with the given keys and lock type * * @param array $paths List of paths - * @param $type integer LockManager::LOCK_* constant + * @param int $type LockManager::LOCK_* constant * @return Status */ abstract protected function doLock( array $paths, $type ); /** * @see LockManager::unlockByType() - * @param array $paths Map of LockManager::LOCK_* constants to lists of paths + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status * @since 1.22 */ @@ -232,6 +234,7 @@ abstract class LockManager { foreach ( $pathsByType as $type => $paths ) { $status->merge( $this->doUnlock( $paths, $type ) ); } + return $status; } @@ -239,7 +242,7 @@ abstract class LockManager { * Unlock resources with the given keys and lock type * * @param array $paths List of paths - * @param $type integer LockManager::LOCK_* constant + * @param int $type LockManager::LOCK_* constant * @return Status */ abstract protected function doUnlock( array $paths, $type ); diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php index 9aff2415..19fc4fef 100644 --- a/includes/filebackend/lockmanager/LockManagerGroup.php +++ b/includes/filebackend/lockmanager/LockManagerGroup.php @@ -29,12 +29,12 @@ * @since 1.19 */ class LockManagerGroup { - /** @var Array (domain => LockManager) */ + /** @var array (domain => LockManager) */ protected static $instances = array(); protected $domain; // string; domain (usually wiki ID) - /** @var Array of (name => ('class' => ..., 'config' => ..., 'instance' => ...)) */ + /** @var array Array of (name => ('class' => ..., 'config' => ..., 'instance' => ...)) */ protected $managers = array(); /** @@ -45,7 +45,7 @@ class LockManagerGroup { } /** - * @param string $domain Domain (usually wiki ID) + * @param bool|string $domain Domain (usually wiki ID). Default: false. * @return LockManagerGroup */ public static function singleton( $domain = false ) { @@ -54,13 +54,12 @@ class LockManagerGroup { self::$instances[$domain] = new self( $domain ); self::$instances[$domain]->initFromGlobals(); } + return self::$instances[$domain]; } /** * Destroy the singleton instances - * - * @return void */ public static function destroySingletons() { self::$instances = array(); @@ -68,8 +67,6 @@ class LockManagerGroup { /** * Register lock managers from the global variables - * - * @return void */ protected function initFromGlobals() { global $wgLockManagers; @@ -80,8 +77,7 @@ class LockManagerGroup { /** * Register an array of file lock manager configurations * - * @param $configs Array - * @return void + * @param array $configs * @throws MWException */ protected function register( array $configs ) { @@ -107,7 +103,7 @@ class LockManagerGroup { /** * Get the lock manager object with a given name * - * @param $name string + * @param string $name * @return LockManager * @throws MWException */ @@ -121,14 +117,15 @@ class LockManagerGroup { $config = $this->managers[$name]['config']; $this->managers[$name]['instance'] = new $class( $config ); } + return $this->managers[$name]['instance']; } /** * Get the config array for a lock manager object with a given name * - * @param $name string - * @return Array + * @param string $name + * @return array * @throws MWException */ public function config( $name ) { @@ -136,6 +133,7 @@ class LockManagerGroup { throw new MWException( "No lock manager defined with the name `$name`." ); } $class = $this->managers[$name]['class']; + return array( 'class' => $class ) + $this->managers[$name]['config']; } diff --git a/includes/filebackend/lockmanager/MemcLockManager.php b/includes/filebackend/lockmanager/MemcLockManager.php index 5eab03ee..9bb01c21 100644 --- a/includes/filebackend/lockmanager/MemcLockManager.php +++ b/includes/filebackend/lockmanager/MemcLockManager.php @@ -36,31 +36,31 @@ * @since 1.20 */ class MemcLockManager extends QuorumLockManager { - /** @var Array Mapping of lock types to the type actually used */ + /** @var array Mapping of lock types to the type actually used */ protected $lockTypeMap = array( self::LOCK_SH => self::LOCK_SH, self::LOCK_UW => self::LOCK_SH, self::LOCK_EX => self::LOCK_EX ); - /** @var Array Map server names to MemcachedBagOStuff objects */ + /** @var array Map server names to MemcachedBagOStuff objects */ protected $bagOStuffs = array(); - /** @var Array */ - protected $serversUp = array(); // (server name => bool) - protected $session = ''; // string; random UUID + /** @var array (server name => bool) */ + protected $serversUp = array(); + + /** @var string Random UUID */ + protected $session = ''; /** * Construct a new instance from configuration. * - * $config paramaters include: + * @param array $config Paramaters include: * - lockServers : Associative array of server names to "<IP>:<port>" strings. * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, * each having an odd-numbered list of server names (peers) as values. * - memcConfig : Configuration array for ObjectCache::newFromParams. [optional] * If set, this must use one of the memcached classes. - * - * @param array $config * @throws MWException */ public function __construct( array $config ) { @@ -88,7 +88,7 @@ class MemcLockManager extends QuorumLockManager { $this->session = wfRandomString( 32 ); } - // @TODO: change this code to work in one batch + // @todo Change this code to work in one batch protected function getLocksOnServer( $lockSrv, array $pathsByType ) { $status = Status::newGood(); @@ -100,8 +100,8 @@ class MemcLockManager extends QuorumLockManager { ? array_merge( $lockedPaths[$type], $paths ) : $paths; } else { - foreach ( $lockedPaths as $type => $paths ) { - $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); + foreach ( $lockedPaths as $lType => $lPaths ) { + $status->merge( $this->doFreeLocksOnServer( $lockSrv, $lPaths, $lType ) ); } break; } @@ -110,7 +110,7 @@ class MemcLockManager extends QuorumLockManager { return $status; } - // @TODO: change this code to work in one batch + // @todo Change this code to work in one batch protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { $status = Status::newGood(); @@ -123,6 +123,9 @@ class MemcLockManager extends QuorumLockManager { /** * @see QuorumLockManager::getLocksOnServer() + * @param string $lockSrv + * @param array $paths + * @param string $type * @return Status */ protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { @@ -136,6 +139,7 @@ class MemcLockManager extends QuorumLockManager { foreach ( $paths as $path ) { $status->fatal( 'lockmanager-fail-acquirelock', $path ); } + return $status; } @@ -195,6 +199,9 @@ class MemcLockManager extends QuorumLockManager { /** * @see QuorumLockManager::freeLocksOnServer() + * @param string $lockSrv + * @param array $paths + * @param string $type * @return Status */ protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) { @@ -208,7 +215,8 @@ class MemcLockManager extends QuorumLockManager { foreach ( $paths as $path ) { $status->fatal( 'lockmanager-fail-releaselock', $path ); } - return; + + return $status; } // Fetch all the existing lock records... @@ -254,6 +262,7 @@ class MemcLockManager extends QuorumLockManager { /** * @see QuorumLockManager::isServerUp() + * @param string $lockSrv * @return bool */ protected function isServerUp( $lockSrv ) { @@ -280,11 +289,12 @@ class MemcLockManager extends QuorumLockManager { return null; // server appears to be down } } + return $memc; } /** - * @param $path string + * @param string $path * @return string */ protected function recordKeyForPath( $path ) { @@ -292,27 +302,28 @@ class MemcLockManager extends QuorumLockManager { } /** - * @return Array An empty lock structure for a key + * @return array An empty lock structure for a key */ protected static function newLockArray() { return array( self::LOCK_SH => array(), self::LOCK_EX => array() ); } /** - * @param $a array - * @return Array An empty lock structure for a key + * @param array $a + * @return array An empty lock structure for a key */ protected static function sanitizeLockArray( $a ) { if ( is_array( $a ) && isset( $a[self::LOCK_EX] ) && isset( $a[self::LOCK_SH] ) ) { return $a; } else { trigger_error( __METHOD__ . ": reset invalid lock array.", E_USER_WARNING ); + return self::newLockArray(); } } /** - * @param $memc MemcachedBagOStuff + * @param MemcachedBagOStuff $memc * @param array $keys List of keys to acquire * @return bool */ @@ -350,9 +361,8 @@ class MemcLockManager extends QuorumLockManager { } /** - * @param $memc MemcachedBagOStuff + * @param MemcachedBagOStuff $memc * @param array $keys List of acquired keys - * @return void */ protected function releaseMutexes( MemcachedBagOStuff $memc, array $keys ) { foreach ( $keys as $key ) { diff --git a/includes/filebackend/lockmanager/QuorumLockManager.php b/includes/filebackend/lockmanager/QuorumLockManager.php index 8356d32a..a692012d 100644 --- a/includes/filebackend/lockmanager/QuorumLockManager.php +++ b/includes/filebackend/lockmanager/QuorumLockManager.php @@ -29,9 +29,10 @@ * @since 1.20 */ abstract class QuorumLockManager extends LockManager { - /** @var Array Map of bucket indexes to peer server lists */ + /** @var array Map of bucket indexes to peer server lists */ protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...)) - /** @var Array Map of degraded buckets */ + + /** @var array Map of degraded buckets */ protected $degradedBuckets = array(); // (buckey index => UNIX timestamp) final protected function doLock( array $paths, $type ) { @@ -65,6 +66,7 @@ abstract class QuorumLockManager extends LockManager { $status->merge( $this->doLockingRequestBucket( $bucket, $pathsToLockByType ) ); if ( !$status->isOK() ) { $status->merge( $this->doUnlockByType( $lockedPaths ) ); + return $status; } // Record these locks as active @@ -120,7 +122,7 @@ abstract class QuorumLockManager extends LockManager { * Attempt to acquire locks with the peers for a bucket. * This is all or nothing; if any key is locked then this totally fails. * - * @param $bucket integer + * @param int $bucket * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ @@ -162,7 +164,7 @@ abstract class QuorumLockManager extends LockManager { /** * Attempt to release locks with the peers for a bucket * - * @param $bucket integer + * @param int $bucket * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ @@ -176,8 +178,8 @@ abstract class QuorumLockManager extends LockManager { foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { if ( !$this->isServerUp( $lockSrv ) ) { $status->warning( 'lockmanager-fail-svr-release', $lockSrv ); - // Attempt to release the lock on this peer } else { + // Attempt to release the lock on this peer $status->merge( $this->freeLocksOnServer( $lockSrv, $pathsByType ) ); ++$yesVotes; // success for this peer // Normally the first peers form the quorum, and the others are ignored. @@ -198,8 +200,8 @@ abstract class QuorumLockManager extends LockManager { * Get the bucket for resource path. * This should avoid throwing any exceptions. * - * @param $path string - * @return integer + * @param string $path + * @return int */ protected function getBucketFromPath( $path ) { $prefix = substr( sha1( $path ), 0, 2 ); // first 2 hex chars (8 bits) @@ -210,7 +212,7 @@ abstract class QuorumLockManager extends LockManager { * Check if a lock server is up. * This should process cache results to reduce RTT. * - * @param $lockSrv string + * @param string $lockSrv * @return bool */ abstract protected function isServerUp( $lockSrv ); @@ -218,7 +220,7 @@ abstract class QuorumLockManager extends LockManager { /** * Get a connection to a lock server and acquire locks * - * @param $lockSrv string + * @param string $lockSrv * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ @@ -229,7 +231,7 @@ abstract class QuorumLockManager extends LockManager { * * Subclasses must effectively implement this or releaseAllLocks(). * - * @param $lockSrv string + * @param string $lockSrv * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ diff --git a/includes/filebackend/lockmanager/RedisLockManager.php b/includes/filebackend/lockmanager/RedisLockManager.php index 43b0198a..90e05817 100644 --- a/includes/filebackend/lockmanager/RedisLockManager.php +++ b/includes/filebackend/lockmanager/RedisLockManager.php @@ -38,7 +38,7 @@ * @since 1.22 */ class RedisLockManager extends QuorumLockManager { - /** @var Array Mapping of lock types to the type actually used */ + /** @var array Mapping of lock types to the type actually used */ protected $lockTypeMap = array( self::LOCK_SH => self::LOCK_SH, self::LOCK_UW => self::LOCK_SH, @@ -47,21 +47,21 @@ class RedisLockManager extends QuorumLockManager { /** @var RedisConnectionPool */ protected $redisPool; - /** @var Array Map server names to hostname/IP and port numbers */ + + /** @var array Map server names to hostname/IP and port numbers */ protected $lockServers = array(); - protected $session = ''; // string; random UUID + /** @var string Random UUID */ + protected $session = ''; /** * Construct a new instance from configuration. * - * $config paramaters include: + * @param array $config Parameters include: * - lockServers : Associative array of server names to "<IP>:<port>" strings. * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, * each having an odd-numbered list of server names (peers) as values. * - redisConfig : Configuration for RedisConnectionPool::__construct(). - * - * @param Array $config * @throws MWException */ public function __construct( array $config ) { @@ -78,115 +78,89 @@ class RedisLockManager extends QuorumLockManager { $this->session = wfRandomString( 32 ); } - // @TODO: change this code to work in one batch protected function getLocksOnServer( $lockSrv, array $pathsByType ) { $status = Status::newGood(); - $lockedPaths = array(); - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); - if ( $status->isOK() ) { - $lockedPaths[$type] = isset( $lockedPaths[$type] ) - ? array_merge( $lockedPaths[$type], $paths ) - : $paths; - } else { - foreach ( $lockedPaths as $type => $paths ) { - $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); - } - break; - } - } - - return $status; - } - - // @TODO: change this code to work in one batch - protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { - $status = Status::newGood(); - - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); - } - - return $status; - } - - protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); - $server = $this->lockServers[$lockSrv]; $conn = $this->redisPool->getConnection( $server ); if ( !$conn ) { - foreach ( $paths as $path ) { + foreach ( array_merge( array_values( $pathsByType ) ) as $path ) { $status->fatal( 'lockmanager-fail-acquirelock', $path ); } + return $status; } - $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records + $pathsByKey = array(); // (type:hash => path) map + foreach ( $pathsByType as $type => $paths ) { + $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; + foreach ( $paths as $path ) { + $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; + } + } try { static $script = <<<LUA - if ARGV[1] ~= 'EX' and ARGV[1] ~= 'SH' then - return redis.error_reply('Unrecognized lock type given (must be EX or SH)') - end local failed = {} + -- Load input params (e.g. session, ttl, time of request) + local rSession, rTTL, rTime = unpack(ARGV) -- Check that all the locks can be acquired - for i,resourceKey in ipairs(KEYS) do + for i,requestKey in ipairs(KEYS) do + local _, _, rType, resourceKey = string.find(requestKey,"(%w+):(%w+)$") local keyIsFree = true local currentLocks = redis.call('hKeys',resourceKey) for i,lockKey in ipairs(currentLocks) do + -- Get the type and session of this lock local _, _, type, session = string.find(lockKey,"(%w+):(%w+)") -- Check any locks that are not owned by this session - if session ~= ARGV[2] then - local lockTimestamp = redis.call('hGet',resourceKey,lockKey) - if 1*lockTimestamp < ( ARGV[4] - ARGV[3] ) then + if session ~= rSession then + local lockExpiry = redis.call('hGet',resourceKey,lockKey) + if 1*lockExpiry < 1*rTime then -- Lock is stale, so just prune it out redis.call('hDel',resourceKey,lockKey) - elseif ARGV[1] == 'EX' or type == 'EX' then + elseif rType == 'EX' or type == 'EX' then keyIsFree = false break end end end if not keyIsFree then - failed[#failed+1] = resourceKey + failed[#failed+1] = requestKey end end -- If all locks could be acquired, then do so if #failed == 0 then - for i,resourceKey in ipairs(KEYS) do - redis.call('hSet',resourceKey,ARGV[1] .. ':' .. ARGV[2],ARGV[4]) + for i,requestKey in ipairs(KEYS) do + local _, _, rType, resourceKey = string.find(requestKey,"(%w+):(%w+)$") + redis.call('hSet',resourceKey,rType .. ':' .. rSession,rTime + rTTL) -- In addition to invalidation logic, be sure to garbage collect - redis.call('expire',resourceKey,ARGV[3]) + redis.call('expire',resourceKey,rTTL) end end return failed LUA; $res = $conn->luaEval( $script, array_merge( - $keys, // KEYS[0], KEYS[1],...KEYS[N] + array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] array( - $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1] - $this->session, // ARGV[2] - $this->lockTTL, // ARGV[3] - time() // ARGV[4] + $this->session, // ARGV[1] + $this->lockTTL, // ARGV[2] + time() // ARGV[3] ) ), - count( $keys ) # number of first argument(s) that are keys + count( $pathsByKey ) # number of first argument(s) that are keys ); } catch ( RedisException $e ) { $res = false; - $this->redisPool->handleException( $server, $conn, $e ); + $this->redisPool->handleError( $conn, $e ); } if ( $res === false ) { - foreach ( $paths as $path ) { + foreach ( array_merge( array_values( $pathsByType ) ) as $path ) { $status->fatal( 'lockmanager-fail-acquirelock', $path ); } } else { - $pathsByKey = array_combine( $keys, $paths ); foreach ( $res as $key ) { $status->fatal( 'lockmanager-fail-acquirelock', $pathsByKey[$key] ); } @@ -195,61 +169,66 @@ LUA; return $status; } - protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) { + protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { $status = Status::newGood(); $server = $this->lockServers[$lockSrv]; $conn = $this->redisPool->getConnection( $server ); if ( !$conn ) { - foreach ( $paths as $path ) { + foreach ( array_merge( array_values( $pathsByType ) ) as $path ) { $status->fatal( 'lockmanager-fail-releaselock', $path ); } + return $status; } - $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records + $pathsByKey = array(); // (type:hash => path) map + foreach ( $pathsByType as $type => $paths ) { + $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; + foreach ( $paths as $path ) { + $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; + } + } try { static $script = <<<LUA - if ARGV[1] ~= 'EX' and ARGV[1] ~= 'SH' then - return redis.error_reply('Unrecognized lock type given (must be EX or SH)') - end local failed = {} - for i,resourceKey in ipairs(KEYS) do - local released = redis.call('hDel',resourceKey,ARGV[1] .. ':' .. ARGV[2]) + -- Load input params (e.g. session) + local rSession = unpack(ARGV) + for i,requestKey in ipairs(KEYS) do + local _, _, rType, resourceKey = string.find(requestKey,"(%w+):(%w+)$") + local released = redis.call('hDel',resourceKey,rType .. ':' .. rSession) if released > 0 then -- Remove the whole structure if it is now empty if redis.call('hLen',resourceKey) == 0 then redis.call('del',resourceKey) end else - failed[#failed+1] = resourceKey + failed[#failed+1] = requestKey end end return failed LUA; $res = $conn->luaEval( $script, array_merge( - $keys, // KEYS[0], KEYS[1],...KEYS[N] + array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] array( - $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1] - $this->session // ARGV[2] + $this->session, // ARGV[1] ) ), - count( $keys ) # number of first argument(s) that are keys + count( $pathsByKey ) # number of first argument(s) that are keys ); } catch ( RedisException $e ) { $res = false; - $this->redisPool->handleException( $server, $conn, $e ); + $this->redisPool->handleError( $conn, $e ); } if ( $res === false ) { - foreach ( $paths as $path ) { + foreach ( array_merge( array_values( $pathsByType ) ) as $path ) { $status->fatal( 'lockmanager-fail-releaselock', $path ); } } else { - $pathsByKey = array_combine( $keys, $paths ); foreach ( $res as $key ) { $status->fatal( 'lockmanager-fail-releaselock', $pathsByKey[$key] ); } @@ -267,11 +246,13 @@ LUA; } /** - * @param $path string + * @param string $path + * @param string $type One of (EX,SH) * @return string */ - protected function recordKeyForPath( $path ) { - return implode( ':', array( __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ) ); + protected function recordKeyForPath( $path, $type ) { + return implode( ':', + array( __CLASS__, 'locks', "$type:" . $this->sha1Base36Absolute( $path ) ) ); } /** @@ -279,10 +260,13 @@ LUA; */ function __destruct() { while ( count( $this->locksHeld ) ) { + $pathsByType = array(); foreach ( $this->locksHeld as $path => $locks ) { - $this->doUnlock( array( $path ), self::LOCK_EX ); - $this->doUnlock( array( $path ), self::LOCK_SH ); + foreach ( $locks as $type => $count ) { + $pathsByType[$type][] = $path; + } } + $this->unlockByType( $pathsByType ); } } } diff --git a/includes/filebackend/lockmanager/ScopedLock.php b/includes/filebackend/lockmanager/ScopedLock.php index 5faad4a6..2056e101 100644 --- a/includes/filebackend/lockmanager/ScopedLock.php +++ b/includes/filebackend/lockmanager/ScopedLock.php @@ -34,9 +34,11 @@ class ScopedLock { /** @var LockManager */ protected $manager; + /** @var Status */ protected $status; - /** @var Array Map of lock types to resource paths */ + + /** @var array Map of lock types to resource paths */ protected $pathsByType; /** @@ -55,14 +57,13 @@ class ScopedLock { * Any locks are released once this object goes out of scope. * The status object is updated with any errors or warnings. * - * $type can be "mixed" and $paths can be a map of types to paths (since 1.22). - * Otherwise $type should be an integer and $paths should be a list of paths. - * * @param LockManager $manager * @param array $paths List of storage paths or map of lock types to path lists - * @param integer|string $type LockManager::LOCK_* constant or "mixed" + * @param int|string $type LockManager::LOCK_* constant or "mixed" and $paths + * can be a map of types to paths (since 1.22). Otherwise $type should be an + * integer and $paths should be a list of paths. * @param Status $status - * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.22) + * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.22) * @return ScopedLock|null Returns null on failure */ public static function factory( @@ -74,6 +75,7 @@ class ScopedLock { if ( $lockStatus->isOK() ) { return new self( $manager, $pathsByType, $status ); } + return null; } @@ -83,7 +85,6 @@ class ScopedLock { * This is the same as setting the lock object to null. * * @param ScopedLock $lock - * @return void * @since 1.21 */ public static function release( ScopedLock &$lock = null ) { |