From 4ac9fa081a7c045f6a9f1cfc529d82423f485b2e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Dec 2013 09:55:49 +0100 Subject: Update to MediaWiki 1.22.0 --- includes/filebackend/SwiftFileBackend.php | 262 ++++++++++++++---------------- 1 file changed, 124 insertions(+), 138 deletions(-) (limited to 'includes/filebackend/SwiftFileBackend.php') diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php index 0f3d97a3..db090a98 100644 --- a/includes/filebackend/SwiftFileBackend.php +++ b/includes/filebackend/SwiftFileBackend.php @@ -24,7 +24,7 @@ */ /** - * @brief Class for an OpenStack Swift based file backend. + * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend. * * This requires the SwiftCloudFiles MediaWiki extension, which includes * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles). @@ -104,7 +104,7 @@ class SwiftFileBackend extends FileBackendStore { */ public function __construct( array $config ) { parent::__construct( $config ); - if ( !MWInit::classExists( 'CF_Constants' ) ) { + if ( !class_exists( 'CF_Constants' ) ) { throw new MWException( 'SwiftCloudFiles extension not installed.' ); } // Required settings @@ -132,7 +132,7 @@ class SwiftFileBackend extends FileBackendStore { : false; $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] ) ? $config['swiftCDNExpiry'] - : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) + : 12 * 3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] ) ? $config['swiftCDNPurgable'] : true; @@ -172,10 +172,6 @@ class SwiftFileBackend extends FileBackendStore { return $relStoragePath; } - /** - * @see FileBackendStore::isPathUsableInternal() - * @return bool - */ public function isPathUsableInternal( $storagePath ) { list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); if ( $rel === null ) { @@ -193,6 +189,18 @@ class SwiftFileBackend extends FileBackendStore { return false; } + /** + * @param array $headers + * @return array + */ + protected function sanitizeHdrs( array $headers ) { + // By default, Swift has annoyingly low maximum header value limits + if ( isset( $headers['Content-Disposition'] ) ) { + $headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] ); + } + return $headers; + } + /** * @param string $disposition Content-Disposition header value * @return string Truncated Content-Disposition header value to meet Swift limits @@ -211,10 +219,6 @@ class SwiftFileBackend extends FileBackendStore { return $res; } - /** - * @see FileBackendStore::doCreateInternal() - * @return Status - */ protected function doCreateInternal( array $params ) { $status = Status::newGood(); @@ -248,17 +252,10 @@ class SwiftFileBackend extends FileBackendStore { // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5( $params['content'] ) ); // Use the same content type as StreamFile for security - $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - if ( !strlen( $obj->content_type ) ) { // special case - $obj->content_type = 'unknown/unknown'; - } - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); - } + $obj->content_type = $this->getContentType( $params['dst'], $params['content'], null ); // Set any other custom headers if requested if ( isset( $params['headers'] ) ) { - $obj->headers += $params['headers']; + $obj->headers += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $obj->write_async( $params['content'] ); @@ -290,10 +287,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doStoreInternal() - * @return Status - */ protected function doStoreInternal( array $params ) { $status = Status::newGood(); @@ -333,17 +326,10 @@ class SwiftFileBackend extends FileBackendStore { // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5_file( $params['src'] ) ); // Use the same content type as StreamFile for security - $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - if ( !strlen( $obj->content_type ) ) { // special case - $obj->content_type = 'unknown/unknown'; - } - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); - } + $obj->content_type = $this->getContentType( $params['dst'], null, $params['src'] ); // Set any other custom headers if requested if ( isset( $params['headers'] ) ) { - $obj->headers += $params['headers']; + $obj->headers += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred wfSuppressWarnings(); @@ -387,10 +373,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doCopyInternal() - * @return Status - */ protected function doCopyInternal( array $params ) { $status = Status::newGood(); @@ -424,8 +406,9 @@ class SwiftFileBackend extends FileBackendStore { try { $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD $hdrs = array(); // source file headers to override with new values - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $hdrs += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); @@ -459,10 +442,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doMoveInternal() - * @return Status - */ protected function doMoveInternal( array $params ) { $status = Status::newGood(); @@ -497,8 +476,9 @@ class SwiftFileBackend extends FileBackendStore { $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD $hdrs = array(); // source file headers to override with new values - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $hdrs += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); @@ -534,10 +514,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doDeleteInternal() - * @return Status - */ protected function doDeleteInternal( array $params ) { $status = Status::newGood(); @@ -590,10 +566,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doDescribeInternal() - * @return Status - */ protected function doDescribeInternal( array $params ) { $status = Status::newGood(); @@ -603,19 +575,15 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - $hdrs = isset( $params['headers'] ) ? $params['headers'] : array(); - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); - } - try { $sContObj = $this->getContainer( $srcCont ); // Get the latest version of the current metadata $srcObj = $sContObj->get_object( $srcRel, $this->headersFromParams( array( 'latest' => true ) ) ); // Merge in the metadata updates... - $srcObj->headers = $hdrs + $srcObj->headers; + if ( isset( $params['headers'] ) ) { + $srcObj->headers = $this->sanitizeHdrs( $params['headers'] ) + $srcObj->headers; + } $srcObj->sync_metadata(); // save to Swift $this->purgeCDNCache( array( $srcObj ) ); } catch ( CDNNotEnabledException $e ) { @@ -631,10 +599,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doPrepareInternal() - * @return Status - */ protected function doPrepareInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -748,10 +712,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doCleanInternal() - * @return Status - */ protected function doCleanInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -787,10 +747,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doFileExists() - * @return array|bool|null - */ protected function doGetFileStat( array $params ) { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { @@ -805,8 +761,8 @@ class SwiftFileBackend extends FileBackendStore { $stat = array( // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ), - 'size' => (int)$srcObj->content_length, - 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) + 'size' => (int)$srcObj->content_length, + 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) ); } catch ( NoSuchContainerException $e ) { } catch ( NoSuchObjectException $e ) { @@ -821,7 +777,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Fill in any missing object metadata and save it to Swift * - * @param $obj CF_Object + * @param CF_Object $obj * @param string $path Storage path to object * @return bool Success * @throws Exception cloudfiles exceptions @@ -852,10 +808,6 @@ class SwiftFileBackend extends FileBackendStore { return false; // failed } - /** - * @see FileBackendStore::doGetFileContentsMulti() - * @return Array - */ protected function doGetFileContentsMulti( array $params ) { $contents = array(); @@ -965,24 +917,25 @@ class SwiftFileBackend extends FileBackendStore { * @param string $fullCont Resolved container name * @param string $dir Resolved storage directory with no trailing slash * @param string|null $after Storage path of file to list items after - * @param $limit integer Max number of items to list - * @param array $params Includes flag for 'topOnly' - * @return Array List of relative paths of dirs directly under $dir + * @param integer $limit Max number of items to list + * @param array $params Parameters for getDirectoryList() + * @return Array List of resolved paths of directories directly under $dir + * @throws FileBackendError */ public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { $dirs = array(); if ( $after === INF ) { return $dirs; // nothing more } - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); try { $container = $this->getContainer( $fullCont ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // Non-recursive: only list dirs right under $dir if ( !empty( $params['topOnly'] ) ) { $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { // files and dirs + foreach ( $objects as $object ) { // files and directories if ( substr( $object, -1 ) === '/' ) { $dirs[] = $object; // directories end in '/' } @@ -1013,6 +966,7 @@ class SwiftFileBackend extends FileBackendStore { } } } + // Page on the unfiltered directory listing (what is returned may be filtered) if ( count( $objects ) < $limit ) { $after = INF; // avoid a second RTT } else { @@ -1022,9 +976,9 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, null, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) ); + throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); return $dirs; } @@ -1038,33 +992,49 @@ class SwiftFileBackend extends FileBackendStore { * @param string $fullCont Resolved container name * @param string $dir Resolved storage directory with no trailing slash * @param string|null $after Storage path of file to list items after - * @param $limit integer Max number of items to list - * @param array $params Includes flag for 'topOnly' - * @return Array List of relative paths of files under $dir + * @param integer $limit Max number of items to list + * @param array $params Parameters for getDirectoryList() + * @return Array List of resolved paths of files under $dir + * @throws FileBackendError */ public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { $files = array(); if ( $after === INF ) { return $files; // nothing more } - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); try { $container = $this->getContainer( $fullCont ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // Non-recursive: only list files right under $dir if ( !empty( $params['topOnly'] ) ) { // files and dirs - $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { - if ( substr( $object, -1 ) !== '/' ) { - $files[] = $object; // directories end in '/' + if ( !empty( $params['adviseStat'] ) ) { + $limit = min( $limit, self::CACHE_CHEAP_SIZE ); + // Note: get_objects() does not include directories + $objects = $this->loadObjectListing( $params, $dir, + $container->get_objects( $limit, $after, $prefix, null, '/' ) ); + $files = $objects; + } else { + $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); + foreach ( $objects as $object ) { // files and directories + if ( substr( $object, -1 ) !== '/' ) { + $files[] = $object; // directories end in '/' + } } } // Recursive: list all files under $dir and its subdirs } else { // files - $objects = $container->list_objects( $limit, $after, $prefix ); + if ( !empty( $params['adviseStat'] ) ) { + $limit = min( $limit, self::CACHE_CHEAP_SIZE ); + $objects = $this->loadObjectListing( $params, $dir, + $container->get_objects( $limit, $after, $prefix ) ); + } else { + $objects = $container->list_objects( $limit, $after, $prefix ); + } $files = $objects; } + // Page on the unfiltered object listing (what is returned may be filtered) if ( count( $objects ) < $limit ) { $after = INF; // avoid a second RTT } else { @@ -1074,29 +1044,57 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, null, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) ); + throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); return $files; } /** - * @see FileBackendStore::doGetFileSha1base36() - * @return bool + * Load a list of objects that belong under $dir into stat cache + * and return a list of the names of the objects in the same order. + * + * @param array $params Parameters for getDirectoryList() + * @param string $dir Resolved container directory path + * @param array $cfObjects List of CF_Object items + * @return array List of object names */ + private function loadObjectListing( array $params, $dir, array $cfObjects ) { + $names = array(); + $storageDir = rtrim( $params['dir'], '/' ); + $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/" + // Iterate over the list *backwards* as this primes the stat cache, which is LRU. + // If this fills the cache and the caller stats an uncached file before stating + // the ones on the listing, there would be zero cache hits if this went forwards. + for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) { + $object = current( $cfObjects ); + $path = "{$storageDir}/" . substr( $object->name, $suffixStart ); + $val = array( + // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW + 'mtime' => wfTimestamp( TS_MW, $object->last_modified ), + 'size' => (int)$object->content_length, + 'latest' => false // eventually consistent + ); + $this->cheapCache->set( $path, 'stat', $val ); + $names[] = $object->name; + } + return array_reverse( $names ); // keep the paths in original order + } + protected function doGetFileSha1base36( array $params ) { $stat = $this->getFileStat( $params ); if ( $stat ) { + if ( !isset( $stat['sha1'] ) ) { + // Stat entries filled by file listings don't include SHA1 + $this->clearCache( array( $params['src'] ) ); + $stat = $this->getFileStat( $params ); + } return $stat['sha1']; } else { return false; } } - /** - * @see FileBackendStore::doStreamFile() - * @return Status - */ protected function doStreamFile( array $params ) { $status = Status::newGood(); @@ -1128,10 +1126,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doGetLocalCopyMulti() - * @return null|TempFSFile - */ protected function doGetLocalCopyMulti( array $params ) { $tmpFiles = array(); @@ -1201,10 +1195,6 @@ class SwiftFileBackend extends FileBackendStore { return $tmpFiles; } - /** - * @see FileBackendStore::getFileHttpUrl() - * @return string|null - */ public function getFileHttpUrl( array $params ) { if ( $this->swiftTempUrlKey != '' || ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) ) @@ -1237,8 +1227,8 @@ class SwiftFileBackend extends FileBackendStore { str_replace( '/swift/v1', '', // S3 API is the rgw default $sContObj->cfs_http->getStorageUrl() . $spath ), array( - 'Signature' => $signature, - 'Expires' => $expires, + 'Signature' => $signature, + 'Expires' => $expires, 'AWSAccessKeyId' => $this->rgwS3AccessKey ) ); } @@ -1250,10 +1240,6 @@ class SwiftFileBackend extends FileBackendStore { return null; } - /** - * @see FileBackendStore::directoriesAreVirtual() - * @return bool - */ protected function directoriesAreVirtual() { return true; } @@ -1274,10 +1260,6 @@ class SwiftFileBackend extends FileBackendStore { return $hdrs; } - /** - * @see FileBackendStore::doExecuteOpHandlesInternal() - * @return Array List of corresponding Status objects - */ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { $statuses = array(); @@ -1314,7 +1296,7 @@ class SwiftFileBackend extends FileBackendStore { * matches the expression and the request is not for a listing. * Setting this to '*' effectively makes a container public. * -".rlistings:" : Grants access if the request is from a referrer host that - * matches the expression and the request for a listing. + * matches the expression and the request is for a listing. * * $writeGrps is a list of the possible criteria for a request to have * access to write to a container. Each item is of the following format: @@ -1325,7 +1307,7 @@ class SwiftFileBackend extends FileBackendStore { * In general, we don't allow listings to end-users. It's not useful, isn't well-defined * (lists are truncated to 10000 item with no way to page), and is just a performance risk. * - * @param $contObj CF_Container Swift container + * @param CF_Container $contObj Swift container * @param array $readGrps List of read access routes * @param array $writeGrps List of write access routes * @return Status @@ -1395,12 +1377,12 @@ class SwiftFileBackend extends FileBackendStore { if ( is_array( $creds ) ) { // cache hit $this->auth->load_cached_credentials( $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] ); - $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case + $this->sessionStarted = time() - ceil( $this->authTTL / 2 ); // skew for worst case } else { // cache miss try { $this->auth->authenticate(); $creds = $this->auth->export_credentials(); - $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache + $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL / 2 ) ); // cache $this->sessionStarted = time(); } catch ( CloudFilesException $e ) { $this->connException = $e; // don't keep re-trying @@ -1433,7 +1415,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Get the cache key for a container * - * @param $username string + * @param string $username * @return string */ private function getCredsCacheKey( $username ) { @@ -1496,10 +1478,6 @@ class SwiftFileBackend extends FileBackendStore { $conn->delete_container( $container ); } - /** - * @see FileBackendStore::doPrimeContainerCache() - * @return void - */ protected function doPrimeContainerCache( array $containerInfo ) { try { $conn = $this->getConnection(); // Swift proxy connection @@ -1517,9 +1495,9 @@ class SwiftFileBackend extends FileBackendStore { * Log an unexpected exception for this backend. * This also sets the Status object to have a fatal error. * - * @param $e Exception - * @param $status Status|null - * @param $func string + * @param Exception $e + * @param Status $status|null + * @param string $func * @param array $params * @return void */ @@ -1554,7 +1532,15 @@ class SwiftFileOpHandle extends FileBackendStoreOpHandle { /** @var Array */ public $affectedObjects = array(); - public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) { + /** + * @param SwiftFileBackend $backend + * @param array $params + * @param string $call + * @param CF_Async_Op $cfOp + */ + public function __construct( + SwiftFileBackend $backend, array $params, $call, CF_Async_Op $cfOp + ) { $this->backend = $backend; $this->params = $params; $this->call = $call; @@ -1586,7 +1572,7 @@ abstract class SwiftFileBackendList implements Iterator { const PAGE_SIZE = 9000; // file listing buffer size /** - * @param $backend SwiftFileBackend + * @param SwiftFileBackend $backend * @param string $fullCont Resolved container name * @param string $dir Resolved directory relative to container * @param array $params @@ -1660,10 +1646,10 @@ abstract class SwiftFileBackendList implements Iterator { * * @param string $container Resolved container name * @param string $dir Resolved path relative to container - * @param $after string|null - * @param $limit integer + * @param string $after|null + * @param integer $limit * @param array $params - * @return Traversable|Array|null Returns null on failure + * @return Traversable|Array */ abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); } @@ -1682,7 +1668,7 @@ class SwiftFileBackendDirList extends SwiftFileBackendList { /** * @see SwiftFileBackendList::pageFromList() - * @return Array|null + * @return Array */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); @@ -1703,7 +1689,7 @@ class SwiftFileBackendFileList extends SwiftFileBackendList { /** * @see SwiftFileBackendList::pageFromList() - * @return Array|null + * @return Array */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); -- cgit v1.2.3-54-g00ecf