diff options
Diffstat (limited to 'includes/filebackend/SwiftFileBackend.php')
| -rw-r--r-- | includes/filebackend/SwiftFileBackend.php | 687 |
1 files changed, 420 insertions, 267 deletions
diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php index b6f0aa60..db090a98 100644 --- a/includes/filebackend/SwiftFileBackend.php +++ b/includes/filebackend/SwiftFileBackend.php @@ -24,7 +24,7 @@ */ /** - * @brief Class for an OpenStack Swift based file backend. + * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend. * * This requires the SwiftCloudFiles MediaWiki extension, which includes * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles). @@ -40,11 +40,16 @@ class SwiftFileBackend extends FileBackendStore { /** @var CF_Authentication */ protected $auth; // Swift authentication handler protected $authTTL; // integer seconds + protected $swiftTempUrlKey; // string; shared secret value for making temp urls protected $swiftAnonUser; // string; username to handle unauthenticated requests protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled protected $swiftCDNExpiry; // integer; how long to cache things in the CDN protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled + // Rados Gateway specific options + protected $rgwS3AccessKey; // string; S3 access key + protected $rgwS3SecretKey; // string; S3 authentication key + /** @var CF_Connection */ protected $conn; // Swift connection handle protected $sessionStarted = 0; // integer UNIX timestamp @@ -66,6 +71,8 @@ class SwiftFileBackend extends FileBackendStore { * - swiftUser : Swift user used by MediaWiki (account:username) * - swiftKey : Swift authentication key for the above user * - swiftAuthTTL : Swift authentication TTL (seconds) + * - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account. + * Do not set this until it has been set in the backend. * - swiftAnonUser : Swift user used for end-user requests (account:username). * If set, then views of public containers are assumed to go * through this user. If not set, then public containers are @@ -84,10 +91,20 @@ class SwiftFileBackend extends FileBackendStore { * - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect. * If those are not available, then the main cache will be used. * This is probably insecure in shared hosting environments. + * - rgwS3AccessKey : Ragos Gateway S3 "access key" value on the account. + * Do not set this until it has been set in the backend. + * This is used for generating expiring pre-authenticated URLs. + * Only use this when using rgw and to work around + * http://tracker.newdream.net/issues/3454. + * - rgwS3SecretKey : Ragos Gateway S3 "secret key" value on the account. + * Do not set this until it has been set in the backend. + * This is used for generating expiring pre-authenticated URLs. + * Only use this when using rgw and to work around + * http://tracker.newdream.net/issues/3454. */ public function __construct( array $config ) { parent::__construct( $config ); - if ( !MWInit::classExists( 'CF_Constants' ) ) { + if ( !class_exists( 'CF_Constants' ) ) { throw new MWException( 'SwiftCloudFiles extension not installed.' ); } // Required settings @@ -104,6 +121,9 @@ class SwiftFileBackend extends FileBackendStore { $this->swiftAnonUser = isset( $config['swiftAnonUser'] ) ? $config['swiftAnonUser'] : ''; + $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] ) + ? $config['swiftTempUrlKey'] + : ''; $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) ? $config['shardViaHashLevels'] : ''; @@ -112,17 +132,23 @@ class SwiftFileBackend extends FileBackendStore { : false; $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] ) ? $config['swiftCDNExpiry'] - : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) + : 12 * 3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] ) ? $config['swiftCDNPurgable'] : true; + $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] ) + ? $config['rgwS3AccessKey'] + : ''; + $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] ) + ? $config['rgwS3SecretKey'] + : ''; // Cache container information to mask latency $this->memCache = wfGetMainCache(); // Process cache for container info $this->connContainerCache = new ProcessCacheLRU( 300 ); // Cache auth token information to avoid RTTs if ( !empty( $config['cacheAuthInfo'] ) ) { - if ( php_sapi_name() === 'cli' ) { + if ( PHP_SAPI === 'cli' ) { $this->srvCache = wfGetMainCache(); // preferrably memcached } else { try { // look for APC, XCache, WinCache, ect... @@ -146,10 +172,6 @@ class SwiftFileBackend extends FileBackendStore { return $relStoragePath; } - /** - * @see FileBackendStore::isPathUsableInternal() - * @return bool - */ public function isPathUsableInternal( $storagePath ) { list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); if ( $rel === null ) { @@ -168,14 +190,26 @@ class SwiftFileBackend extends FileBackendStore { } /** - * @param $disposition string Content-Disposition header value + * @param array $headers + * @return array + */ + protected function sanitizeHdrs( array $headers ) { + // By default, Swift has annoyingly low maximum header value limits + if ( isset( $headers['Content-Disposition'] ) ) { + $headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] ); + } + return $headers; + } + + /** + * @param string $disposition Content-Disposition header value * @return string Truncated Content-Disposition header value to meet Swift limits */ protected function truncDisp( $disposition ) { $res = ''; foreach ( explode( ';', $disposition ) as $part ) { $part = trim( $part ); - $new = ( $res === '' ) ? $part : "{$res};{$part}"; + $new = ( $res === '' ) ? $part : "{$res};{$part}"; if ( strlen( $new ) <= 255 ) { $res = $new; } else { @@ -185,10 +219,6 @@ class SwiftFileBackend extends FileBackendStore { return $res; } - /** - * @see FileBackendStore::doCreateInternal() - * @return Status - */ protected function doCreateInternal( array $params ) { $status = Status::newGood(); @@ -201,12 +231,6 @@ class SwiftFileBackend extends FileBackendStore { // (a) Check the destination container and object try { $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-create', $params['dst'] ); return $status; @@ -223,31 +247,23 @@ class SwiftFileBackend extends FileBackendStore { // Create a fresh CF_Object with no fields preloaded. // We don't want to preserve headers, metadata, and such. $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - // Note: metadata keys stored as [Upper case char][[Lower case char]...] - $obj->metadata = array( 'Sha1base36' => $sha1Hash ); + $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59). // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5( $params['content'] ) ); // Use the same content type as StreamFile for security - $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - if ( !strlen( $obj->content_type ) ) { // special case - $obj->content_type = 'unknown/unknown'; - } - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + $obj->content_type = $this->getContentType( $params['dst'], $params['content'], null ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $obj->headers += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $obj->write_async( $params['content'] ); $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $obj; - } + $status->value->affectedObjects[] = $obj; } else { // actually write the object in Swift $obj->write( $params['content'] ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $obj ) ); - } + $this->purgeCDNCache( array( $obj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here @@ -271,10 +287,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doStoreInternal() - * @return Status - */ protected function doStoreInternal( array $params ) { $status = Status::newGood(); @@ -287,12 +299,6 @@ class SwiftFileBackend extends FileBackendStore { // (a) Check the destination container and object try { $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); return $status; @@ -302,7 +308,9 @@ class SwiftFileBackend extends FileBackendStore { } // (b) Get a SHA-1 hash of the object + wfSuppressWarnings(); $sha1Hash = sha1_file( $params['src'] ); + wfRestoreWarnings(); if ( $sha1Hash === false ) { // source doesn't exist? $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); return $status; @@ -314,18 +322,14 @@ class SwiftFileBackend extends FileBackendStore { // Create a fresh CF_Object with no fields preloaded. // We don't want to preserve headers, metadata, and such. $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - // Note: metadata keys stored as [Upper case char][[Lower case char]...] - $obj->metadata = array( 'Sha1base36' => $sha1Hash ); + $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5_file( $params['src'] ) ); // Use the same content type as StreamFile for security - $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - if ( !strlen( $obj->content_type ) ) { // special case - $obj->content_type = 'unknown/unknown'; - } - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + $obj->content_type = $this->getContentType( $params['dst'], null, $params['src'] ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $obj->headers += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred wfSuppressWarnings(); @@ -337,15 +341,11 @@ class SwiftFileBackend extends FileBackendStore { $op = $obj->write_async( $fp, filesize( $params['src'] ), true ); $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op ); $status->value->resourcesToClose[] = $fp; - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $obj; - } + $status->value->affectedObjects[] = $obj; } } else { // actually write the object in Swift $obj->load_from_filename( $params['src'], true ); // calls $obj->write() - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $obj ) ); - } + $this->purgeCDNCache( array( $obj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here @@ -373,10 +373,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doCopyInternal() - * @return Status - */ protected function doCopyInternal( array $params ) { $status = Status::newGood(); @@ -396,14 +392,10 @@ class SwiftFileBackend extends FileBackendStore { try { $sContObj = $this->getContainer( $srcCont ); $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } return $status; } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); @@ -414,25 +406,24 @@ class SwiftFileBackend extends FileBackendStore { try { $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD $hdrs = array(); // source file headers to override with new values - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $hdrs += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $dstObj; - } + $status->value->affectedObjects[] = $dstObj; } else { // actually write the object in Swift $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $dstObj ) ); - } + $this->purgeCDNCache( array( $dstObj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here } catch ( NoSuchObjectException $e ) { // source object does not exist - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); } @@ -451,10 +442,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doMoveInternal() - * @return Status - */ protected function doMoveInternal( array $params ) { $status = Status::newGood(); @@ -474,14 +461,10 @@ class SwiftFileBackend extends FileBackendStore { try { $sContObj = $this->getContainer( $srcCont ); $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + } return $status; } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); @@ -493,27 +476,26 @@ class SwiftFileBackend extends FileBackendStore { $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD $hdrs = array(); // source file headers to override with new values - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $hdrs += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op ); $status->value->affectedObjects[] = $srcObj; - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $dstObj; - } + $status->value->affectedObjects[] = $dstObj; } else { // actually write the object in Swift $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); $this->purgeCDNCache( array( $srcObj ) ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $dstObj ) ); - } + $this->purgeCDNCache( array( $dstObj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here } catch ( NoSuchObjectException $e ) { // source object does not exist - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + } } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); } @@ -532,10 +514,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doDeleteInternal() - * @return Status - */ protected function doDeleteInternal( array $params ) { $status = Status::newGood(); @@ -559,7 +537,9 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } } catch ( NoSuchObjectException $e ) { if ( empty( $params['ignoreMissingSource'] ) ) { $status->fatal( 'backend-fail-delete', $params['src'] ); @@ -586,16 +566,45 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doPrepareInternal() - * @return Status - */ + protected function doDescribeInternal( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + try { + $sContObj = $this->getContainer( $srcCont ); + // Get the latest version of the current metadata + $srcObj = $sContObj->get_object( $srcRel, + $this->headersFromParams( array( 'latest' => true ) ) ); + // Merge in the metadata updates... + if ( isset( $params['headers'] ) ) { + $srcObj->headers = $this->sanitizeHdrs( $params['headers'] ) + $srcObj->headers; + } + $srcObj->sync_metadata(); // save to Swift + $this->purgeCDNCache( array( $srcObj ) ); + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-describe', $params['src'] ); + } catch ( NoSuchObjectException $e ) { + $status->fatal( 'backend-fail-describe', $params['src'] ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + + return $status; + } + protected function doPrepareInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); // (a) Check if container already exists try { - $contObj = $this->getContainer( $fullCont ); + $this->getContainer( $fullCont ); // NoSuchContainerException not thrown: container must exist return $status; // already exists } catch ( NoSuchContainerException $e ) { @@ -703,10 +712,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doCleanInternal() - * @return Status - */ protected function doCleanInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -742,10 +747,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doFileExists() - * @return array|bool|null - */ protected function doGetFileStat( array $params ) { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { @@ -760,8 +761,8 @@ class SwiftFileBackend extends FileBackendStore { $stat = array( // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ), - 'size' => (int)$srcObj->content_length, - 'sha1' => $srcObj->metadata['Sha1base36'] + 'size' => (int)$srcObj->content_length, + 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) ); } catch ( NoSuchContainerException $e ) { } catch ( NoSuchObjectException $e ) { @@ -776,61 +777,103 @@ class SwiftFileBackend extends FileBackendStore { /** * Fill in any missing object metadata and save it to Swift * - * @param $obj CF_Object - * @param $path string Storage path to object + * @param CF_Object $obj + * @param string $path Storage path to object * @return bool Success * @throws Exception cloudfiles exceptions */ protected function addMissingMetadata( CF_Object $obj, $path ) { - if ( isset( $obj->metadata['Sha1base36'] ) ) { + if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) { return true; // nothing to do } wfProfileIn( __METHOD__ ); + trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING ); $status = Status::newGood(); $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status ); if ( $status->isOK() ) { - # Do not stat the file in getLocalCopy() to avoid infinite loops - $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) ); + $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) ); if ( $tmpFile ) { $hash = $tmpFile->getSha1Base36(); if ( $hash !== false ) { - $obj->metadata['Sha1base36'] = $hash; + $obj->setMetadataValues( array( 'Sha1base36' => $hash ) ); $obj->sync_metadata(); // save to Swift wfProfileOut( __METHOD__ ); return true; // success } } } - $obj->metadata['Sha1base36'] = false; + trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING ); + $obj->setMetadataValues( array( 'Sha1base36' => false ) ); wfProfileOut( __METHOD__ ); return false; // failed } - /** - * @see FileBackend::getFileContents() - * @return bool|null|string - */ - public function getFileContents( array $params ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - return false; // invalid storage path - } + protected function doGetFileContentsMulti( array $params ) { + $contents = array(); - if ( !$this->fileExists( $params ) ) { - return null; - } + $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging + // Blindly create tmp files and stream to them, catching any exception if the file does + // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata(). + foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { + $cfOps = array(); // (path => CF_Async_Op) - $data = false; - try { - $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - $data = $obj->read( $this->headersFromParams( $params ) ); - } catch ( NoSuchContainerException $e ) { - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, $params ); + foreach ( $pathBatch as $path ) { // each path in this concurrent batch + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null ) { + $contents[$path] = false; + continue; + } + $data = false; + try { + $sContObj = $this->getContainer( $srcCont ); + $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD + // Create a new temporary memory file... + $handle = fopen( 'php://temp', 'wb' ); + if ( $handle ) { + $headers = $this->headersFromParams( $params ); + if ( count( $pathBatch ) > 1 ) { + $cfOps[$path] = $obj->stream_async( $handle, $headers ); + $cfOps[$path]->_file_handle = $handle; // close this later + } else { + $obj->stream( $handle, $headers ); + rewind( $handle ); // start from the beginning + $data = stream_get_contents( $handle ); + fclose( $handle ); + } + } else { + $data = false; + } + } catch ( NoSuchContainerException $e ) { + $data = false; + } catch ( NoSuchObjectException $e ) { + $data = false; + } catch ( CloudFilesException $e ) { // some other exception? + $data = false; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + } + $contents[$path] = $data; + } + + $batch = new CF_Async_Op_Batch( $cfOps ); + $cfOps = $batch->execute(); + foreach ( $cfOps as $path => $cfOp ) { + try { + $cfOp->getLastResponse(); + rewind( $cfOp->_file_handle ); // start from the beginning + $contents[$path] = stream_get_contents( $cfOp->_file_handle ); + } catch ( NoSuchContainerException $e ) { + $contents[$path] = false; + } catch ( NoSuchObjectException $e ) { + $contents[$path] = false; + } catch ( CloudFilesException $e ) { // some other exception? + $contents[$path] = false; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + } + fclose( $cfOp->_file_handle ); // close open handle + } } - return $data; + return $contents; } /** @@ -871,27 +914,28 @@ class SwiftFileBackend extends FileBackendStore { /** * Do not call this function outside of SwiftFileBackendFileList * - * @param $fullCont string Resolved container name - * @param $dir string Resolved storage directory with no trailing slash - * @param $after string|null Storage path of file to list items after - * @param $limit integer Max number of items to list - * @param $params Array Includes flag for 'topOnly' - * @return Array List of relative paths of dirs directly under $dir + * @param string $fullCont Resolved container name + * @param string $dir Resolved storage directory with no trailing slash + * @param string|null $after Storage path of file to list items after + * @param integer $limit Max number of items to list + * @param array $params Parameters for getDirectoryList() + * @return Array List of resolved paths of directories directly under $dir + * @throws FileBackendError */ public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { $dirs = array(); if ( $after === INF ) { return $dirs; // nothing more } - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); try { $container = $this->getContainer( $fullCont ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // Non-recursive: only list dirs right under $dir if ( !empty( $params['topOnly'] ) ) { $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { // files and dirs + foreach ( $objects as $object ) { // files and directories if ( substr( $object, -1 ) === '/' ) { $dirs[] = $object; // directories end in '/' } @@ -903,7 +947,7 @@ class SwiftFileBackend extends FileBackendStore { $objects = $container->list_objects( $limit, $after, $prefix ); foreach ( $objects as $object ) { // files $objectDir = $this->getParentDir( $object ); // directory of object - if ( $objectDir !== false ) { // file has a parent dir + if ( $objectDir !== false && $objectDir !== $dir ) { // Swift stores paths in UTF-8, using binary sorting. // See function "create_container_table" in common/db.py. // If a directory is not "greater" than the last one, @@ -922,6 +966,7 @@ class SwiftFileBackend extends FileBackendStore { } } } + // Page on the unfiltered directory listing (what is returned may be filtered) if ( count( $objects ) < $limit ) { $after = INF; // avoid a second RTT } else { @@ -931,9 +976,9 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, null, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) ); + throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); return $dirs; } @@ -944,36 +989,52 @@ class SwiftFileBackend extends FileBackendStore { /** * Do not call this function outside of SwiftFileBackendFileList * - * @param $fullCont string Resolved container name - * @param $dir string Resolved storage directory with no trailing slash - * @param $after string|null Storage path of file to list items after - * @param $limit integer Max number of items to list - * @param $params Array Includes flag for 'topOnly' - * @return Array List of relative paths of files under $dir + * @param string $fullCont Resolved container name + * @param string $dir Resolved storage directory with no trailing slash + * @param string|null $after Storage path of file to list items after + * @param integer $limit Max number of items to list + * @param array $params Parameters for getDirectoryList() + * @return Array List of resolved paths of files under $dir + * @throws FileBackendError */ public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { $files = array(); if ( $after === INF ) { return $files; // nothing more } - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); try { $container = $this->getContainer( $fullCont ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // Non-recursive: only list files right under $dir if ( !empty( $params['topOnly'] ) ) { // files and dirs - $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { - if ( substr( $object, -1 ) !== '/' ) { - $files[] = $object; // directories end in '/' + if ( !empty( $params['adviseStat'] ) ) { + $limit = min( $limit, self::CACHE_CHEAP_SIZE ); + // Note: get_objects() does not include directories + $objects = $this->loadObjectListing( $params, $dir, + $container->get_objects( $limit, $after, $prefix, null, '/' ) ); + $files = $objects; + } else { + $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); + foreach ( $objects as $object ) { // files and directories + if ( substr( $object, -1 ) !== '/' ) { + $files[] = $object; // directories end in '/' + } } } // Recursive: list all files under $dir and its subdirs } else { // files - $objects = $container->list_objects( $limit, $after, $prefix ); + if ( !empty( $params['adviseStat'] ) ) { + $limit = min( $limit, self::CACHE_CHEAP_SIZE ); + $objects = $this->loadObjectListing( $params, $dir, + $container->get_objects( $limit, $after, $prefix ) ); + } else { + $objects = $container->list_objects( $limit, $after, $prefix ); + } $files = $objects; } + // Page on the unfiltered object listing (what is returned may be filtered) if ( count( $objects ) < $limit ) { $after = INF; // avoid a second RTT } else { @@ -983,29 +1044,57 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, null, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) ); + throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); return $files; } /** - * @see FileBackendStore::doGetFileSha1base36() - * @return bool + * Load a list of objects that belong under $dir into stat cache + * and return a list of the names of the objects in the same order. + * + * @param array $params Parameters for getDirectoryList() + * @param string $dir Resolved container directory path + * @param array $cfObjects List of CF_Object items + * @return array List of object names */ + private function loadObjectListing( array $params, $dir, array $cfObjects ) { + $names = array(); + $storageDir = rtrim( $params['dir'], '/' ); + $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/" + // Iterate over the list *backwards* as this primes the stat cache, which is LRU. + // If this fills the cache and the caller stats an uncached file before stating + // the ones on the listing, there would be zero cache hits if this went forwards. + for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) { + $object = current( $cfObjects ); + $path = "{$storageDir}/" . substr( $object->name, $suffixStart ); + $val = array( + // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW + 'mtime' => wfTimestamp( TS_MW, $object->last_modified ), + 'size' => (int)$object->content_length, + 'latest' => false // eventually consistent + ); + $this->cheapCache->set( $path, 'stat', $val ); + $names[] = $object->name; + } + return array_reverse( $names ); // keep the paths in original order + } + protected function doGetFileSha1base36( array $params ) { $stat = $this->getFileStat( $params ); if ( $stat ) { + if ( !isset( $stat['sha1'] ) ) { + // Stat entries filled by file listings don't include SHA1 + $this->clearCache( array( $params['src'] ) ); + $stat = $this->getFileStat( $params ); + } return $stat['sha1']; } else { return false; } } - /** - * @see FileBackendStore::doStreamFile() - * @return Status - */ protected function doStreamFile( array $params ) { $status = Status::newGood(); @@ -1037,51 +1126,120 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::getLocalCopy() - * @return null|TempFSFile - */ - public function getLocalCopy( array $params ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - return null; - } + protected function doGetLocalCopyMulti( array $params ) { + $tmpFiles = array(); - // Blindly create a tmp file and stream to it, catching any exception if the file does - // not exist. Also, doing a stat here will cause infinite loops when filling metadata. - $tmpFile = null; - try { - $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - // Get source file extension - $ext = FileBackend::extensionFromPath( $srcRel ); - // Create a new temporary file... - $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); - if ( $tmpFile ) { - $handle = fopen( $tmpFile->getPath(), 'wb' ); - if ( $handle ) { - $obj->stream( $handle, $this->headersFromParams( $params ) ); - fclose( $handle ); - } else { - $tmpFile = null; // couldn't open temp file + $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging + // Blindly create tmp files and stream to them, catching any exception if the file does + // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). + foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { + $cfOps = array(); // (path => CF_Async_Op) + + foreach ( $pathBatch as $path ) { // each path in this concurrent batch + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null ) { + $tmpFiles[$path] = null; + continue; + } + $tmpFile = null; + try { + $sContObj = $this->getContainer( $srcCont ); + $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD + // Get source file extension + $ext = FileBackend::extensionFromPath( $path ); + // Create a new temporary file... + $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); + if ( $tmpFile ) { + $handle = fopen( $tmpFile->getPath(), 'wb' ); + if ( $handle ) { + $headers = $this->headersFromParams( $params ); + if ( count( $pathBatch ) > 1 ) { + $cfOps[$path] = $obj->stream_async( $handle, $headers ); + $cfOps[$path]->_file_handle = $handle; // close this later + } else { + $obj->stream( $handle, $headers ); + fclose( $handle ); + } + } else { + $tmpFile = null; + } + } + } catch ( NoSuchContainerException $e ) { + $tmpFile = null; + } catch ( NoSuchObjectException $e ) { + $tmpFile = null; + } catch ( CloudFilesException $e ) { // some other exception? + $tmpFile = null; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); } + $tmpFiles[$path] = $tmpFile; + } + + $batch = new CF_Async_Op_Batch( $cfOps ); + $cfOps = $batch->execute(); + foreach ( $cfOps as $path => $cfOp ) { + try { + $cfOp->getLastResponse(); + } catch ( NoSuchContainerException $e ) { + $tmpFiles[$path] = null; + } catch ( NoSuchObjectException $e ) { + $tmpFiles[$path] = null; + } catch ( CloudFilesException $e ) { // some other exception? + $tmpFiles[$path] = null; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + } + fclose( $cfOp->_file_handle ); // close open handle } - } catch ( NoSuchContainerException $e ) { - $tmpFile = null; - } catch ( NoSuchObjectException $e ) { - $tmpFile = null; - } catch ( CloudFilesException $e ) { // some other exception? - $tmpFile = null; - $this->handleException( $e, null, __METHOD__, $params ); } - return $tmpFile; + return $tmpFiles; + } + + public function getFileHttpUrl( array $params ) { + if ( $this->swiftTempUrlKey != '' || + ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) ) + { + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + return null; // invalid path + } + try { + $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400; + $sContObj = $this->getContainer( $srcCont ); + $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD + if ( $this->swiftTempUrlKey != '' ) { + return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" ); + } else { // give S3 API URL for rgw + $expires = time() + $ttl; + // Path for signature starts with the bucket + $spath = '/' . rawurlencode( $srcCont ) . '/' . + str_replace( '%2F', '/', rawurlencode( $srcRel ) ); + // Calculate the hash + $signature = base64_encode( hash_hmac( + 'sha1', + "GET\n\n\n{$expires}\n{$spath}", + $this->rgwS3SecretKey, + true // raw + ) ); + // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. + // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. + return wfAppendQuery( + str_replace( '/swift/v1', '', // S3 API is the rgw default + $sContObj->cfs_http->getStorageUrl() . $spath ), + array( + 'Signature' => $signature, + 'Expires' => $expires, + 'AWSAccessKeyId' => $this->rgwS3AccessKey ) + ); + } + } catch ( NoSuchContainerException $e ) { + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, $params ); + } + } + return null; } - /** - * @see FileBackendStore::directoriesAreVirtual() - * @return bool - */ protected function directoriesAreVirtual() { return true; } @@ -1091,7 +1249,7 @@ class SwiftFileBackend extends FileBackendStore { * on a FileBackend params array, e.g. that of getLocalCopy(). * $params is currently only checked for a 'latest' flag. * - * @param $params Array + * @param array $params * @return Array */ protected function headersFromParams( array $params ) { @@ -1102,10 +1260,6 @@ class SwiftFileBackend extends FileBackendStore { return $hdrs; } - /** - * @see FileBackendStore::doExecuteOpHandlesInternal() - * @return Array List of corresponding Status objects - */ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { $statuses = array(); @@ -1118,8 +1272,8 @@ class SwiftFileBackend extends FileBackendStore { $cfOps = $batch->execute(); foreach ( $cfOps as $index => $cfOp ) { $status = Status::newGood(); + $function = '_getResponse' . $fileOpHandles[$index]->call; try { // catch exceptions; update status - $function = '_getResponse' . $fileOpHandles[$index]->call; $this->$function( $cfOp, $status, $fileOpHandles[$index]->params ); $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects ); } catch ( CloudFilesException $e ) { // some other exception? @@ -1137,12 +1291,12 @@ class SwiftFileBackend extends FileBackendStore { * * $readGrps is a list of the possible criteria for a request to have * access to read a container. Each item is one of the following formats: - * - account:user : Grants access if the request is by the given user - * - .r:<regex> : Grants access if the request is from a referrer host that - * matches the expression and the request is not for a listing. - * Setting this to '*' effectively makes a container public. - * - .rlistings:<regex> : Grants access if the request is from a referrer host that - * matches the expression and the request for a listing. + * - account:user : Grants access if the request is by the given user + * - ".r:<regex>" : Grants access if the request is from a referrer host that + * matches the expression and the request is not for a listing. + * Setting this to '*' effectively makes a container public. + * -".rlistings:<regex>" : Grants access if the request is from a referrer host that + * matches the expression and the request is for a listing. * * $writeGrps is a list of the possible criteria for a request to have * access to write to a container. Each item is of the following format: @@ -1153,9 +1307,9 @@ class SwiftFileBackend extends FileBackendStore { * In general, we don't allow listings to end-users. It's not useful, isn't well-defined * (lists are truncated to 10000 item with no way to page), and is just a performance risk. * - * @param $contObj CF_Container Swift container - * @param $readGrps Array List of read access routes - * @param $writeGrps Array List of write access routes + * @param CF_Container $contObj Swift container + * @param array $readGrps List of read access routes + * @param array $writeGrps List of write access routes * @return Status */ protected function setContainerAccess( @@ -1178,7 +1332,7 @@ class SwiftFileBackend extends FileBackendStore { * Purge the CDN cache of affected objects if CDN caching is enabled. * This is for Rackspace/Akamai CDNs. * - * @param $objects Array List of CF_Object items + * @param array $objects List of CF_Object items * @return void */ public function purgeCDNCache( array $objects ) { @@ -1199,8 +1353,9 @@ class SwiftFileBackend extends FileBackendStore { /** * Get an authenticated connection handle to the Swift proxy * - * @return CF_Connection|bool False on failure * @throws CloudFilesException + * @throws CloudFilesException|Exception + * @return CF_Connection|bool False on failure */ protected function getConnection() { if ( $this->connException instanceof CloudFilesException ) { @@ -1222,12 +1377,12 @@ class SwiftFileBackend extends FileBackendStore { if ( is_array( $creds ) ) { // cache hit $this->auth->load_cached_credentials( $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] ); - $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case + $this->sessionStarted = time() - ceil( $this->authTTL / 2 ); // skew for worst case } else { // cache miss try { $this->auth->authenticate(); $creds = $this->auth->export_credentials(); - $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache + $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL / 2 ) ); // cache $this->sessionStarted = time(); } catch ( CloudFilesException $e ) { $this->connException = $e; // don't keep re-trying @@ -1251,6 +1406,7 @@ class SwiftFileBackend extends FileBackendStore { protected function closeConnection() { if ( $this->conn ) { $this->conn->close(); // close active cURL handles in CF_Http object + $this->conn = null; $this->sessionStarted = 0; $this->connContainerCache->clear(); } @@ -1259,7 +1415,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Get the cache key for a container * - * @param $username string + * @param string $username * @return string */ private function getCredsCacheKey( $username ) { @@ -1267,18 +1423,11 @@ class SwiftFileBackend extends FileBackendStore { } /** - * @see FileBackendStore::doClearCache() - */ - protected function doClearCache( array $paths = null ) { - $this->connContainerCache->clear(); // clear container object cache - } - - /** * Get a Swift container object, possibly from process cache. * Use $reCache if the file count or byte count is needed. * - * @param $container string Container name - * @param $bypassCache bool Bypass all caches and load from Swift + * @param string $container Container name + * @param bool $bypassCache Bypass all caches and load from Swift * @return CF_Container * @throws CloudFilesException */ @@ -1305,7 +1454,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Create a Swift container * - * @param $container string Container name + * @param string $container Container name * @return CF_Container * @throws CloudFilesException */ @@ -1319,7 +1468,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Delete a Swift container * - * @param $container string Container name + * @param string $container Container name * @return void * @throws CloudFilesException */ @@ -1329,10 +1478,6 @@ class SwiftFileBackend extends FileBackendStore { $conn->delete_container( $container ); } - /** - * @see FileBackendStore::doPrimeContainerCache() - * @return void - */ protected function doPrimeContainerCache( array $containerInfo ) { try { $conn = $this->getConnection(); // Swift proxy connection @@ -1350,10 +1495,10 @@ class SwiftFileBackend extends FileBackendStore { * Log an unexpected exception for this backend. * This also sets the Status object to have a fatal error. * - * @param $e Exception - * @param $status Status|null - * @param $func string - * @param $params Array + * @param Exception $e + * @param Status $status|null + * @param string $func + * @param array $params * @return void */ protected function handleException( Exception $e, $status, $func, array $params ) { @@ -1387,7 +1532,15 @@ class SwiftFileOpHandle extends FileBackendStoreOpHandle { /** @var Array */ public $affectedObjects = array(); - public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) { + /** + * @param SwiftFileBackend $backend + * @param array $params + * @param string $call + * @param CF_Async_Op $cfOp + */ + public function __construct( + SwiftFileBackend $backend, array $params, $call, CF_Async_Op $cfOp + ) { $this->backend = $backend; $this->params = $params; $this->call = $call; @@ -1419,10 +1572,10 @@ abstract class SwiftFileBackendList implements Iterator { const PAGE_SIZE = 9000; // file listing buffer size /** - * @param $backend SwiftFileBackend - * @param $fullCont string Resolved container name - * @param $dir string Resolved directory relative to container - * @param $params Array + * @param SwiftFileBackend $backend + * @param string $fullCont Resolved container name + * @param string $dir Resolved directory relative to container + * @param array $params */ public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) { $this->backend = $backend; @@ -1491,12 +1644,12 @@ abstract class SwiftFileBackendList implements Iterator { /** * Get the given list portion (page) * - * @param $container string Resolved container name - * @param $dir string Resolved path relative to container - * @param $after string|null - * @param $limit integer - * @param $params Array - * @return Traversable|Array|null Returns null on failure + * @param string $container Resolved container name + * @param string $dir Resolved path relative to container + * @param string $after|null + * @param integer $limit + * @param array $params + * @return Traversable|Array */ abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); } @@ -1515,7 +1668,7 @@ class SwiftFileBackendDirList extends SwiftFileBackendList { /** * @see SwiftFileBackendList::pageFromList() - * @return Array|null + * @return Array */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); @@ -1536,7 +1689,7 @@ class SwiftFileBackendFileList extends SwiftFileBackendList { /** * @see SwiftFileBackendList::pageFromList() - * @return Array|null + * @return Array */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); |