summaryrefslogtreecommitdiff
path: root/includes/filebackend/SwiftFileBackend.php
diff options
context:
space:
mode:
Diffstat (limited to 'includes/filebackend/SwiftFileBackend.php')
-rw-r--r--includes/filebackend/SwiftFileBackend.php81
1 files changed, 53 insertions, 28 deletions
diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php
index f40ec46e..5f406c9b 100644
--- a/includes/filebackend/SwiftFileBackend.php
+++ b/includes/filebackend/SwiftFileBackend.php
@@ -112,7 +112,7 @@ class SwiftFileBackend extends FileBackendStore {
// Optional settings
$this->authTTL = isset( $config['swiftAuthTTL'] )
? $config['swiftAuthTTL']
- : 5 * 60; // some sane number
+ : 15 * 60; // some sane number
$this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
? $config['swiftTempUrlKey']
: '';
@@ -537,6 +537,7 @@ class SwiftFileBackend extends FileBackendStore {
return $status; // already there
} elseif ( $stat === null ) {
$status->fatal( 'backend-fail-internal', $this->name );
+ wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
return $status;
}
@@ -568,6 +569,7 @@ class SwiftFileBackend extends FileBackendStore {
$status->fatal( 'backend-fail-usable', $params['dir'] );
} else {
$status->fatal( 'backend-fail-internal', $this->name );
+ wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
}
return $status;
@@ -588,6 +590,7 @@ class SwiftFileBackend extends FileBackendStore {
$status->fatal( 'backend-fail-usable', $params['dir'] );
} else {
$status->fatal( 'backend-fail-internal', $this->name );
+ wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
}
return $status;
@@ -607,6 +610,7 @@ class SwiftFileBackend extends FileBackendStore {
return $status; // ok, nothing to do
} elseif ( !is_array( $stat ) ) {
$status->fatal( 'backend-fail-internal', $this->name );
+ wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' );
return $status;
}
@@ -643,7 +647,7 @@ class SwiftFileBackend extends FileBackendStore {
$timestamp = new MWTimestamp( $ts );
return $timestamp->getTimestamp( $format );
- } catch ( MWException $e ) {
+ } catch ( Exception $e ) {
throw new FileBackendError( $e->getMessage() );
}
}
@@ -660,7 +664,7 @@ class SwiftFileBackend extends FileBackendStore {
return $objHdrs; // nothing to do
}
- $section = new ProfileSection( __METHOD__ . '-' . $this->name );
+ $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
$auth = $this->getAuthentication();
@@ -794,7 +798,7 @@ class SwiftFileBackend extends FileBackendStore {
return $dirs; // nothing more
}
- $section = new ProfileSection( __METHOD__ . '-' . $this->name );
+ $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
$prefix = ( $dir == '' ) ? null : "{$dir}/";
// Non-recursive: only list dirs right under $dir
@@ -874,7 +878,7 @@ class SwiftFileBackend extends FileBackendStore {
return $files; // nothing more
}
- $section = new ProfileSection( __METHOD__ . '-' . $this->name );
+ $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
$prefix = ( $dir == '' ) ? null : "{$dir}/";
// $objects will contain a list of unfiltered names or CF_Object items
@@ -933,6 +937,7 @@ class SwiftFileBackend extends FileBackendStore {
// Convert various random Swift dates to TS_MW
'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
'size' => (int)$object->bytes,
+ 'sha1' => null,
// Note: manifiest ETags are not an MD5 of the file
'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
'latest' => false // eventually consistent
@@ -1060,6 +1065,7 @@ class SwiftFileBackend extends FileBackendStore {
$tmpFiles[$path] = $tmpFile;
}
+ $isLatest = ( $this->isRGW || !empty( $params['latest'] ) );
$opts = array( 'maxConnsPerHost' => $params['concurrency'] );
$reqs = $this->http->runMulti( $reqs, $opts );
foreach ( $reqs as $path => $op ) {
@@ -1074,6 +1080,10 @@ class SwiftFileBackend extends FileBackendStore {
$this->onError( null, __METHOD__,
array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc );
}
+ // Set the file stat process cache in passing
+ $stat = $this->getStatFromHeaders( $rhdrs );
+ $stat['latest'] = $isLatest;
+ $this->cheapCache->set( $path, 'stat', $stat );
} elseif ( $rcode === 404 ) {
$tmpFiles[$path] = false;
} else {
@@ -1161,6 +1171,11 @@ class SwiftFileBackend extends FileBackendStore {
return $hdrs;
}
+ /**
+ * @param FileBackendStoreOpHandle[] $fileOpHandles
+ *
+ * @return Status[]
+ */
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
$statuses = array();
@@ -1253,6 +1268,7 @@ class SwiftFileBackend extends FileBackendStore {
if ( $rcode != 204 && $rcode !== 202 ) {
$status->fatal( 'backend-fail-internal', $this->name );
+ wfDebugLog( 'SwiftBackend', __METHOD__ . ': unexpected rcode value (' . $rcode . ')' );
}
return $status;
@@ -1267,7 +1283,7 @@ class SwiftFileBackend extends FileBackendStore {
* @return array|bool|null False on 404, null on failure
*/
protected function getContainerStat( $container, $bypassCache = false ) {
- $section = new ProfileSection( __METHOD__ . '-' . $this->name );
+ $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" );
if ( $bypassCache ) { // purge cache
$this->containerStatCache->clear( $container );
@@ -1280,13 +1296,11 @@ class SwiftFileBackend extends FileBackendStore {
return null;
}
- wfProfileIn( __METHOD__ . "-{$this->name}-miss" );
list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
'method' => 'HEAD',
'url' => $this->storageUrl( $auth, $container ),
'headers' => $this->authTokenHeaders( $auth )
) );
- wfProfileOut( __METHOD__ . "-{$this->name}-miss" );
if ( $rcode === 204 ) {
$stat = array(
@@ -1507,25 +1521,8 @@ class SwiftFileBackend extends FileBackendStore {
if ( $rcode === 200 || $rcode === 204 ) {
// Update the object if it is missing some headers
$rhdrs = $this->addMissingMetadata( $rhdrs, $path );
- // Fetch all of the custom metadata headers
- $metadata = array();
- foreach ( $rhdrs as $name => $value ) {
- if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
- $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
- }
- }
- // Fetch all of the custom raw HTTP headers
- $headers = $this->sanitizeHdrs( array( 'headers' => $rhdrs ) );
- $stat = array(
- // Convert various random Swift dates to TS_MW
- 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
- // Empty objects actually return no content-length header in Ceph
- 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
- 'sha1' => $rhdrs['x-object-meta-sha1base36'],
- // Note: manifiest ETags are not an MD5 of the file
- 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
- 'xattr' => array( 'metadata' => $metadata, 'headers' => $headers )
- );
+ // Load the stat array from the headers
+ $stat = $this->getStatFromHeaders( $rhdrs );
if ( $this->isRGW ) {
$stat['latest'] = true; // strong consistency
}
@@ -1542,6 +1539,34 @@ class SwiftFileBackend extends FileBackendStore {
}
/**
+ * @param array $rhdrs
+ * @return array
+ */
+ protected function getStatFromHeaders( array $rhdrs ) {
+ // Fetch all of the custom metadata headers
+ $metadata = array();
+ foreach ( $rhdrs as $name => $value ) {
+ if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
+ $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
+ }
+ }
+ // Fetch all of the custom raw HTTP headers
+ $headers = $this->sanitizeHdrs( array( 'headers' => $rhdrs ) );
+ return array(
+ // Convert various random Swift dates to TS_MW
+ 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
+ // Empty objects actually return no content-length header in Ceph
+ 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
+ 'sha1' => isset( $rhdrs['x-object-meta-sha1base36'] )
+ ? $rhdrs['x-object-meta-sha1base36']
+ : null,
+ // Note: manifiest ETags are not an MD5 of the file
+ 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
+ 'xattr' => array( 'metadata' => $metadata, 'headers' => $headers )
+ );
+ }
+
+ /**
* @return array|null Credential map
*/
protected function getAuthentication() {
@@ -1595,7 +1620,7 @@ class SwiftFileBackend extends FileBackendStore {
}
// Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
- $this->isRGW = true; // take advantage of strong consistency
+ $this->isRGW = true; // take advantage of strong consistency in Ceph
}
}
generated by cgit v1.2.3-54-g00ecf (git 2.45.2) at 2024-11-22 02:09:37 +0000