summaryrefslogtreecommitdiff
path: root/includes/filerepo
diff options
context:
space:
mode:
Diffstat (limited to 'includes/filerepo')
-rw-r--r--includes/filerepo/FileBackendDBRepoWrapper.php356
-rw-r--r--includes/filerepo/FileRepo.php43
-rw-r--r--includes/filerepo/ForeignAPIRepo.php4
-rw-r--r--includes/filerepo/ForeignDBRepo.php32
-rw-r--r--includes/filerepo/ForeignDBViaLBRepo.php10
-rw-r--r--includes/filerepo/LocalRepo.php113
-rw-r--r--includes/filerepo/file/ArchivedFile.php4
-rw-r--r--includes/filerepo/file/File.php65
-rw-r--r--includes/filerepo/file/ForeignAPIFile.php19
-rw-r--r--includes/filerepo/file/LocalFile.php289
10 files changed, 733 insertions, 202 deletions
diff --git a/includes/filerepo/FileBackendDBRepoWrapper.php b/includes/filerepo/FileBackendDBRepoWrapper.php
new file mode 100644
index 00000000..c83e5b1b
--- /dev/null
+++ b/includes/filerepo/FileBackendDBRepoWrapper.php
@@ -0,0 +1,356 @@
+<?php
+/**
+ * Proxy backend that manages file layout rewriting for FileRepo.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup FileRepo
+ * @ingroup FileBackend
+ * @author Aaron Schulz
+ */
+
+/**
+ * @brief Proxy backend that manages file layout rewriting for FileRepo.
+ *
+ * LocalRepo may be configured to store files under their title names or by SHA-1.
+ * This acts as a shim in the later case, providing backwards compatability for
+ * most callers. All "public"/"deleted" zone files actually go in an "original"
+ * container and are never changed.
+ *
+ * This requires something like thumb_handler.php and img_auth.php for client viewing of files.
+ *
+ * @ingroup FileRepo
+ * @ingroup FileBackend
+ * @since 1.25
+ */
+class FileBackendDBRepoWrapper extends FileBackend {
+ /** @var FileBackend */
+ protected $backend;
+ /** @var string */
+ protected $repoName;
+ /** @var Closure */
+ protected $dbHandleFunc;
+ /** @var ProcessCacheLRU */
+ protected $resolvedPathCache;
+ /** @var DBConnRef[] */
+ protected $dbs;
+
+ public function __construct( array $config ) {
+ $config['name'] = $config['backend']->getName();
+ $config['wikiId'] = $config['backend']->getWikiId();
+ parent::__construct( $config );
+ $this->backend = $config['backend'];
+ $this->repoName = $config['repoName'];
+ $this->dbHandleFunc = $config['dbHandleFactory'];
+ $this->resolvedPathCache = new ProcessCacheLRU( 100 );
+ }
+
+ /**
+ * Get the underlying FileBackend that is being wrapped
+ *
+ * @return FileBackend
+ */
+ public function getInternalBackend() {
+ return $this->backend;
+ }
+
+ /**
+ * Translate a legacy "title" path to it's "sha1" counterpart
+ *
+ * E.g. mwstore://local-backend/local-public/a/ab/<name>.jpg
+ * => mwstore://local-backend/local-original/x/y/z/<sha1>.jpg
+ *
+ * @param string $path
+ * @param bool $latest
+ * @return string
+ */
+ public function getBackendPath( $path, $latest = true ) {
+ $paths = $this->getBackendPaths( array( $path ), $latest );
+ return current( $paths );
+ }
+
+ /**
+ * Translate legacy "title" paths to their "sha1" counterparts
+ *
+ * E.g. mwstore://local-backend/local-public/a/ab/<name>.jpg
+ * => mwstore://local-backend/local-original/x/y/z/<sha1>.jpg
+ *
+ * @param array $paths
+ * @param bool $latest
+ * @return array Translated paths in same order
+ */
+ public function getBackendPaths( array $paths, $latest = true ) {
+ $db = $this->getDB( $latest ? DB_MASTER : DB_SLAVE );
+
+ // @TODO: batching
+ $resolved = array();
+ foreach ( $paths as $i => $path ) {
+ if ( !$latest && $this->resolvedPathCache->has( $path, 'target', 10 ) ) {
+ $resolved[$i] = $this->resolvedPathCache->get( $path, 'target' );
+ continue;
+ }
+
+ list( , $container ) = FileBackend::splitStoragePath( $path );
+
+ if ( $container === "{$this->repoName}-public" ) {
+ $name = basename( $path );
+ if ( strpos( $path, '!' ) !== false ) {
+ $sha1 = $db->selectField( 'oldimage', 'oi_sha1',
+ array( 'oi_archive_name' => $name ),
+ __METHOD__
+ );
+ } else {
+ $sha1 = $db->selectField( 'image', 'img_sha1',
+ array( 'img_name' => $name ),
+ __METHOD__
+ );
+ }
+ if ( !strlen( $sha1 ) ) {
+ $resolved[$i] = $path; // give up
+ continue;
+ }
+ $resolved[$i] = $this->getPathForSHA1( $sha1 );
+ $this->resolvedPathCache->set( $path, 'target', $resolved[$i] );
+ } elseif ( $container === "{$this->repoName}-deleted" ) {
+ $name = basename( $path ); // <hash>.<ext>
+ $sha1 = substr( $name, 0, strpos( $name, '.' ) ); // ignore extension
+ $resolved[$i] = $this->getPathForSHA1( $sha1 );
+ $this->resolvedPathCache->set( $path, 'target', $resolved[$i] );
+ } else {
+ $resolved[$i] = $path;
+ }
+ }
+
+ $res = array();
+ foreach ( $paths as $i => $path ) {
+ $res[$i] = $resolved[$i];
+ }
+
+ return $res;
+ }
+
+ protected function doOperationsInternal( array $ops, array $opts ) {
+ return $this->backend->doOperationsInternal( $this->mungeOpPaths( $ops ), $opts );
+ }
+
+ protected function doQuickOperationsInternal( array $ops ) {
+ return $this->backend->doQuickOperationsInternal( $this->mungeOpPaths( $ops ) );
+ }
+
+ protected function doPrepare( array $params ) {
+ return $this->backend->doPrepare( $params );
+ }
+
+ protected function doSecure( array $params ) {
+ return $this->backend->doSecure( $params );
+ }
+
+ protected function doPublish( array $params ) {
+ return $this->backend->doPublish( $params );
+ }
+
+ protected function doClean( array $params ) {
+ return $this->backend->doClean( $params );
+ }
+
+ public function concatenate( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function fileExists( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getFileTimestamp( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getFileSize( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getFileStat( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getFileXAttributes( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getFileSha1Base36( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getFileProps( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function streamFile( array $params ) {
+ // The stream methods use the file extension to determine the
+ // Content-Type (as MediaWiki should already validate it on upload).
+ // The translated SHA1 path has no extension, so this needs to use
+ // the untranslated path extension.
+ $type = StreamFile::contentTypeFromPath( $params['src'] );
+ if ( $type && $type != 'unknown/unknown' ) {
+ $params['headers'][] = "Content-type: $type";
+ }
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getFileContentsMulti( array $params ) {
+ return $this->translateArrayResults( __FUNCTION__, $params );
+ }
+
+ public function getLocalReferenceMulti( array $params ) {
+ return $this->translateArrayResults( __FUNCTION__, $params );
+ }
+
+ public function getLocalCopyMulti( array $params ) {
+ return $this->translateArrayResults( __FUNCTION__, $params );
+ }
+
+ public function getFileHttpUrl( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function directoryExists( array $params ) {
+ return $this->backend->directoryExists( $params );
+ }
+
+ public function getDirectoryList( array $params ) {
+ return $this->backend->getDirectoryList( $params );
+ }
+
+ public function getFileList( array $params ) {
+ return $this->backend->getFileList( $params );
+ }
+
+ public function getFeatures() {
+ return $this->backend->getFeatures();
+ }
+
+ public function clearCache( array $paths = null ) {
+ $this->backend->clearCache( null ); // clear all
+ }
+
+ public function preloadCache( array $paths ) {
+ $paths = $this->getBackendPaths( $paths );
+ $this->backend->preloadCache( $paths );
+ }
+
+ public function preloadFileStat( array $params ) {
+ return $this->translateSrcParams( __FUNCTION__, $params );
+ }
+
+ public function getScopedLocksForOps( array $ops, Status $status ) {
+ return $this->backend->getScopedLocksForOps( $ops, $status );
+ }
+
+ /**
+ * Get the ultimate original storage path for a file
+ *
+ * Use this when putting a new file into the system
+ *
+ * @param string $sha1 File SHA-1 base36
+ * @return string
+ */
+ public function getPathForSHA1( $sha1 ) {
+ if ( strlen( $sha1 ) < 3 ) {
+ throw new InvalidArgumentException( "Invalid file SHA-1." );
+ }
+ return $this->backend->getContainerStoragePath( "{$this->repoName}-original" ) .
+ "/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
+ }
+
+ /**
+ * Get a connection to the repo file registry DB
+ *
+ * @param integer $index
+ * @return DBConnRef
+ */
+ protected function getDB( $index ) {
+ if ( !isset( $this->dbs[$index] ) ) {
+ $func = $this->dbHandleFunc;
+ $this->dbs[$index] = $func( $index );
+ }
+ return $this->dbs[$index];
+ }
+
+ /**
+ * Translates paths found in the "src" or "srcs" keys of a params array
+ *
+ * @param string $function
+ * @param array $params
+ */
+ protected function translateSrcParams( $function, array $params ) {
+ $latest = !empty( $params['latest'] );
+
+ if ( isset( $params['src'] ) ) {
+ $params['src'] = $this->getBackendPath( $params['src'], $latest );
+ }
+
+ if ( isset( $params['srcs'] ) ) {
+ $params['srcs'] = $this->getBackendPaths( $params['srcs'], $latest );
+ }
+
+ return $this->backend->$function( $params );
+ }
+
+ /**
+ * Translates paths when the backend function returns results keyed by paths
+ *
+ * @param string $function
+ * @param array $params
+ * @return array
+ */
+ protected function translateArrayResults( $function, array $params ) {
+ $origPaths = $params['srcs'];
+ $params['srcs'] = $this->getBackendPaths( $params['srcs'], !empty( $params['latest'] ) );
+ $pathMap = array_combine( $params['srcs'], $origPaths );
+
+ $results = $this->backend->$function( $params );
+
+ $contents = array();
+ foreach ( $results as $path => $result ) {
+ $contents[$pathMap[$path]] = $result;
+ }
+
+ return $contents;
+ }
+
+ /**
+ * Translate legacy "title" source paths to their "sha1" counterparts
+ *
+ * This leaves destination paths alone since we don't want those to mutate
+ *
+ * @param array $ops
+ * @return array
+ */
+ protected function mungeOpPaths( array $ops ) {
+ // Ops that use 'src' and do not mutate core file data there
+ static $srcRefOps = array( 'store', 'copy', 'describe' );
+ foreach ( $ops as &$op ) {
+ if ( isset( $op['src'] ) && in_array( $op['op'], $srcRefOps ) ) {
+ $op['src'] = $this->getBackendPath( $op['src'], true );
+ }
+ if ( isset( $op['srcs'] ) ) {
+ $op['srcs'] = $this->getBackendPaths( $op['srcs'], true );
+ }
+ }
+ return $ops;
+ }
+}
diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php
index cef1176d..7370c5cd 100644
--- a/includes/filerepo/FileRepo.php
+++ b/includes/filerepo/FileRepo.php
@@ -49,6 +49,9 @@ class FileRepo {
/** @var int */
public $descriptionCacheExpiry;
+ /** @var bool */
+ protected $hasSha1Storage = false;
+
/** @var FileBackend */
protected $backend;
@@ -63,7 +66,7 @@ class FileRepo {
protected $transformVia404;
/** @var string URL of image description pages, e.g.
- * http://en.wikipedia.org/wiki/File:
+ * https://en.wikipedia.org/wiki/File:
*/
protected $descBaseUrl;
@@ -76,7 +79,7 @@ class FileRepo {
* to $wgScriptExtension, e.g. .php5 defaults to .php */
protected $scriptExtension;
- /** @var string Equivalent to $wgArticlePath, e.g. http://en.wikipedia.org/wiki/$1 */
+ /** @var string Equivalent to $wgArticlePath, e.g. https://en.wikipedia.org/wiki/$1 */
protected $articleUrl;
/** @var bool Equivalent to $wgCapitalLinks (or $wgCapitalLinkOverrides[NS_FILE],
@@ -433,16 +436,16 @@ class FileRepo {
$img = $this->newFile( $title, $time );
if ( $img ) {
$img->load( $flags );
- }
- if ( $img && $img->exists() ) {
- if ( !$img->isDeleted( File::DELETED_FILE ) ) {
- return $img; // always OK
- } elseif ( !empty( $options['private'] ) &&
- $img->userCan( File::DELETED_FILE,
- $options['private'] instanceof User ? $options['private'] : null
- )
- ) {
- return $img;
+ if ( $img->exists() ) {
+ if ( !$img->isDeleted( File::DELETED_FILE ) ) {
+ return $img; // always OK
+ } elseif ( !empty( $options['private'] ) &&
+ $img->userCan( File::DELETED_FILE,
+ $options['private'] instanceof User ? $options['private'] : null
+ )
+ ) {
+ return $img;
+ }
}
}
}
@@ -909,9 +912,9 @@ class FileRepo {
$status->merge( $backend->doOperations( $operations, $opts ) );
// Cleanup for disk source files...
foreach ( $sourceFSFilesToDelete as $file ) {
- wfSuppressWarnings();
+ MediaWiki\suppressWarnings();
unlink( $file ); // FS cleanup
- wfRestoreWarnings();
+ MediaWiki\restoreWarnings();
}
return $status;
@@ -1297,9 +1300,9 @@ class FileRepo {
}
// Cleanup for disk source files...
foreach ( $sourceFSFilesToDelete as $file ) {
- wfSuppressWarnings();
+ MediaWiki\suppressWarnings();
unlink( $file ); // FS cleanup
- wfRestoreWarnings();
+ MediaWiki\restoreWarnings();
}
return $status;
@@ -1885,6 +1888,14 @@ class FileRepo {
return $ret;
}
+
+ /**
+ * Returns whether or not storage is SHA-1 based
+ * @return boolean
+ */
+ public function hasSha1Storage() {
+ return $this->hasSha1Storage;
+ }
}
/**
diff --git a/includes/filerepo/ForeignAPIRepo.php b/includes/filerepo/ForeignAPIRepo.php
index 71d2b919..4ffbf4ad 100644
--- a/includes/filerepo/ForeignAPIRepo.php
+++ b/includes/filerepo/ForeignAPIRepo.php
@@ -31,7 +31,7 @@ use MediaWiki\Logger\LoggerFactory;
* $wgForeignFileRepos[] = array(
* 'class' => 'ForeignAPIRepo',
* 'name' => 'shared',
- * 'apibase' => 'http://en.wikipedia.org/w/api.php',
+ * 'apibase' => 'https://en.wikipedia.org/w/api.php',
* 'fetchDescription' => true, // Optional
* 'descriptionCacheExpiry' => 3600,
* );
@@ -74,7 +74,7 @@ class ForeignAPIRepo extends FileRepo {
global $wgLocalFileRepo;
parent::__construct( $info );
- // http://commons.wikimedia.org/w/api.php
+ // https://commons.wikimedia.org/w/api.php
$this->mApiBase = isset( $info['apibase'] ) ? $info['apibase'] : null;
if ( isset( $info['apiThumbCacheExpiry'] ) ) {
diff --git a/includes/filerepo/ForeignDBRepo.php b/includes/filerepo/ForeignDBRepo.php
index 6e9e6add..dfdb3753 100644
--- a/includes/filerepo/ForeignDBRepo.php
+++ b/includes/filerepo/ForeignDBRepo.php
@@ -76,17 +76,8 @@ class ForeignDBRepo extends LocalRepo {
*/
function getMasterDB() {
if ( !isset( $this->dbConn ) ) {
- $this->dbConn = DatabaseBase::factory( $this->dbType,
- array(
- 'host' => $this->dbServer,
- 'user' => $this->dbUser,
- 'password' => $this->dbPassword,
- 'dbname' => $this->dbName,
- 'flags' => $this->dbFlags,
- 'tablePrefix' => $this->tablePrefix,
- 'foreign' => true,
- )
- );
+ $func = $this->getDBFactory();
+ $this->dbConn = $func( DB_MASTER );
}
return $this->dbConn;
@@ -100,6 +91,25 @@ class ForeignDBRepo extends LocalRepo {
}
/**
+ * @return Closure
+ */
+ protected function getDBFactory() {
+ return function( $index ) {
+ return DatabaseBase::factory( $this->dbType,
+ array(
+ 'host' => $this->dbServer,
+ 'user' => $this->dbUser,
+ 'password' => $this->dbPassword,
+ 'dbname' => $this->dbName,
+ 'flags' => $this->dbFlags,
+ 'tablePrefix' => $this->tablePrefix,
+ 'foreign' => true,
+ )
+ );
+ };
+ }
+
+ /**
* @return bool
*/
function hasSharedCache() {
diff --git a/includes/filerepo/ForeignDBViaLBRepo.php b/includes/filerepo/ForeignDBViaLBRepo.php
index 8153ffb4..f49b716f 100644
--- a/includes/filerepo/ForeignDBViaLBRepo.php
+++ b/includes/filerepo/ForeignDBViaLBRepo.php
@@ -66,6 +66,16 @@ class ForeignDBViaLBRepo extends LocalRepo {
return wfGetDB( DB_SLAVE, array(), $this->wiki );
}
+ /**
+ * @return Closure
+ */
+ protected function getDBFactory() {
+ $wiki = $this->wiki;
+ return function( $index ) use ( $wiki ) {
+ return wfGetDB( $index, array(), $wiki );
+ };
+ }
+
function hasSharedCache() {
return $this->hasSharedCache;
}
diff --git a/includes/filerepo/LocalRepo.php b/includes/filerepo/LocalRepo.php
index 926fd0b8..6a2c0640 100644
--- a/includes/filerepo/LocalRepo.php
+++ b/includes/filerepo/LocalRepo.php
@@ -29,6 +29,9 @@
* @ingroup FileRepo
*/
class LocalRepo extends FileRepo {
+ /** @var bool */
+ protected $hasSha1Storage = false;
+
/** @var array */
protected $fileFactory = array( 'LocalFile', 'newFromTitle' );
@@ -47,6 +50,20 @@ class LocalRepo extends FileRepo {
/** @var array */
protected $oldFileFactoryKey = array( 'OldLocalFile', 'newFromKey' );
+ function __construct( array $info = null ) {
+ parent::__construct( $info );
+
+ $this->hasSha1Storage = isset( $info['storageLayout'] ) && $info['storageLayout'] === 'sha1';
+
+ if ( $this->hasSha1Storage() ) {
+ $this->backend = new FileBackendDBRepoWrapper( array(
+ 'backend' => $this->backend,
+ 'repoName' => $this->name,
+ 'dbHandleFactory' => $this->getDBFactory()
+ ) );
+ }
+ }
+
/**
* @throws MWException
* @param stdClass $row
@@ -82,6 +99,11 @@ class LocalRepo extends FileRepo {
* @return FileRepoStatus
*/
function cleanupDeletedBatch( array $storageKeys ) {
+ if ( $this->hasSha1Storage() ) {
+ wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
+ return Status::newGood();
+ }
+
$backend = $this->backend; // convenience
$root = $this->getZonePath( 'deleted' );
$dbw = $this->getMasterDB();
@@ -90,7 +112,7 @@ class LocalRepo extends FileRepo {
foreach ( $storageKeys as $key ) {
$hashPath = $this->getDeletedHashPath( $key );
$path = "$root/$hashPath$key";
- $dbw->begin( __METHOD__ );
+ $dbw->startAtomic( __METHOD__ );
// Check for usage in deleted/hidden files and preemptively
// lock the key to avoid any future use until we are finished.
$deleted = $this->deletedFileHasKey( $key, 'lock' );
@@ -106,7 +128,7 @@ class LocalRepo extends FileRepo {
wfDebug( __METHOD__ . ": $key still in use\n" );
$status->successCount++;
}
- $dbw->commit( __METHOD__ );
+ $dbw->endAtomic( __METHOD__ );
}
return $status;
@@ -170,7 +192,7 @@ class LocalRepo extends FileRepo {
* @return bool|Title
*/
function checkRedirect( Title $title ) {
- global $wgMemc;
+ $cache = ObjectCache::getMainWANInstance();
$title = File::normalizeTitle( $title, 'exception' );
@@ -181,7 +203,7 @@ class LocalRepo extends FileRepo {
} else {
$expiry = 86400; // has invalidation, 1 day
}
- $cachedValue = $wgMemc->get( $memcKey );
+ $cachedValue = $cache->get( $memcKey );
if ( $cachedValue === ' ' || $cachedValue === '' ) {
// Does not exist
return false;
@@ -191,7 +213,7 @@ class LocalRepo extends FileRepo {
$id = $this->getArticleID( $title );
if ( !$id ) {
- $wgMemc->add( $memcKey, " ", $expiry );
+ $cache->set( $memcKey, " ", $expiry );
return false;
}
@@ -205,11 +227,11 @@ class LocalRepo extends FileRepo {
if ( $row && $row->rd_namespace == NS_FILE ) {
$targetTitle = Title::makeTitle( $row->rd_namespace, $row->rd_title );
- $wgMemc->add( $memcKey, $targetTitle->getDBkey(), $expiry );
+ $cache->set( $memcKey, $targetTitle->getDBkey(), $expiry );
return $targetTitle;
} else {
- $wgMemc->add( $memcKey, '', $expiry );
+ $cache->set( $memcKey, '', $expiry );
return false;
}
@@ -275,17 +297,17 @@ class LocalRepo extends FileRepo {
);
};
- $repo = $this;
+ $that = $this;
$applyMatchingFiles = function ( ResultWrapper $res, &$searchSet, &$finalFiles )
- use ( $repo, $fileMatchesSearch, $flags )
+ use ( $that, $fileMatchesSearch, $flags )
{
global $wgContLang;
- $info = $repo->getInfo();
+ $info = $that->getInfo();
foreach ( $res as $row ) {
- $file = $repo->newFileFromRow( $row );
+ $file = $that->newFileFromRow( $row );
// There must have been a search for this DB key, but this has to handle the
// cases were title capitalization is different on the client and repo wikis.
- $dbKeysLook = array( str_replace( ' ', '_', $file->getName() ) );
+ $dbKeysLook = array( strtr( $file->getName(), ' ', '_' ) );
if ( !empty( $info['initialCapital'] ) ) {
// Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
$dbKeysLook[] = $wgContLang->lcfirst( $file->getName() );
@@ -470,6 +492,16 @@ class LocalRepo extends FileRepo {
}
/**
+ * Get a callback to get a DB handle given an index (DB_SLAVE/DB_MASTER)
+ * @return Closure
+ */
+ protected function getDBFactory() {
+ return function( $index ) {
+ return wfGetDB( $index );
+ };
+ }
+
+ /**
* Get a key on the primary cache for this repository.
* Returns false if the repository's cache is not accessible at this site.
* The parameters are the parts of the key, as for wfMemcKey().
@@ -489,14 +521,15 @@ class LocalRepo extends FileRepo {
* @return void
*/
function invalidateImageRedirect( Title $title ) {
- global $wgMemc;
+ $cache = ObjectCache::getMainWANInstance();
+
$memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
if ( $memcKey ) {
// Set a temporary value for the cache key, to ensure
// that this value stays purged long enough so that
// it isn't refreshed with a stale value due to a
// lagged slave.
- $wgMemc->set( $memcKey, ' PURGED', 12 );
+ $cache->delete( $memcKey, 12 );
}
}
@@ -513,4 +546,56 @@ class LocalRepo extends FileRepo {
'favicon' => wfExpandUrl( $wgFavicon ),
) );
}
+
+ public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
+ return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
+ }
+
+ public function storeBatch( array $triplets, $flags = 0 ) {
+ return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
+ }
+
+ public function cleanupBatch( array $files, $flags = 0 ) {
+ return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
+ }
+
+ public function publish(
+ $srcPath,
+ $dstRel,
+ $archiveRel,
+ $flags = 0,
+ array $options = array()
+ ) {
+ return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
+ }
+
+ public function publishBatch( array $ntuples, $flags = 0 ) {
+ return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
+ }
+
+ public function delete( $srcRel, $archiveRel ) {
+ return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
+ }
+
+ public function deleteBatch( array $sourceDestPairs ) {
+ return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
+ }
+
+ /**
+ * Skips the write operation if storage is sha1-based, executes it normally otherwise
+ *
+ * @param string $function
+ * @param array $args
+ * @return FileRepoStatus
+ */
+ protected function skipWriteOperationIfSha1( $function, array $args ) {
+ $this->assertWritableRepo(); // fail out if read-only
+
+ if ( $this->hasSha1Storage() ) {
+ wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
+ return Status::newGood();
+ } else {
+ return call_user_func_array( 'parent::' . $function, $args );
+ }
+ }
}
diff --git a/includes/filerepo/file/ArchivedFile.php b/includes/filerepo/file/ArchivedFile.php
index 1d454283..1aec4464 100644
--- a/includes/filerepo/file/ArchivedFile.php
+++ b/includes/filerepo/file/ArchivedFile.php
@@ -485,7 +485,7 @@ class ArchivedFile {
if ( $type == 'text' ) {
return $this->user_text;
} elseif ( $type == 'id' ) {
- return $this->user;
+ return (int)$this->user;
}
throw new MWException( "Unknown type '$type'." );
@@ -587,6 +587,6 @@ class ArchivedFile {
$this->load();
$title = $this->getTitle();
- return Revision::userCanBitfield( $this->deleted, $field, $user, $title ? : null );
+ return Revision::userCanBitfield( $this->deleted, $field, $user, $title ?: null );
}
}
diff --git a/includes/filerepo/file/File.php b/includes/filerepo/file/File.php
index 6edd6fcc..f40d216c 100644
--- a/includes/filerepo/file/File.php
+++ b/includes/filerepo/file/File.php
@@ -163,7 +163,8 @@ abstract class File implements IDBAccessObject {
* @param FileRepo|bool $repo
*/
function __construct( $title, $repo ) {
- if ( $title !== false ) { // subclasses may not use MW titles
+ // Some subclasses do not use $title, but set name/title some other way
+ if ( $title !== false ) {
$title = self::normalizeTitle( $title, 'exception' );
}
$this->title = $title;
@@ -212,14 +213,15 @@ abstract class File implements IDBAccessObject {
}
/**
- * Normalize a file extension to the common form, and ensure it's clean.
- * Extensions with non-alphanumeric characters will be discarded.
+ * Normalize a file extension to the common form, making it lowercase and checking some synonyms,
+ * and ensure it's clean. Extensions with non-alphanumeric characters will be discarded.
+ * Keep in sync with mw.Title.normalizeExtension() in JS.
*
- * @param string $ext (without the .)
- * @return string
+ * @param string $extension File extension (without the leading dot)
+ * @return string File extension in canonical form
*/
- static function normalizeExtension( $ext ) {
- $lower = strtolower( $ext );
+ static function normalizeExtension( $extension ) {
+ $lower = strtolower( $extension );
$squish = array(
'htm' => 'html',
'jpeg' => 'jpg',
@@ -420,7 +422,13 @@ abstract class File implements IDBAccessObject {
public function getLocalRefPath() {
$this->assertRepoDefined();
if ( !isset( $this->fsFile ) ) {
+ $starttime = microtime( true );
$this->fsFile = $this->repo->getLocalReference( $this->getPath() );
+
+ $statTiming = microtime( true ) - $starttime;
+ RequestContext::getMain()->getStats()->timing(
+ 'media.thumbnail.generate.fetchoriginal', 1000 * $statTiming );
+
if ( !$this->fsFile ) {
$this->fsFile = false; // null => false; cache negative hits
}
@@ -1091,7 +1099,9 @@ abstract class File implements IDBAccessObject {
* @return bool|MediaTransformOutput
*/
public function generateAndSaveThumb( $tmpFile, $transformParams, $flags ) {
- global $wgUseSquid, $wgIgnoreImageErrors;
+ global $wgIgnoreImageErrors;
+
+ $stats = RequestContext::getMain()->getStats();
$handler = $this->getHandler();
@@ -1108,10 +1118,15 @@ abstract class File implements IDBAccessObject {
$this->generateBucketsIfNeeded( $normalisedParams, $flags );
}
+ $starttime = microtime( true );
+
// Actually render the thumbnail...
$thumb = $handler->doTransform( $this, $tmpThumbPath, $thumbUrl, $transformParams );
$tmpFile->bind( $thumb ); // keep alive with $thumb
+ $statTiming = microtime( true ) - $starttime;
+ $stats->timing( 'media.thumbnail.generate.transform', 1000 * $statTiming );
+
if ( !$thumb ) { // bad params?
$thumb = false;
} elseif ( $thumb->isError() ) { // transform error
@@ -1122,6 +1137,9 @@ abstract class File implements IDBAccessObject {
}
} elseif ( $this->repo && $thumb->hasFile() && !$thumb->fileIsSource() ) {
// Copy the thumbnail from the file system into storage...
+
+ $starttime = microtime( true );
+
$disposition = $this->getThumbDisposition( $thumbName );
$status = $this->repo->quickImport( $tmpThumbPath, $thumbPath, $disposition );
if ( $status->isOK() ) {
@@ -1129,19 +1147,14 @@ abstract class File implements IDBAccessObject {
} else {
$thumb = $this->transformErrorOutput( $thumbPath, $thumbUrl, $transformParams, $flags );
}
+
+ $statTiming = microtime( true ) - $starttime;
+ $stats->timing( 'media.thumbnail.generate.store', 1000 * $statTiming );
+
// Give extensions a chance to do something with this thumbnail...
Hooks::run( 'FileTransformed', array( $this, $thumb, $tmpThumbPath, $thumbPath ) );
}
- // Purge. Useful in the event of Core -> Squid connection failure or squid
- // purge collisions from elsewhere during failure. Don't keep triggering for
- // "thumbs" which have the main image URL though (bug 13776)
- if ( $wgUseSquid ) {
- if ( !$thumb || $thumb->isError() || $thumb->getUrl() != $this->getURL() ) {
- SquidUpdate::purge( array( $thumbUrl ) );
- }
- }
-
return $thumb;
}
@@ -1166,13 +1179,13 @@ abstract class File implements IDBAccessObject {
return false;
}
+ $starttime = microtime( true );
+
$params['physicalWidth'] = $bucket;
$params['width'] = $bucket;
$params = $this->getHandler()->sanitizeParamsForBucketing( $params );
- $bucketName = $this->getBucketThumbName( $bucket );
-
$tmpFile = $this->makeTransformTmpFile( $bucketPath );
if ( !$tmpFile ) {
@@ -1181,6 +1194,8 @@ abstract class File implements IDBAccessObject {
$thumb = $this->generateAndSaveThumb( $tmpFile, $params, $flags );
+ $buckettime = microtime( true ) - $starttime;
+
if ( !$thumb || $thumb->isError() ) {
return false;
}
@@ -1190,6 +1205,9 @@ abstract class File implements IDBAccessObject {
// this object exists
$tmpFile->bind( $this );
+ RequestContext::getMain()->getStats()->timing(
+ 'media.thumbnail.generate.bucket', 1000 * $buckettime );
+
return true;
}
@@ -2230,4 +2248,13 @@ abstract class File implements IDBAccessObject {
$handler = $this->getHandler();
return $handler ? $handler->isExpensiveToThumbnail( $this ) : false;
}
+
+ /**
+ * Whether the thumbnails created on the same server as this code is running.
+ * @since 1.25
+ * @return bool
+ */
+ public function isTransformedLocally() {
+ return true;
+ }
}
diff --git a/includes/filerepo/file/ForeignAPIFile.php b/includes/filerepo/file/ForeignAPIFile.php
index 3d5d5d60..3c78290c 100644
--- a/includes/filerepo/file/ForeignAPIFile.php
+++ b/includes/filerepo/file/ForeignAPIFile.php
@@ -219,11 +219,15 @@ class ForeignAPIFile extends File {
}
/**
- * @param string $method
+ * @param string $type
* @return int|null|string
*/
- public function getUser( $method = 'text' ) {
- return isset( $this->mInfo['user'] ) ? strval( $this->mInfo['user'] ) : null;
+ public function getUser( $type = 'text' ) {
+ if ( $type == 'text' ) {
+ return isset( $this->mInfo['user'] ) ? strval( $this->mInfo['user'] ) : null;
+ } elseif ( $type == 'id' ) {
+ return 0; // What makes sense here, for a remote user?
+ }
}
/**
@@ -365,4 +369,13 @@ class ForeignAPIFile extends File {
# Clear out the thumbnail directory if empty
$this->repo->quickCleanDir( $dir );
}
+
+ /**
+ * The thumbnail is created on the foreign server and fetched over internet
+ * @since 1.25
+ * @return bool
+ */
+ public function isTransformedLocally() {
+ return false;
+ }
}
diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php
index b4cced38..d2c37e61 100644
--- a/includes/filerepo/file/LocalFile.php
+++ b/includes/filerepo/file/LocalFile.php
@@ -243,21 +243,19 @@ class LocalFile extends File {
* @return bool
*/
function loadFromCache() {
- global $wgMemc;
-
$this->dataLoaded = false;
$this->extraDataLoaded = false;
$key = $this->getCacheKey();
if ( !$key ) {
-
return false;
}
- $cachedValues = $wgMemc->get( $key );
+ $cache = ObjectCache::getMainWANInstance();
+ $cachedValues = $cache->get( $key );
// Check if the key existed and belongs to this version of MediaWiki
- if ( isset( $cachedValues['version'] ) && $cachedValues['version'] == MW_FILE_VERSION ) {
+ if ( is_array( $cachedValues ) && $cachedValues['version'] == MW_FILE_VERSION ) {
wfDebug( "Pulling file metadata from cache key $key\n" );
$this->fileExists = $cachedValues['fileExists'];
if ( $this->fileExists ) {
@@ -271,9 +269,9 @@ class LocalFile extends File {
}
if ( $this->dataLoaded ) {
- wfIncrStats( 'image_cache_hit' );
+ wfIncrStats( 'image_cache.hit' );
} else {
- wfIncrStats( 'image_cache_miss' );
+ wfIncrStats( 'image_cache.miss' );
}
return $this->dataLoaded;
@@ -283,22 +281,20 @@ class LocalFile extends File {
* Save the file metadata to memcached
*/
function saveToCache() {
- global $wgMemc;
-
$this->load();
- $key = $this->getCacheKey();
+ $key = $this->getCacheKey();
if ( !$key ) {
return;
}
$fields = $this->getCacheFields( '' );
- $cache = array( 'version' => MW_FILE_VERSION );
- $cache['fileExists'] = $this->fileExists;
+ $cacheVal = array( 'version' => MW_FILE_VERSION );
+ $cacheVal['fileExists'] = $this->fileExists;
if ( $this->fileExists ) {
foreach ( $fields as $field ) {
- $cache[$field] = $this->$field;
+ $cacheVal[$field] = $this->$field;
}
}
@@ -306,13 +302,26 @@ class LocalFile extends File {
// If the cache value gets to large it will not fit in memcached and nothing will
// get cached at all, causing master queries for any file access.
foreach ( $this->getLazyCacheFields( '' ) as $field ) {
- if ( isset( $cache[$field] ) && strlen( $cache[$field] ) > 100 * 1024 ) {
- unset( $cache[$field] ); // don't let the value get too big
+ if ( isset( $cacheVal[$field] ) && strlen( $cacheVal[$field] ) > 100 * 1024 ) {
+ unset( $cacheVal[$field] ); // don't let the value get too big
}
}
// Cache presence for 1 week and negatives for 1 day
- $wgMemc->set( $key, $cache, $this->fileExists ? 86400 * 7 : 86400 );
+ $cache = ObjectCache::getMainWANInstance();
+ $cache->set( $key, $cacheVal, $this->fileExists ? 86400 * 7 : 86400 );
+ }
+
+ /**
+ * Purge the file object/metadata cache
+ */
+ function invalidateCache() {
+ $key = $this->getCacheKey();
+ if ( !$key ) {
+ return;
+ }
+
+ ObjectCache::getMainWANInstance()->delete( $key );
}
/**
@@ -493,9 +502,17 @@ class LocalFile extends File {
$decoded['mime'] = $decoded['major_mime'] . '/' . $decoded['minor_mime'];
}
- # Trim zero padding from char/binary field
+ // Trim zero padding from char/binary field
$decoded['sha1'] = rtrim( $decoded['sha1'], "\0" );
+ // Normalize some fields to integer type, per their database definition.
+ // Use unary + so that overflows will be upgraded to double instead of
+ // being trucated as with intval(). This is important to allow >2GB
+ // files on 32-bit systems.
+ foreach ( array( 'size', 'width', 'height', 'bits' ) as $field ) {
+ $decoded[$field] = +$decoded[$field];
+ }
+
return $decoded;
}
@@ -612,7 +629,7 @@ class LocalFile extends File {
__METHOD__
);
- $this->saveToCache();
+ $this->invalidateCache();
$this->unlock(); // done
@@ -734,7 +751,7 @@ class LocalFile extends File {
if ( $type == 'text' ) {
return $this->user_text;
} elseif ( $type == 'id' ) {
- return $this->user;
+ return (int)$this->user;
}
}
@@ -753,7 +770,7 @@ class LocalFile extends File {
function getBitDepth() {
$this->load();
- return $this->bits;
+ return (int)$this->bits;
}
/**
@@ -842,25 +859,7 @@ class LocalFile extends File {
* Refresh metadata in memcached, but don't touch thumbnails or squid
*/
function purgeMetadataCache() {
- $this->loadFromDB( File::READ_LATEST );
- $this->saveToCache();
- $this->purgeHistory();
- }
-
- /**
- * Purge the shared history (OldLocalFile) cache.
- *
- * @note This used to purge old thumbnails as well.
- */
- function purgeHistory() {
- global $wgMemc;
-
- $hashedName = md5( $this->getName() );
- $oldKey = $this->repo->getSharedCacheKey( 'oldfile', $hashedName );
-
- if ( $oldKey ) {
- $wgMemc->delete( $oldKey );
- }
+ $this->invalidateCache();
}
/**
@@ -1406,11 +1405,8 @@ class LocalFile extends File {
# to after $wikiPage->doEdit has been called.
$dbw->commit( __METHOD__ );
- # Save to memcache.
- # We shall not saveToCache before the commit since otherwise
- # in case of a rollback there is an usable file from memcached
- # which in fact doesn't really exist (bug 24978)
- $this->saveToCache();
+ # Update memcache after the commit
+ $this->invalidateCache();
if ( $exists ) {
# Invalidate the cache for the description page
@@ -1471,7 +1467,7 @@ class LocalFile extends File {
* The archive name should be passed through to recordUpload for database
* registration.
*
- * @param string $srcPath Local filesystem path to the source image
+ * @param string $srcPath Local filesystem path or virtual URL to the source image
* @param int $flags A bitwise combination of:
* File::DELETE_SOURCE Delete the source file, i.e. move rather than copy
* @param array $options Optional additional parameters
@@ -1489,7 +1485,7 @@ class LocalFile extends File {
* The archive name should be passed through to recordUpload for database
* registration.
*
- * @param string $srcPath Local filesystem path to the source image
+ * @param string $srcPath Local filesystem path or virtual URL to the source image
* @param string $dstRel Target relative path
* @param int $flags A bitwise combination of:
* File::DELETE_SOURCE Delete the source file, i.e. move rather than copy
@@ -1498,7 +1494,8 @@ class LocalFile extends File {
* archive name, or an empty string if it was a new file.
*/
function publishTo( $srcPath, $dstRel, $flags = 0, array $options = array() ) {
- if ( $this->getRepo()->getReadOnlyReason() !== false ) {
+ $repo = $this->getRepo();
+ if ( $repo->getReadOnlyReason() !== false ) {
return $this->readOnlyFatalStatus();
}
@@ -1506,13 +1503,29 @@ class LocalFile extends File {
$archiveName = wfTimestamp( TS_MW ) . '!' . $this->getName();
$archiveRel = 'archive/' . $this->getHashPath() . $archiveName;
- $flags = $flags & File::DELETE_SOURCE ? LocalRepo::DELETE_SOURCE : 0;
- $status = $this->repo->publish( $srcPath, $dstRel, $archiveRel, $flags, $options );
- if ( $status->value == 'new' ) {
- $status->value = '';
+ if ( $repo->hasSha1Storage() ) {
+ $sha1 = $repo->isVirtualUrl( $srcPath )
+ ? $repo->getFileSha1( $srcPath )
+ : File::sha1Base36( $srcPath );
+ $dst = $repo->getBackend()->getPathForSHA1( $sha1 );
+ $status = $repo->quickImport( $srcPath, $dst );
+ if ( $flags & File::DELETE_SOURCE ) {
+ unlink( $srcPath );
+ }
+
+ if ( $this->exists() ) {
+ $status->value = $archiveName;
+ }
} else {
- $status->value = $archiveName;
+ $flags = $flags & File::DELETE_SOURCE ? LocalRepo::DELETE_SOURCE : 0;
+ $status = $repo->publish( $srcPath, $dstRel, $archiveRel, $flags, $options );
+
+ if ( $status->value == 'new' ) {
+ $status->value = '';
+ } else {
+ $status->value = $archiveName;
+ }
}
$this->unlock(); // done
@@ -1612,21 +1625,21 @@ class LocalFile extends File {
// Hack: the lock()/unlock() pair is nested in a transaction so the locking is not
// tied to BEGIN/COMMIT. To avoid slow purges in the transaction, move them outside.
- $file = $this;
+ $that = $this;
$this->getRepo()->getMasterDB()->onTransactionIdle(
- function () use ( $file, $archiveNames ) {
+ function () use ( $that, $archiveNames ) {
global $wgUseSquid;
- $file->purgeEverything();
+ $that->purgeEverything();
foreach ( $archiveNames as $archiveName ) {
- $file->purgeOldThumbnails( $archiveName );
+ $that->purgeOldThumbnails( $archiveName );
}
if ( $wgUseSquid ) {
// Purge the squid
$purgeUrls = array();
foreach ( $archiveNames as $archiveName ) {
- $purgeUrls[] = $file->getArchiveUrl( $archiveName );
+ $purgeUrls[] = $that->getArchiveUrl( $archiveName );
}
SquidUpdate::purge( $purgeUrls );
}
@@ -1667,7 +1680,6 @@ class LocalFile extends File {
$this->purgeOldThumbnails( $archiveName );
if ( $status->isOK() ) {
$this->purgeDescription();
- $this->purgeHistory();
}
if ( $wgUseSquid ) {
@@ -1811,7 +1823,7 @@ class LocalFile extends File {
array( 'img_sha1' => $this->sha1 ),
array( 'img_name' => $this->getName() ),
__METHOD__ );
- $this->saveToCache();
+ $this->invalidateCache();
}
$this->unlock(); // done
@@ -1954,14 +1966,14 @@ class LocalFileDeleteBatch {
$this->status = $file->repo->newGood();
}
- function addCurrent() {
+ public function addCurrent() {
$this->srcRels['.'] = $this->file->getRel();
}
/**
* @param string $oldName
*/
- function addOld( $oldName ) {
+ public function addOld( $oldName ) {
$this->srcRels[$oldName] = $this->file->getArchiveRel( $oldName );
$this->archiveUrls[] = $this->file->getArchiveUrl( $oldName );
}
@@ -1970,7 +1982,7 @@ class LocalFileDeleteBatch {
* Add the old versions of the image to the batch
* @return array List of archive names from old versions
*/
- function addOlds() {
+ public function addOlds() {
$archiveNames = array();
$dbw = $this->file->repo->getMasterDB();
@@ -1991,7 +2003,7 @@ class LocalFileDeleteBatch {
/**
* @return array
*/
- function getOldRels() {
+ protected function getOldRels() {
if ( !isset( $this->srcRels['.'] ) ) {
$oldRels =& $this->srcRels;
$deleteCurrent = false;
@@ -2063,7 +2075,7 @@ class LocalFileDeleteBatch {
return $hashes;
}
- function doDBInserts() {
+ protected function doDBInserts() {
$dbw = $this->file->repo->getMasterDB();
$encTimestamp = $dbw->addQuotes( $dbw->timestamp() );
$encUserId = $dbw->addQuotes( $this->user->getId() );
@@ -2178,8 +2190,8 @@ class LocalFileDeleteBatch {
* Run the transaction
* @return FileRepoStatus
*/
- function execute() {
-
+ public function execute() {
+ $repo = $this->file->getRepo();
$this->file->lock();
// Prepare deletion batch
@@ -2193,7 +2205,7 @@ class LocalFileDeleteBatch {
if ( isset( $hashes[$name] ) ) {
$hash = $hashes[$name];
$key = $hash . $dotExt;
- $dstRel = $this->file->repo->getDeletedHashPath( $key ) . $key;
+ $dstRel = $repo->getDeletedHashPath( $key ) . $key;
$this->deletionBatch[$name] = array( $srcRel, $dstRel );
}
}
@@ -2206,20 +2218,22 @@ class LocalFileDeleteBatch {
// them in a separate transaction, then run the file ops, then update the fa_name fields.
$this->doDBInserts();
- // Removes non-existent file from the batch, so we don't get errors.
- // This also handles files in the 'deleted' zone deleted via revision deletion.
- $checkStatus = $this->removeNonexistentFiles( $this->deletionBatch );
- if ( !$checkStatus->isGood() ) {
- $this->status->merge( $checkStatus );
- return $this->status;
- }
- $this->deletionBatch = $checkStatus->value;
+ if ( !$repo->hasSha1Storage() ) {
+ // Removes non-existent file from the batch, so we don't get errors.
+ // This also handles files in the 'deleted' zone deleted via revision deletion.
+ $checkStatus = $this->removeNonexistentFiles( $this->deletionBatch );
+ if ( !$checkStatus->isGood() ) {
+ $this->status->merge( $checkStatus );
+ return $this->status;
+ }
+ $this->deletionBatch = $checkStatus->value;
- // Execute the file deletion batch
- $status = $this->file->repo->deleteBatch( $this->deletionBatch );
+ // Execute the file deletion batch
+ $status = $this->file->repo->deleteBatch( $this->deletionBatch );
- if ( !$status->isGood() ) {
- $this->status->merge( $status );
+ if ( !$status->isGood() ) {
+ $this->status->merge( $status );
+ }
}
if ( !$this->status->isOK() ) {
@@ -2245,7 +2259,7 @@ class LocalFileDeleteBatch {
* @param array $batch
* @return Status
*/
- function removeNonexistentFiles( $batch ) {
+ protected function removeNonexistentFiles( $batch ) {
$files = $newBatch = array();
foreach ( $batch as $batchItem ) {
@@ -2306,7 +2320,7 @@ class LocalFileRestoreBatch {
* Add a file by ID
* @param int $fa_id
*/
- function addId( $fa_id ) {
+ public function addId( $fa_id ) {
$this->ids[] = $fa_id;
}
@@ -2314,14 +2328,14 @@ class LocalFileRestoreBatch {
* Add a whole lot of files by ID
* @param int[] $ids
*/
- function addIds( $ids ) {
+ public function addIds( $ids ) {
$this->ids = array_merge( $this->ids, $ids );
}
/**
* Add all revisions of the file
*/
- function addAll() {
+ public function addAll() {
$this->all = true;
}
@@ -2333,12 +2347,13 @@ class LocalFileRestoreBatch {
* So we save the batch and let the caller call cleanup()
* @return FileRepoStatus
*/
- function execute() {
+ public function execute() {
global $wgLang;
+ $repo = $this->file->getRepo();
if ( !$this->all && !$this->ids ) {
// Do nothing
- return $this->file->repo->newGood();
+ return $repo->newGood();
}
$lockOwnsTrx = $this->file->lock();
@@ -2395,9 +2410,9 @@ class LocalFileRestoreBatch {
continue;
}
- $deletedRel = $this->file->repo->getDeletedHashPath( $row->fa_storage_key ) .
+ $deletedRel = $repo->getDeletedHashPath( $row->fa_storage_key ) .
$row->fa_storage_key;
- $deletedUrl = $this->file->repo->getVirtualUrl() . '/deleted/' . $deletedRel;
+ $deletedUrl = $repo->getVirtualUrl() . '/deleted/' . $deletedRel;
if ( isset( $row->fa_sha1 ) ) {
$sha1 = $row->fa_sha1;
@@ -2511,27 +2526,29 @@ class LocalFileRestoreBatch {
$status->error( 'undelete-missing-filearchive', $id );
}
- // Remove missing files from batch, so we don't get errors when undeleting them
- $checkStatus = $this->removeNonexistentFiles( $storeBatch );
- if ( !$checkStatus->isGood() ) {
- $status->merge( $checkStatus );
- return $status;
- }
- $storeBatch = $checkStatus->value;
+ if ( !$repo->hasSha1Storage() ) {
+ // Remove missing files from batch, so we don't get errors when undeleting them
+ $checkStatus = $this->removeNonexistentFiles( $storeBatch );
+ if ( !$checkStatus->isGood() ) {
+ $status->merge( $checkStatus );
+ return $status;
+ }
+ $storeBatch = $checkStatus->value;
- // Run the store batch
- // Use the OVERWRITE_SAME flag to smooth over a common error
- $storeStatus = $this->file->repo->storeBatch( $storeBatch, FileRepo::OVERWRITE_SAME );
- $status->merge( $storeStatus );
+ // Run the store batch
+ // Use the OVERWRITE_SAME flag to smooth over a common error
+ $storeStatus = $this->file->repo->storeBatch( $storeBatch, FileRepo::OVERWRITE_SAME );
+ $status->merge( $storeStatus );
- if ( !$status->isGood() ) {
- // Even if some files could be copied, fail entirely as that is the
- // easiest thing to do without data loss
- $this->cleanupFailedBatch( $storeStatus, $storeBatch );
- $status->ok = false;
- $this->file->unlock();
+ if ( !$status->isGood() ) {
+ // Even if some files could be copied, fail entirely as that is the
+ // easiest thing to do without data loss
+ $this->cleanupFailedBatch( $storeStatus, $storeBatch );
+ $status->ok = false;
+ $this->file->unlock();
- return $status;
+ return $status;
+ }
}
// Run the DB updates
@@ -2555,7 +2572,7 @@ class LocalFileRestoreBatch {
}
// If store batch is empty (all files are missing), deletion is to be considered successful
- if ( $status->successCount > 0 || !$storeBatch ) {
+ if ( $status->successCount > 0 || !$storeBatch || $repo->hasSha1Storage() ) {
if ( !$exists ) {
wfDebug( __METHOD__ . " restored {$status->successCount} items, creating a new current\n" );
@@ -2565,7 +2582,6 @@ class LocalFileRestoreBatch {
} else {
wfDebug( __METHOD__ . " restored {$status->successCount} as archived versions\n" );
$this->file->purgeDescription();
- $this->file->purgeHistory();
}
}
@@ -2579,7 +2595,7 @@ class LocalFileRestoreBatch {
* @param array $triplets
* @return Status
*/
- function removeNonexistentFiles( $triplets ) {
+ protected function removeNonexistentFiles( $triplets ) {
$files = $filteredTriplets = array();
foreach ( $triplets as $file ) {
$files[$file[0]] = $file[0];
@@ -2605,7 +2621,7 @@ class LocalFileRestoreBatch {
* @param array $batch
* @return array
*/
- function removeNonexistentFromCleanup( $batch ) {
+ protected function removeNonexistentFromCleanup( $batch ) {
$files = $newBatch = array();
$repo = $this->file->repo;
@@ -2630,7 +2646,7 @@ class LocalFileRestoreBatch {
* This should be called from outside the transaction in which execute() was called.
* @return FileRepoStatus
*/
- function cleanup() {
+ public function cleanup() {
if ( !$this->cleanupBatch ) {
return $this->file->repo->newGood();
}
@@ -2649,7 +2665,7 @@ class LocalFileRestoreBatch {
* @param Status $storeStatus
* @param array $storeBatch
*/
- function cleanupFailedBatch( $storeStatus, $storeBatch ) {
+ protected function cleanupFailedBatch( $storeStatus, $storeBatch ) {
$cleanupBatch = array();
foreach ( $storeStatus->success as $i => $success ) {
@@ -2707,7 +2723,7 @@ class LocalFileMoveBatch {
/**
* Add the current image to the batch
*/
- function addCurrent() {
+ public function addCurrent() {
$this->cur = array( $this->oldRel, $this->newRel );
}
@@ -2715,7 +2731,7 @@ class LocalFileMoveBatch {
* Add the old versions of the image to the batch
* @return array List of archive names from old versions
*/
- function addOlds() {
+ public function addOlds() {
$archiveBase = 'archive';
$this->olds = array();
$this->oldCount = 0;
@@ -2765,7 +2781,7 @@ class LocalFileMoveBatch {
* Perform the move.
* @return FileRepoStatus
*/
- function execute() {
+ public function execute() {
$repo = $this->file->repo;
$status = $repo->newGood();
@@ -2796,22 +2812,26 @@ class LocalFileMoveBatch {
wfDebugLog( 'imagemove', "Renamed {$this->file->getName()} in database: " .
"{$statusDb->successCount} successes, {$statusDb->failCount} failures" );
- // Copy the files into their new location.
- // If a prior process fataled copying or cleaning up files we tolerate any
- // of the existing files if they are identical to the ones being stored.
- $statusMove = $repo->storeBatch( $triplets, FileRepo::OVERWRITE_SAME );
- wfDebugLog( 'imagemove', "Moved files for {$this->file->getName()}: " .
- "{$statusMove->successCount} successes, {$statusMove->failCount} failures" );
- if ( !$statusMove->isGood() ) {
- // Delete any files copied over (while the destination is still locked)
- $this->cleanupTarget( $triplets );
- $destFile->unlock();
- $this->file->unlockAndRollback(); // unlocks the destination
- wfDebugLog( 'imagemove', "Error in moving files: " . $statusMove->getWikiText() );
- $statusMove->ok = false;
-
- return $statusMove;
+ if ( !$repo->hasSha1Storage() ) {
+ // Copy the files into their new location.
+ // If a prior process fataled copying or cleaning up files we tolerate any
+ // of the existing files if they are identical to the ones being stored.
+ $statusMove = $repo->storeBatch( $triplets, FileRepo::OVERWRITE_SAME );
+ wfDebugLog( 'imagemove', "Moved files for {$this->file->getName()}: " .
+ "{$statusMove->successCount} successes, {$statusMove->failCount} failures" );
+ if ( !$statusMove->isGood() ) {
+ // Delete any files copied over (while the destination is still locked)
+ $this->cleanupTarget( $triplets );
+ $destFile->unlock();
+ $this->file->unlockAndRollback(); // unlocks the destination
+ wfDebugLog( 'imagemove', "Error in moving files: " . $statusMove->getWikiText() );
+ $statusMove->ok = false;
+
+ return $statusMove;
+ }
+ $status->merge( $statusMove );
}
+
$destFile->unlock();
$this->file->unlock(); // done
@@ -2819,7 +2839,6 @@ class LocalFileMoveBatch {
$this->cleanupSource( $triplets );
$status->merge( $statusDb );
- $status->merge( $statusMove );
return $status;
}
@@ -2830,7 +2849,7 @@ class LocalFileMoveBatch {
*
* @return FileRepoStatus
*/
- function doDBUpdates() {
+ protected function doDBUpdates() {
$repo = $this->file->repo;
$status = $repo->newGood();
$dbw = $this->db;
@@ -2882,7 +2901,7 @@ class LocalFileMoveBatch {
* Generate triplets for FileRepo::storeBatch().
* @return array
*/
- function getMoveTriplets() {
+ protected function getMoveTriplets() {
$moves = array_merge( array( $this->cur ), $this->olds );
$triplets = array(); // The format is: (srcUrl, destZone, destUrl)
@@ -2904,7 +2923,7 @@ class LocalFileMoveBatch {
* @param array $triplets
* @return Status
*/
- function removeNonexistentFiles( $triplets ) {
+ protected function removeNonexistentFiles( $triplets ) {
$files = array();
foreach ( $triplets as $file ) {
@@ -2934,7 +2953,7 @@ class LocalFileMoveBatch {
* files. Called if something went wrong half way.
* @param array $triplets
*/
- function cleanupTarget( $triplets ) {
+ protected function cleanupTarget( $triplets ) {
// Create dest pairs from the triplets
$pairs = array();
foreach ( $triplets as $triplet ) {
@@ -2950,7 +2969,7 @@ class LocalFileMoveBatch {
* Called at the end of the move process if everything else went ok.
* @param array $triplets
*/
- function cleanupSource( $triplets ) {
+ protected function cleanupSource( $triplets ) {
// Create source file names from the triplets
$files = array();
foreach ( $triplets as $triplet ) {
generated by cgit v1.2.3-54-g00ecf (git 2.45.2) at 2024-11-05 22:02:56 +0000