diff options
Diffstat (limited to 'includes/upload')
| -rw-r--r-- | includes/upload/UploadBase.php | 159 | ||||
| -rw-r--r-- | includes/upload/UploadFromChunks.php | 56 | ||||
| -rw-r--r-- | includes/upload/UploadFromFile.php | 12 | ||||
| -rw-r--r-- | includes/upload/UploadFromStash.php | 17 | ||||
| -rw-r--r-- | includes/upload/UploadFromUrl.php | 22 | ||||
| -rw-r--r-- | includes/upload/UploadStash.php | 74 |
6 files changed, 211 insertions, 129 deletions
diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php index 0848780f..5a823622 100644 --- a/includes/upload/UploadBase.php +++ b/includes/upload/UploadBase.php @@ -65,6 +65,8 @@ abstract class UploadBase { const WINDOWS_NONASCII_FILENAME = 13; const FILENAME_TOO_LONG = 14; + const SESSION_STATUS_KEY = 'wsUploadStatusData'; + /** * @param $error int * @return string @@ -78,7 +80,7 @@ abstract class UploadBase { self::ILLEGAL_FILENAME => 'illegal-filename', self::OVERWRITE_EXISTING_FILE => 'overwrite', self::VERIFICATION_ERROR => 'verification-error', - self::HOOK_ABORTED => 'hookaborted', + self::HOOK_ABORTED => 'hookaborted', self::WINDOWS_NONASCII_FILENAME => 'windows-nonascii-filename', self::FILENAME_TOO_LONG => 'filename-toolong', ); @@ -108,7 +110,7 @@ abstract class UploadBase { /** * Returns true if the user can use this upload module or else a string * identifying the missing permission. - * Can be overriden by subclasses. + * Can be overridden by subclasses. * * @param $user User * @return bool @@ -190,10 +192,10 @@ abstract class UploadBase { /** * Initialize the path information - * @param $name string the desired destination name - * @param $tempPath string the temporary path - * @param $fileSize int the file size - * @param $removeTempFile bool (false) remove the temporary file? + * @param string $name the desired destination name + * @param string $tempPath the temporary path + * @param int $fileSize the file size + * @param bool $removeTempFile (false) remove the temporary file? * @throws MWException */ public function initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile = false ) { @@ -209,7 +211,7 @@ abstract class UploadBase { /** * Initialize from a WebRequest. Override this in a subclass. */ - public abstract function initializeFromRequest( &$request ); + abstract public function initializeFromRequest( &$request ); /** * Fetch the file. Usually a no-op @@ -236,7 +238,15 @@ abstract class UploadBase { } /** - * @param $srcPath String: the source path + * Get the base 36 SHA1 of the file + * @return string + */ + public function getTempFileSha1Base36() { + return FSFile::getSha1Base36FromPath( $this->mTempPath ); + } + + /** + * @param string $srcPath the source path * @return string the real path if it was a virtual URL */ function getRealPath( $srcPath ) { @@ -244,9 +254,9 @@ abstract class UploadBase { $repo = RepoGroup::singleton()->getLocalRepo(); if ( $repo->isVirtualUrl( $srcPath ) ) { // @TODO: just make uploads work with storage paths - // UploadFromStash loads files via virtuals URLs + // UploadFromStash loads files via virtual URLs $tmpFile = $repo->getLocalCopy( $srcPath ); - $tmpFile->bind( $this ); // keep alive with $thumb + $tmpFile->bind( $this ); // keep alive with $this wfProfileOut( __METHOD__ ); return $tmpFile->getPath(); } @@ -347,14 +357,14 @@ abstract class UploadBase { * * @note Only checks that it is not an evil mime. The does it have * correct extension given its mime type check is in verifyFile. - * @param $mime string representing the mime + * @param string $mime representing the mime * @return mixed true if the file is verified, an array otherwise */ protected function verifyMimeType( $mime ) { global $wgVerifyMimeType; wfProfileIn( __METHOD__ ); if ( $wgVerifyMimeType ) { - wfDebug ( "\n\nmime: <$mime> extension: <{$this->mFinalExtension}>\n\n"); + wfDebug ( "\n\nmime: <$mime> extension: <{$this->mFinalExtension}>\n\n" ); global $wgMimeTypeBlacklist; if ( $this->checkFileExtension( $mime, $wgMimeTypeBlacklist ) ) { wfProfileOut( __METHOD__ ); @@ -447,7 +457,7 @@ abstract class UploadBase { $this->mFileProps = FSFile::getPropsFromPath( $this->mTempPath, $this->mFinalExtension ); # check mime type, if desired - $mime = $this->mFileProps[ 'file-mime' ]; + $mime = $this->mFileProps['file-mime']; $status = $this->verifyMimeType( $mime ); if ( $status !== true ) { wfProfileOut( __METHOD__ ); @@ -575,7 +585,9 @@ abstract class UploadBase { } /** - * Check for non fatal problems with the file + * Check for non fatal problems with the file. + * + * This should not assume that mTempPath is set. * * @return Array of warnings */ @@ -610,7 +622,7 @@ abstract class UploadBase { global $wgUploadSizeWarning; if ( $wgUploadSizeWarning && ( $this->mFileSize > $wgUploadSizeWarning ) ) { - $warnings['large-file'] = $wgUploadSizeWarning; + $warnings['large-file'] = array( $wgUploadSizeWarning, $this->mFileSize ); } if ( $this->mFileSize == 0 ) { @@ -623,7 +635,7 @@ abstract class UploadBase { } // Check dupes against existing files - $hash = FSFile::getSha1Base36FromPath( $this->mTempPath ); + $hash = $this->getTempFileSha1Base36(); $dupes = RepoGroup::singleton()->findBySha1( $hash ); $title = $this->getTitle(); // Remove all matches against self @@ -723,8 +735,6 @@ abstract class UploadBase { } $this->mFilteredName = $nt->getDBkey(); - - /** * We'll want to blacklist against *any* 'extension', and use * only the final one for the whitelist. @@ -752,7 +762,6 @@ abstract class UploadBase { $ext = array( $this->mFinalExtension ); } } - } /* Don't allow users to override the blacklist (check file extension) */ @@ -787,7 +796,7 @@ abstract class UploadBase { } if( strlen( $partname ) < 1 ) { - $this->mTitleError = self::MIN_LENGTH_PARTNAME; + $this->mTitleError = self::MIN_LENGTH_PARTNAME; return $this->mTitle = null; } @@ -816,13 +825,14 @@ abstract class UploadBase { * This method returns the file object, which also has a 'fileKey' property which can be passed through a form or * API request to find this stashed file again. * + * @param $user User * @return UploadStashFile stashed file */ - public function stashFile() { + public function stashFile( User $user = null ) { // was stashSessionFile wfProfileIn( __METHOD__ ); - $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash(); + $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash( $user ); $file = $stash->stashFile( $this->mTempPath, $this->getSourceType() ); $this->mLocalFile = $file; @@ -905,8 +915,8 @@ abstract class UploadBase { /** * Checks if the mime type of the uploaded file matches the file extension. * - * @param $mime String: the mime type of the uploaded file - * @param $extension String: the filename extension that the file is to be served with + * @param string $mime the mime type of the uploaded file + * @param string $extension the filename extension that the file is to be served with * @return Boolean */ public static function verifyExtension( $mime, $extension ) { @@ -946,9 +956,9 @@ abstract class UploadBase { * potentially harmful. The present implementation will produce false * positives in some situations. * - * @param $file String: pathname to the temporary upload file - * @param $mime String: the mime type of the file - * @param $extension String: the extension of the file + * @param string $file pathname to the temporary upload file + * @param string $mime the mime type of the file + * @param string $extension the extension of the file * @return Boolean: true if the file contains something looking like embedded scripts */ public static function detectScript( $file, $mime, $extension ) { @@ -958,7 +968,7 @@ abstract class UploadBase { # ugly hack: for text files, always look at the entire file. # For binary field, just check the first K. - if( strpos( $mime,'text/' ) === 0 ) { + if( strpos( $mime, 'text/' ) === 0 ) { $chunk = file_get_contents( $file ); } else { $fp = fopen( $file, 'rb' ); @@ -988,7 +998,7 @@ abstract class UploadBase { $chunk = trim( $chunk ); - # @todo FIXME: Convert from UTF-16 if necessarry! + # @todo FIXME: Convert from UTF-16 if necessary! wfDebug( __METHOD__ . ": checking for embedded scripts and HTML stuff\n" ); # check for HTML doctype @@ -1173,7 +1183,7 @@ abstract class UploadBase { foreach( $attribs as $attrib => $value ) { $stripped = $this->stripXmlNamespace( $attrib ); - $value = strtolower($value); + $value = strtolower( $value ); if( substr( $stripped, 0, 2 ) == 'on' ) { wfDebug( __METHOD__ . ": Found event-handler attribute '$attrib'='$value' in uploaded file.\n" ); @@ -1186,13 +1196,13 @@ abstract class UploadBase { return true; } - # href with embeded svg as target + # href with embedded svg as target if( $stripped == 'href' && preg_match( '!data:[^,]*image/svg[^,]*,!sim', $value ) ) { wfDebug( __METHOD__ . ": Found href to embedded svg \"<$strippedElement '$attrib'='$value'...\" in uploaded file.\n" ); return true; } - # href with embeded (text/xml) svg as target + # href with embedded (text/xml) svg as target if( $stripped == 'href' && preg_match( '!data:[^,]*text/xml[^,]*,!sim', $value ) ) { wfDebug( __METHOD__ . ": Found href to embedded svg \"<$strippedElement '$attrib'='$value'...\" in uploaded file.\n" ); return true; @@ -1206,19 +1216,18 @@ abstract class UploadBase { # use set to add href attribute to parent element if( $strippedElement == 'set' && $stripped == 'attributename' && strpos( $value, 'href' ) !== false ) { - wfDebug( __METHOD__ . ": Found svg setting href attibute '$value' in uploaded file.\n" ); + wfDebug( __METHOD__ . ": Found svg setting href attribute '$value' in uploaded file.\n" ); return true; } # use set to add a remote / data / script target to an element - if( $strippedElement == 'set' && $stripped == 'to' && preg_match( '!(http|https|data|script):!sim', $value ) ) { - wfDebug( __METHOD__ . ": Found svg setting attibute to '$value' in uploaded file.\n" ); + if( $strippedElement == 'set' && $stripped == 'to' && preg_match( '!(http|https|data|script):!sim', $value ) ) { + wfDebug( __METHOD__ . ": Found svg setting attribute to '$value' in uploaded file.\n" ); return true; } - # use handler attribute with remote / data / script - if( $stripped == 'handler' && preg_match( '!(http|https|data|script):!sim', $value ) ) { + if( $stripped == 'handler' && preg_match( '!(http|https|data|script):!sim', $value ) ) { wfDebug( __METHOD__ . ": Found svg setting handler with remote/data/script '$attrib'='$value' in uploaded file.\n" ); return true; } @@ -1226,8 +1235,8 @@ abstract class UploadBase { # use CSS styles to bring in remote code # catch url("http:..., url('http:..., url(http:..., but not url("#..., url('#..., url(#.... if( $stripped == 'style' && preg_match_all( '!((?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*["\']?\s*[^#]+.*?\))!sim', $value, $matches ) ) { - foreach ($matches[1] as $match) { - if (!preg_match( '!(?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*(#|\'#|"#)!sim', $match ) ) { + foreach ( $matches[1] as $match ) { + if ( !preg_match( '!(?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*(#|\'#|"#)!sim', $match ) ) { wfDebug( __METHOD__ . ": Found svg setting a style with remote url '$attrib'='$value' in uploaded file.\n" ); return true; } @@ -1260,7 +1269,7 @@ abstract class UploadBase { * This relies on the $wgAntivirus and $wgAntivirusSetup variables. * $wgAntivirusRequired may be used to deny upload if the scan fails. * - * @param $file String: pathname to the temporary upload file + * @param string $file pathname to the temporary upload file * @return mixed false if not virus is found, NULL if the scan fails or is disabled, * or a string containing feedback from the virus scanner if a virus was found. * If textual feedback is missing but a virus was found, this function returns true. @@ -1317,27 +1326,22 @@ abstract class UploadBase { } } + /* NB: AV_NO_VIRUS is 0 but AV_SCAN_FAILED is false, + * so we need the strict equalities === and thus can't use a switch here + */ if ( $mappedCode === AV_SCAN_FAILED ) { # scan failed (code was mapped to false by $exitCodeMap) wfDebug( __METHOD__ . ": failed to scan $file (code $exitCode).\n" ); - if ( $wgAntivirusRequired ) { - wfProfileOut( __METHOD__ ); - return wfMessage( 'virus-scanfailed', array( $exitCode ) )->text(); - } else { - wfProfileOut( __METHOD__ ); - return null; - } + $output = $wgAntivirusRequired ? wfMessage( 'virus-scanfailed', array( $exitCode ) )->text() : null; } elseif ( $mappedCode === AV_SCAN_ABORTED ) { # scan failed because filetype is unknown (probably imune) wfDebug( __METHOD__ . ": unsupported file type $file (code $exitCode).\n" ); - wfProfileOut( __METHOD__ ); - return null; + $output = null; } elseif ( $mappedCode === AV_NO_VIRUS ) { # no virus found wfDebug( __METHOD__ . ": file passed virus scan.\n" ); - wfProfileOut( __METHOD__ ); - return false; + $output = false; } else { $output = trim( $output ); @@ -1353,9 +1357,10 @@ abstract class UploadBase { } wfDebug( __METHOD__ . ": FOUND VIRUS! scanner feedback: $output \n" ); - wfProfileOut( __METHOD__ ); - return $output; } + + wfProfileOut( __METHOD__ ); + return $output; } /** @@ -1392,7 +1397,7 @@ abstract class UploadBase { * Check if a user is the last uploader * * @param $user User object - * @param $img String: image name + * @param string $img image name * @return Boolean */ public static function userCanReUpload( User $user, $img ) { @@ -1464,9 +1469,20 @@ abstract class UploadBase { } } + // Check for files with the same name but a different extension + $similarFiles = RepoGroup::singleton()->getLocalRepo()->findFilesByPrefix( + "{$partname}.", 1 ); + if ( count( $similarFiles ) ) { + return array( + 'warning' => 'exists-normalized', + 'file' => $file, + 'normalizedFile' => $similarFiles[0], + ); + } + if ( self::isThumbName( $file->getName() ) ) { # Check for filenames like 50px- or 180px-, these are mostly thumbnails - $nt_thb = Title::newFromText( substr( $partname , strpos( $partname , '-' ) +1 ) . '.' . $extension, NS_FILE ); + $nt_thb = Title::newFromText( substr( $partname, strpos( $partname, '-' ) + 1 ) . '.' . $extension, NS_FILE ); $file_thb = wfLocalFile( $nt_thb ); if( $file_thb->exists() ) { return array( @@ -1484,7 +1500,6 @@ abstract class UploadBase { } } - foreach( self::getFilenamePrefixBlacklist() as $prefix ) { if ( substr( $partname, 0, strlen( $prefix ) ) == $prefix ) { return array( @@ -1507,10 +1522,10 @@ abstract class UploadBase { $n = strrpos( $filename, '.' ); $partname = $n ? substr( $filename, 0, $n ) : $filename; return ( - substr( $partname , 3, 3 ) == 'px-' || - substr( $partname , 2, 3 ) == 'px-' + substr( $partname, 3, 3 ) == 'px-' || + substr( $partname, 2, 3 ) == 'px-' ) && - preg_match( "/[0-9]{2}/" , substr( $partname , 0, 2 ) ); + preg_match( "/[0-9]{2}/", substr( $partname, 0, 2 ) ); } /** @@ -1590,6 +1605,32 @@ abstract class UploadBase { } else { return intval( $wgMaxUploadSize ); } + } + + /** + * Get the current status of a chunked upload (used for polling). + * The status will be read from the *current* user session. + * @param $statusKey string + * @return Array|bool + */ + public static function getSessionStatus( $statusKey ) { + return isset( $_SESSION[self::SESSION_STATUS_KEY][$statusKey] ) + ? $_SESSION[self::SESSION_STATUS_KEY][$statusKey] + : false; + } + /** + * Set the current status of a chunked upload (used for polling). + * The status will be stored in the *current* user session. + * @param $statusKey string + * @param $value array|false + * @return void + */ + public static function setSessionStatus( $statusKey, $value ) { + if ( $value === false ) { + unset( $_SESSION[self::SESSION_STATUS_KEY][$statusKey] ); + } else { + $_SESSION[self::SESSION_STATUS_KEY][$statusKey] = $value; + } } } diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php index 531f7be4..4b331e98 100644 --- a/includes/upload/UploadFromChunks.php +++ b/includes/upload/UploadFromChunks.php @@ -37,7 +37,7 @@ class UploadFromChunks extends UploadFromFile { * @param $stash UploadStash * @param $repo FileRepo */ - public function __construct( $user = false, $stash = false, $repo = false ) { + public function __construct( $user = null, $stash = false, $repo = false ) { // user object. sometimes this won't exist, as when running from cron. $this->user = $user; @@ -60,12 +60,13 @@ class UploadFromChunks extends UploadFromFile { return true; } + /** * Calls the parent stashFile and updates the uploadsession table to handle "chunks" * * @return UploadStashFile stashed file */ - public function stashFile() { + public function stashFile( User $user = null ) { // Stash file is the called on creating a new chunk session: $this->mChunkIndex = 0; $this->mOffset = 0; @@ -78,7 +79,7 @@ class UploadFromChunks extends UploadFromFile { $this->mFileKey = $this->mLocalFile->getFileKey(); // Output a copy of this first to chunk 0 location: - $status = $this->outputChunk( $this->mLocalFile->getPath() ); + $this->outputChunk( $this->mLocalFile->getPath() ); // Update db table to reflect initial "chunk" state $this->updateChunkStatus(); @@ -113,7 +114,7 @@ class UploadFromChunks extends UploadFromFile { // Concatenate all the chunks to mVirtualTempPath $fileList = Array(); // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1" - for( $i = 0; $i <= $this->getChunkIndex(); $i++ ){ + for( $i = 0; $i <= $this->getChunkIndex(); $i++ ) { $fileList[] = $this->getVirtualChunkLocation( $i ); } @@ -122,13 +123,16 @@ class UploadFromChunks extends UploadFromFile { // Get a 0-byte temp file to perform the concatenation at $tmpFile = TempFSFile::factory( 'chunkedupload_', $ext ); $tmpPath = $tmpFile - ? $tmpFile->getPath() + ? $tmpFile->bind( $this )->getPath() // keep alive with $this : false; // fail in concatenate() // Concatenate the chunks at the temp file + $tStart = microtime( true ); $status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE ); - if( !$status->isOk() ){ + $tAmount = microtime( true ) - $tStart; + if( !$status->isOk() ) { return $status; } + wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds.\n" ); $this->mTempPath = $tmpPath; // file system path $this->mFileSize = filesize( $this->mTempPath ); //Since this was set for the last chunk previously @@ -141,7 +145,11 @@ class UploadFromChunks extends UploadFromFile { // Update the mTempPath and mLocalFile // ( for FileUpload or normal Stash to take over ) - $this->mLocalFile = parent::stashFile(); + $tStart = microtime( true ); + $this->mLocalFile = parent::stashFile( $this->user ); + $tAmount = microtime( true ) - $tStart; + $this->mLocalFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo()) + wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds.\n" ); return $status; } @@ -164,7 +172,7 @@ class UploadFromChunks extends UploadFromFile { * @param $index * @return string */ - function getVirtualChunkLocation( $index ){ + function getVirtualChunkLocation( $index ) { return $this->repo->getVirtualUrl( 'temp' ) . '/' . $this->repo->getHashPath( @@ -176,9 +184,9 @@ class UploadFromChunks extends UploadFromFile { /** * Add a chunk to the temporary directory * - * @param $chunkPath string path to temporary chunk file - * @param $chunkSize int size of the current chunk - * @param $offset int offset of current chunk ( mutch match database chunk offset ) + * @param string $chunkPath path to temporary chunk file + * @param int $chunkSize size of the current chunk + * @param int $offset offset of current chunk ( mutch match database chunk offset ) * @return Status */ public function addChunk( $chunkPath, $chunkSize, $offset ) { @@ -202,7 +210,7 @@ class UploadFromChunks extends UploadFromFile { return Status::newFatal( $e->getMessage() ); } $status = $this->outputChunk( $chunkPath ); - if( $status->isGood() ){ + if( $status->isGood() ) { // Update local offset: $this->mOffset = $preAppendOffset + $chunkSize; // Update chunk table status db @@ -218,11 +226,14 @@ class UploadFromChunks extends UploadFromFile { /** * Update the chunk db table with the current status: */ - private function updateChunkStatus(){ + private function updateChunkStatus() { wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" . $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" ); $dbw = $this->repo->getMasterDb(); + // Use a quick transaction since we will upload the full temp file into shared + // storage, which takes time for large files. We don't want to hold locks then. + $dbw->begin( __METHOD__ ); $dbw->update( 'uploadstash', array( @@ -233,12 +244,13 @@ class UploadFromChunks extends UploadFromFile { array( 'us_key' => $this->mFileKey ), __METHOD__ ); + $dbw->commit( __METHOD__ ); } /** * Get the chunk db state and populate update relevant local values */ - private function getChunkStatus(){ + private function getChunkStatus() { // get Master db to avoid race conditions. // Otherwise, if chunk upload time < replag there will be spurious errors $dbw = $this->repo->getMasterDb(); @@ -264,8 +276,8 @@ class UploadFromChunks extends UploadFromFile { * Get the current Chunk index * @return Integer index of the current chunk */ - private function getChunkIndex(){ - if( $this->mChunkIndex !== null ){ + private function getChunkIndex() { + if( $this->mChunkIndex !== null ) { return $this->mChunkIndex; } return 0; @@ -275,8 +287,8 @@ class UploadFromChunks extends UploadFromFile { * Gets the current offset in fromt the stashedupload table * @return Integer current byte offset of the chunk file set */ - private function getOffset(){ - if ( $this->mOffset !== null ){ + private function getOffset() { + if ( $this->mOffset !== null ) { return $this->mOffset; } return 0; @@ -289,7 +301,7 @@ class UploadFromChunks extends UploadFromFile { * @throws UploadChunkFileException * @return FileRepoStatus */ - private function outputChunk( $chunkPath ){ + private function outputChunk( $chunkPath ) { // Key is fileKey + chunk index $fileKey = $this->getChunkFileKey(); @@ -314,11 +326,11 @@ class UploadFromChunks extends UploadFromFile { return $storeStatus; } - private function getChunkFileKey( $index = null ){ - if( $index === null ){ + private function getChunkFileKey( $index = null ) { + if( $index === null ) { $index = $this->getChunkIndex(); } - return $this->mFileKey . '.' . $index ; + return $this->mFileKey . '.' . $index; } /** diff --git a/includes/upload/UploadFromFile.php b/includes/upload/UploadFromFile.php index aa0cc77b..ab2a7a39 100644 --- a/includes/upload/UploadFromFile.php +++ b/includes/upload/UploadFromFile.php @@ -79,21 +79,21 @@ class UploadFromFile extends UploadBase { * @return array */ public function verifyUpload() { - # Check for a post_max_size or upload_max_size overflow, so that a + # Check for a post_max_size or upload_max_size overflow, so that a # proper error can be shown to the user if ( is_null( $this->mTempPath ) || $this->isEmptyFile() ) { if ( $this->mUpload->isIniSizeOverflow() ) { - return array( + return array( 'status' => UploadBase::FILE_TOO_LARGE, - 'max' => min( - self::getMaxUploadSize( $this->getSourceType() ), - wfShorthandToInteger( ini_get( 'upload_max_filesize' ) ), + 'max' => min( + self::getMaxUploadSize( $this->getSourceType() ), + wfShorthandToInteger( ini_get( 'upload_max_filesize' ) ), wfShorthandToInteger( ini_get( 'post_max_size' ) ) ), ); } } - + return parent::verifyUpload(); } } diff --git a/includes/upload/UploadFromStash.php b/includes/upload/UploadFromStash.php index d79641ce..c82103cb 100644 --- a/includes/upload/UploadFromStash.php +++ b/includes/upload/UploadFromStash.php @@ -89,7 +89,7 @@ class UploadFromStash extends UploadBase { * @param $key string * @param $name string */ - public function initialize( $key, $name = 'upload_file' ) { + public function initialize( $key, $name = 'upload_file', $initTempFile = true ) { /** * Confirming a temporarily stashed upload. * We don't want path names to be forged, so we keep @@ -98,7 +98,7 @@ class UploadFromStash extends UploadBase { */ $metadata = $this->stash->getMetadata( $key ); $this->initializePathInfo( $name, - $this->getRealPath ( $metadata['us_path'] ), + $initTempFile ? $this->getRealPath( $metadata['us_path'] ) : false, $metadata['us_size'], false ); @@ -129,6 +129,14 @@ class UploadFromStash extends UploadBase { return $this->mSourceType; } + /** + * Get the base 36 SHA1 of the file + * @return string + */ + public function getTempFileSha1Base36() { + return $this->mFileProps['sha1']; + } + /* * protected function verifyFile() inherited */ @@ -136,12 +144,13 @@ class UploadFromStash extends UploadBase { /** * Stash the file. * + * @param $user User * @return UploadStashFile */ - public function stashFile() { + public function stashFile( User $user = null ) { // replace mLocalFile with an instance of UploadStashFile, which adds some methods // that are useful for stashed files. - $this->mLocalFile = parent::stashFile(); + $this->mLocalFile = parent::stashFile( $user ); return $this->mLocalFile; } diff --git a/includes/upload/UploadFromUrl.php b/includes/upload/UploadFromUrl.php index 927c3cd9..70b69034 100644 --- a/includes/upload/UploadFromUrl.php +++ b/includes/upload/UploadFromUrl.php @@ -61,6 +61,8 @@ class UploadFromUrl extends UploadBase { /** * Checks whether the URL is for an allowed host + * The domains in the whitelist can include wildcard characters (*) in place + * of any of the domain levels, e.g. '*.flickr.com' or 'upload.*.gov.uk'. * * @param $url string * @return bool @@ -76,10 +78,28 @@ class UploadFromUrl extends UploadBase { } $valid = false; foreach( $wgCopyUploadsDomains as $domain ) { + // See if the domain for the upload matches this whitelisted domain + $whitelistedDomainPieces = explode( '.', $domain ); + $uploadDomainPieces = explode( '.', $parsedUrl['host'] ); + if ( count( $whitelistedDomainPieces ) === count( $uploadDomainPieces ) ) { + $valid = true; + // See if all the pieces match or not (excluding wildcards) + foreach ( $whitelistedDomainPieces as $index => $piece ) { + if ( $piece !== '*' && $piece !== $uploadDomainPieces[$index] ) { + $valid = false; + } + } + if ( $valid ) { + // We found a match, so quit comparing against the list + break; + } + } + /* Non-wildcard test if ( $parsedUrl['host'] === $domain ) { $valid = true; break; } + */ } return $valid; } @@ -312,7 +332,7 @@ class UploadFromUrl extends UploadBase { 'sessionKey' => $sessionKey, ) ); $job->initializeSessionData(); - $job->insert(); + JobQueueGroup::singleton()->push( $job ); return $sessionKey; } diff --git a/includes/upload/UploadStash.php b/includes/upload/UploadStash.php index 53a90582..089bd8b7 100644 --- a/includes/upload/UploadStash.php +++ b/includes/upload/UploadStash.php @@ -97,7 +97,7 @@ class UploadStash { * Get a file and its metadata from the stash. * The noAuth param is a bit janky but is required for automated scripts which clean out the stash. * - * @param $key String: key under which file information is stored + * @param string $key key under which file information is stored * @param $noAuth Boolean (optional) Don't check authentication. Used by maintenance scripts. * @throws UploadStashFileNotFoundException * @throws UploadStashNotLoggedInException @@ -106,15 +106,13 @@ class UploadStash { * @return UploadStashFile */ public function getFile( $key, $noAuth = false ) { - if ( ! preg_match( self::KEY_FORMAT_REGEX, $key ) ) { throw new UploadStashBadPathException( "key '$key' is not in a proper format" ); } - if ( !$noAuth ) { - if ( !$this->isLoggedIn ) { - throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' ); - } + if ( !$noAuth && !$this->isLoggedIn ) { + throw new UploadStashNotLoggedInException( __METHOD__ . + ' No user is logged in, files must belong to users' ); } if ( !isset( $this->fileMetadata[$key] ) ) { @@ -131,8 +129,13 @@ class UploadStash { $this->initFile( $key ); // fetch fileprops - $path = $this->fileMetadata[$key]['us_path']; - $this->fileProps[$key] = $this->repo->getFileProps( $path ); + if ( strlen( $this->fileMetadata[$key]['us_props'] ) ) { + $this->fileProps[$key] = unserialize( $this->fileMetadata[$key]['us_props'] ); + } else { // b/c for rows with no us_props + wfDebug( __METHOD__ . " fetched props for $key from file\n" ); + $path = $this->fileMetadata[$key]['us_path']; + $this->fileProps[$key] = $this->repo->getFileProps( $path ); + } } if ( ! $this->files[$key]->exists() ) { @@ -152,7 +155,7 @@ class UploadStash { /** * Getter for file metadata. * - * @param key String: key under which file information is stored + * @param string $key key under which file information is stored * @return Array */ public function getMetadata ( $key ) { @@ -163,7 +166,7 @@ class UploadStash { /** * Getter for fileProps * - * @param key String: key under which file information is stored + * @param string $key key under which file information is stored * @return Array */ public function getFileProps ( $key ) { @@ -174,15 +177,15 @@ class UploadStash { /** * Stash a file in a temp directory and record that we did this in the database, along with other metadata. * - * @param $path String: path to file you want stashed - * @param $sourceType String: the type of upload that generated this file (currently, I believe, 'file' or null) + * @param string $path path to file you want stashed + * @param string $sourceType the type of upload that generated this file (currently, I believe, 'file' or null) * @throws UploadStashBadPathException * @throws UploadStashFileException * @throws UploadStashNotLoggedInException * @return UploadStashFile: file, or null on failure */ public function stashFile( $path, $sourceType = null ) { - if ( ! file_exists( $path ) ) { + if ( !is_file( $path ) ) { wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" ); throw new UploadStashBadPathException( "path doesn't exist" ); } @@ -192,12 +195,10 @@ class UploadStash { // we will be initializing from some tmpnam files that don't have extensions. // most of MediaWiki assumes all uploaded files have good extensions. So, we fix this. $extension = self::getExtensionForPath( $path ); - if ( ! preg_match( "/\\.\\Q$extension\\E$/", $path ) ) { + if ( !preg_match( "/\\.\\Q$extension\\E$/", $path ) ) { $pathWithGoodExtension = "$path.$extension"; - if ( ! rename( $path, $pathWithGoodExtension ) ) { - throw new UploadStashFileException( "couldn't rename $path to have a better extension at $pathWithGoodExtension" ); - } - $path = $pathWithGoodExtension; + } else { + $pathWithGoodExtension = $path; } // If no key was supplied, make one. a mysql insertid would be totally reasonable here, except @@ -205,8 +206,8 @@ class UploadStash { // // some things that when combined will make a suitably unique key. // see: http://www.jwz.org/doc/mid.html - list ($usec, $sec) = explode( ' ', microtime() ); - $usec = substr($usec, 2); + list( $usec, $sec ) = explode( ' ', microtime() ); + $usec = substr( $usec, 2 ); $key = wfBaseConvert( $sec . $usec, 10, 36 ) . '.' . wfBaseConvert( mt_rand(), 10, 36 ) . '.'. $this->userId . '.' . @@ -221,7 +222,7 @@ class UploadStash { wfDebug( __METHOD__ . " key for '$path': $key\n" ); // if not already in a temporary area, put it there - $storeStatus = $this->repo->storeTemp( basename( $path ), $path ); + $storeStatus = $this->repo->storeTemp( basename( $pathWithGoodExtension ), $path ); if ( ! $storeStatus->isOK() ) { // It is a convention in MediaWiki to only return one error per API exception, even if multiple errors @@ -244,9 +245,6 @@ class UploadStash { } $stashPath = $storeStatus->value; - // we have renamed the file so we have to cleanup once done - unlink($path); - // fetch the current user ID if ( !$this->isLoggedIn ) { throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' ); @@ -262,6 +260,7 @@ class UploadStash { 'us_key' => $key, 'us_orig_path' => $path, 'us_path' => $stashPath, // virtual URL + 'us_props' => serialize( $fileProps ), 'us_size' => $fileProps['size'], 'us_sha1' => $fileProps['sha1'], 'us_mime' => $fileProps['mime'], @@ -319,8 +318,8 @@ class UploadStash { /** * Remove a particular file from the stash. Also removes it from the repo. * - * @throws UploadStashNotLoggedInException - * @throws UploadStashWrongOwnerException + * @param $key + * @throws UploadStashNoSuchKeyException|UploadStashNotLoggedInException|UploadStashWrongOwnerException * @return boolean: success */ public function removeFile( $key ) { @@ -350,7 +349,6 @@ class UploadStash { return $this->removeFileNoAuth( $key ); } - /** * Remove a file (see removeFile), but doesn't check ownership first. * @@ -359,16 +357,16 @@ class UploadStash { public function removeFileNoAuth( $key ) { wfDebug( __METHOD__ . " clearing row $key\n" ); + // Ensure we have the UploadStashFile loaded for this key + $this->getFile( $key ); + $dbw = $this->repo->getMasterDb(); - // this gets its own transaction since it's called serially by the cleanupUploadStash maintenance script - $dbw->begin( __METHOD__ ); $dbw->delete( 'uploadstash', array( 'us_key' => $key ), __METHOD__ ); - $dbw->commit( __METHOD__ ); // TODO: look into UnregisteredLocalFile and find out why the rv here is sometimes wrong (false when file was removed) // for now, ignore. @@ -419,6 +417,8 @@ class UploadStash { * with an extension. * XXX this is somewhat redundant with the checks that ApiUpload.php does with incoming * uploads versus the desired filename. Maybe we can get that passed to us... + * @param $path + * @throws UploadStashFileException * @return string */ public static function getExtensionForPath( $path ) { @@ -456,7 +456,7 @@ class UploadStash { /** * Helper function: do the actual database query to fetch file metadata. * - * @param $key String: key + * @param string $key key * @param $readFromDB: constant (default: DB_SLAVE) * @return boolean */ @@ -490,7 +490,7 @@ class UploadStash { /** * Helper function: Initialize the UploadStashFile for a given file. * - * @param $key String: key under which to store the object + * @param string $key key under which to store the object * @throws UploadStashZeroLengthFileException * @return bool */ @@ -514,8 +514,8 @@ class UploadStashFile extends UnregisteredLocalFile { * Arguably UnregisteredLocalFile should be handling its own file repo but that class is a bit retarded currently * * @param $repo FileRepo: repository where we should find the path - * @param $path String: path to file - * @param $key String: key to store the path and any stashed data under + * @param string $path path to file + * @param string $key key to store the path and any stashed data under * @throws UploadStashBadPathException * @throws UploadStashFileNotFoundException */ @@ -564,7 +564,7 @@ class UploadStashFile extends UnregisteredLocalFile { * The actual argument is the result of thumbName although we seem to have * buggy code elsewhere that expects a boolean 'suffix' * - * @param $thumbName String: name of thumbnail (e.g. "120px-123456.jpg" ), or false to just get the path + * @param string $thumbName name of thumbnail (e.g. "120px-123456.jpg" ), or false to just get the path * @return String: path thumbnail should take on filesystem, or containing directory if thumbname is false */ public function getThumbPath( $thumbName = false ) { @@ -580,7 +580,7 @@ class UploadStashFile extends UnregisteredLocalFile { * We override this because we want to use the pretty url name instead of the * ugly file name. * - * @param $params Array: handler-specific parameters + * @param array $params handler-specific parameters * @param $flags integer Bitfield that supports THUMB_* constants * @return String: base name for URL, like '120px-12345.jpg', or null if there is no handler */ @@ -603,7 +603,7 @@ class UploadStashFile extends UnregisteredLocalFile { * the thumbnail urls be predictable. However, in our model the URL is not based on the filename * (that's hidden in the db) * - * @param $thumbName String: basename of thumbnail file -- however, we don't want to use the file exactly + * @param string $thumbName basename of thumbnail file -- however, we don't want to use the file exactly * @return String: URL to access thumbnail, or URL with partial path */ public function getThumbUrl( $thumbName = false ) { |