diff options
Diffstat (limited to 'maintenance/hiphop/make')
-rw-r--r-- | maintenance/hiphop/make | 308 |
1 files changed, 308 insertions, 0 deletions
diff --git a/maintenance/hiphop/make b/maintenance/hiphop/make new file mode 100644 index 00000000..e792e08b --- /dev/null +++ b/maintenance/hiphop/make @@ -0,0 +1,308 @@ +#!/usr/bin/hphpi -f +<?php + +require( dirname( __FILE__ ) . '/../Maintenance.php' ); + +class MakeHipHop extends Maintenance { + function execute() { + global $wgHipHopBuildDirectory; + + $startTime = time(); + + $thisDir = realpath( dirname( __FILE__ ) ); + $IP = realpath( "$thisDir/../.." ); + if ( strval( $wgHipHopBuildDirectory ) !== '' ) { + $buildDir = $wgHipHopBuildDirectory; + } else { + $buildDir = "$thisDir/build"; + } + $extensionsDir = realpath( MWInit::getExtensionsDirectory() ); + $outDir = "$buildDir/hiphop-output"; + $persistentDir = "$buildDir/persistent"; + + if ( !is_dir( $buildDir ) ) { + mkdir( $buildDir, 0777, true ); + } + if ( !is_dir( $persistentDir ) ) { + mkdir( $persistentDir, 0777, true ); + } + + if ( realpath( "$IP/../phase3" ) !== $IP + || realpath( "$IP/../extensions" ) !== $extensionsDir ) + { + # Set up a fake source directory with the correct layout + $sourceBase = "$buildDir/source"; + $this->setupFakeSourceBase( $IP, $extensionsDir, $sourceBase ); + } else { + $sourceBase = realpath( "$IP/.." ); + unlink( "$buildDir/source" ); + } + + # With the CentOS RPMs, you just get g++44, no g++, so we have to + # use the environment + if ( isset( $_ENV['CXX'] ) ) { + $cxx = $_ENV['CXX']; + } else { + $cxx = 'g++'; + } + + # Create a function that provides the HipHop compiler version, and + # doesn't exist when MediaWiki is invoked in interpreter mode. + $version = str_replace( PHP_EOL, ' ', trim( `hphp --version` ) ); + file_put_contents( + "$buildDir/HipHopCompilerVersion.php", + "<" . "?php\n" . + "function wfHipHopCompilerVersion() {\n" . + "return " . var_export( $version, true ) . ";\n" . + "}\n" + ); + + # Generate the file list + $files = $this->getFileList(); + file_put_contents( + "$buildDir/file-list", + implode( "\n", $files ) . "\n" ); + + # Generate the C++ + passthru( + 'hphp' . + ' --target=cpp' . + ' --format=file' . + ' --input-dir=' . wfEscapeShellArg( $sourceBase ) . + ' --input-list=' . wfEscapeShellArg( "$buildDir/file-list" ) . + ' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) . + ' -c ' . wfEscapeShellArg( "$thisDir/compiler.conf" ) . + ' --parse-on-demand=false' . + ' --program=mediawiki-hphp' . + ' --output-dir=' . wfEscapeShellArg( $outDir ) . + ' --log=3', $ret ); + + if ( $ret ) { + $this->error( "hphp hit an error. Stopping build.\n" ); + exit( 1 ); + } + + # Sanity check, quickly make sure we've got an output directory + if( !is_dir( $outDir ) ) { + $this->error( "No output directory", true ); + } + + # Warn about volatile classes + $this->checkVolatileClasses( $outDir ); + + # Copy the generated C++ files into the source directory for cmake + $iter = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator( $outDir ), + RecursiveIteratorIterator::SELF_FIRST ); + $sourceFiles = array(); + $regenerateMakefile = false; + $numFiles = 0; + $numFilesChanged = 0; + foreach ( $iter as $sourcePath => $file ) { + $name = substr( $sourcePath, strlen( $outDir ) + 1 ); + $sourceFiles[$name] = true; + $destPath = "$persistentDir/$name"; + if ( $file->isDir() ) { + if ( !is_dir( $destPath ) ) { + mkdir( $destPath ); + } + continue; + } + + $numFiles++; + # Remove any files that weren't touched, these may have been removed + # from file-list, we should not compile them + if ( $file->getMTime() < $startTime ) { + if ( file_exists( $destPath ) ) { + unlink( $destPath ); + # Files removed, regenerate the makefile + $regenerateMakefile = true; + } + unlink( $sourcePath ); + $numFilesChanged++; + continue; + } + + if ( file_exists( $destPath ) ) { + $sourceHash = md5( file_get_contents( $sourcePath ) ); + $destHash = md5( file_get_contents( $destPath ) ); + if ( $sourceHash == $destHash ) { + continue; + } + } else { + # New files added, regenerate the makefile + $regenerateMakefile = true; + } + $numFilesChanged++; + copy( $sourcePath, $destPath ); + } + + echo "MediaWiki: $numFilesChanged files changed out of $numFiles\n"; + + if ( !file_exists( "$persistentDir/CMakeLists.txt" ) ) { + # Run cmake for the first time + $regenerateMakefile = true; + } + + # Do our own version of $HPHP_HOME/bin/run.sh, which isn't so broken. + # HipHop's RELEASE mode seems to be stuck always on, so symbols get + # stripped. Also we will try keeping the generated .o files instead of + # throwing away hours of CPU time every time you make a typo. + + chdir( $persistentDir ); + + if ( $regenerateMakefile ) { + copy( $_ENV['HPHP_HOME'] . '/bin/CMakeLists.base.txt', + "$persistentDir/CMakeLists.txt" ); + + if ( file_exists( "$persistentDir/CMakeCache.txt" ) ) { + unlink( "$persistentDir/CMakeCache.txt" ); + } + + $cmd = 'cmake' . + " -D CMAKE_BUILD_TYPE:string=" . wfEscapeShellArg( $GLOBALS['wgHipHopBuildType'] ) . + ' -D PROGRAM_NAME:string=mediawiki-hphp'; + + if ( file_exists( '/usr/bin/ccache' ) ) { + $cmd .= ' -D CMAKE_CXX_COMPILER:string=ccache' . + ' -D CMAKE_CXX_COMPILER_ARG1:string=' . wfEscapeShellArg( $cxx ); + } + + $cmd .= ' .'; + echo "$cmd\n"; + passthru( $cmd ); + } + + # Determine appropriate make concurrency + # Compilation can take a lot of memory, let's assume that that is limiting. + $procs = $this->getNumProcs(); + + # Run make. This is the slow step. + passthru( 'make -j' . wfEscapeShellArg( $procs ) ); + + $elapsed = time() - $startTime; + + echo "Completed in "; + if ( $elapsed >= 3600 ) { + $hours = floor( $elapsed / 3600 ); + echo $hours . 'h '; + $elapsed -= $hours * 3600; + } + if ( $elapsed >= 60 ) { + $minutes = floor( $elapsed / 60 ); + echo $minutes . 'm '; + $elapsed -= $minutes * 60; + } + echo $elapsed . "s\n"; + echo "The MediaWiki executable is at $buildDir/persistent/mediawiki-hphp\n"; + } + + function checkVolatileClasses( $dir ) { + $lines = file( "$dir/sys/dynamic_table_class.cpp" ); + $classes = array(); + foreach ( $lines as $line ) { + if ( preg_match( '/^\s+\(const char \*\)"([^"]*)", \(const char \*\)-1/', $line, $m ) ) { + $classes[] = $m[1]; + } + } + if ( !count( $classes ) ) { + print "No volatile classes found\n"; + return; + } + sort( $classes ); + $classes = array_unique( $classes ); + print "WARNING: The following classes are volatile: " . implode( ', ', $classes ) . "\n"; + } + + function getNumProcs() { + global $wgHipHopCompilerProcs; + if ( $wgHipHopCompilerProcs !== 'detect' ) { + return intval( $wgHipHopCompilerProcs ); + } + + if ( !file_exists( '/proc/meminfo' ) ) { + return 1; + } + $mem = false; + foreach ( file( '/proc/meminfo' ) as $line ) { + if ( preg_match( '/^MemTotal:\s+(\d+)\s+kB/', $line, $m ) ) { + $mem = intval( $m[1] ); + break; + } + } + if ( $mem ) { + // At least one process + return max( 1, floor( $mem / 1000000 ) ); + } else { + return 1; + } + } + + function setupFakeSourceBase( $phase3, $extensions, $dest ) { + if ( !file_exists( $dest ) ) { + mkdir( $dest, 0777, true ); + } + + $this->forceCreateLink( "$dest/phase3", $phase3 ); + $this->forceCreateLink( "$dest/extensions", $extensions ); + } + + function forceCreateLink( $target, $link ) { + if ( file_exists( $target ) ) { + if ( readlink( $target ) === $link ) { + return; + } + unlink( $target ); + } + symlink( $target, $link ); + } + + function getFileList() { + global $wgAutoloadClasses, $wgAutoloadLocalClasses, $wgCompiledFiles; + $inputFiles = array_merge( + array_values( $wgAutoloadClasses ), + array_values( $wgAutoloadLocalClasses ), + $wgCompiledFiles + ); + $processedFiles = array(); + foreach ( $inputFiles as $file ) { + if ( substr( $file, 0, 1 ) === '/' ) { + $processedFiles[] = $this->absoluteToRelative( $file ); + } elseif ( preg_match( '/^extensions/', $file ) ) { + $processedFiles[] = $file; + } else { + $processedFiles[] = "phase3/$file"; + } + } + + $extraCoreFiles = array_map( 'trim', file( dirname( __FILE__ ) . '/extra-files' ) ); + foreach ( $extraCoreFiles as $file ) { + if ( $file === '' ) { + continue; + } + $processedFiles[] = "phase3/$file"; + } + return array_unique( $processedFiles ); + } + + function absoluteToRelative( $file ) { + global $IP; + + $coreBase = realpath( $IP ) . '/'; + $extBase = realpath( MWInit::getExtensionsDirectory() ) . '/'; + $file = realpath( $file ); + + if ( substr( $file, 0, strlen( $extBase ) ) === $extBase ) { + return 'extensions/' . substr( $file, strlen( $extBase ) ); + } elseif ( substr( $file, 0, strlen( $coreBase ) ) === $coreBase ) { + return 'phase3/' . substr( $file, strlen( $coreBase ) ); + } else { + $this->error( "The following file is registered for compilation but is not in \$IP or " . + "\$wgExtensionsDirectory: $file \n" ); + exit( 1 ); + } + } +} + +$maintClass = 'MakeHipHop'; +require_once( RUN_MAINTENANCE_IF_MAIN ); |