diff options
Diffstat (limited to 'includes/site')
| -rw-r--r-- | includes/site/CachingSiteStore.php | 195 | ||||
| -rw-r--r-- | includes/site/DBSiteStore.php | 345 | ||||
| -rw-r--r-- | includes/site/FileBasedSiteLookup.php | 139 | ||||
| -rw-r--r-- | includes/site/HashSiteStore.php | 123 | ||||
| -rw-r--r-- | includes/site/MediaWikiSite.php | 4 | ||||
| -rw-r--r-- | includes/site/SiteExporter.php | 114 | ||||
| -rw-r--r-- | includes/site/SiteImporter.php | 263 | ||||
| -rw-r--r-- | includes/site/SiteLookup.php | 50 | ||||
| -rw-r--r-- | includes/site/SiteSQLStore.php | 459 | ||||
| -rw-r--r-- | includes/site/SiteStore.php | 29 | ||||
| -rw-r--r-- | includes/site/SitesCacheFileBuilder.php | 113 |
11 files changed, 1353 insertions, 481 deletions
diff --git a/includes/site/CachingSiteStore.php b/includes/site/CachingSiteStore.php new file mode 100644 index 00000000..9243f12b --- /dev/null +++ b/includes/site/CachingSiteStore.php @@ -0,0 +1,195 @@ +<?php + +/** + * Represents the site configuration of a wiki. + * Holds a list of sites (ie SiteList), with a caching layer. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @since 1.25 + * + * @file + * @ingroup Site + * + * @license GNU GPL v2+ + * @author Jeroen De Dauw < jeroendedauw@gmail.com > + * @author Katie Filbert < aude.wiki@gmail.com > + */ +class CachingSiteStore implements SiteStore { + + /** + * @var SiteList|null + */ + private $sites = null; + + /** + * @var string|null + */ + private $cacheKey; + + /** + * @var int + */ + private $cacheTimeout; + + /** + * @var BagOStuff + */ + private $cache; + + /** + * @var SiteStore + */ + private $siteStore; + + /** + * @param SiteStore $siteStore + * @param BagOStuff $cache + * @param string|null $cacheKey + * @param int $cacheTimeout + */ + public function __construct( + SiteStore $siteStore, + BagOStuff $cache, + $cacheKey = null, + $cacheTimeout = 3600 + ) { + $this->siteStore = $siteStore; + $this->cache = $cache; + $this->cacheKey = $cacheKey; + $this->cacheTimeout = $cacheTimeout; + } + + /** + * Constructs a cache key to use for caching the list of sites. + * + * This includes the concrete class name of the site list as well as a version identifier + * for the list's serialization, to avoid problems when unserializing site lists serialized + * by an older version, e.g. when reading from a cache. + * + * The cache key also includes information about where the sites were loaded from, e.g. + * the name of a database table. + * + * @see SiteList::getSerialVersionId + * + * @return string The cache key. + */ + private function getCacheKey() { + if ( $this->cacheKey === null ) { + $type = 'SiteList#' . SiteList::getSerialVersionId(); + $this->cacheKey = wfMemcKey( "sites/$type" ); + } + + return $this->cacheKey; + } + + /** + * @see SiteStore::getSites + * + * @since 1.25 + * + * @return SiteList + */ + public function getSites() { + if ( $this->sites === null ) { + $this->sites = $this->cache->get( $this->getCacheKey() ); + + if ( !is_object( $this->sites ) ) { + $this->sites = $this->siteStore->getSites(); + + $this->cache->set( $this->getCacheKey(), $this->sites, $this->cacheTimeout ); + } + } + + return $this->sites; + } + + /** + * @see SiteStore::getSite + * + * @since 1.25 + * + * @param string $globalId + * + * @return Site|null + */ + public function getSite( $globalId ) { + $sites = $this->getSites(); + + return $sites->hasSite( $globalId ) ? $sites->getSite( $globalId ) : null; + } + + /** + * @see SiteStore::saveSite + * + * @since 1.25 + * + * @param Site $site + * + * @return bool Success indicator + */ + public function saveSite( Site $site ) { + return $this->saveSites( array( $site ) ); + } + + /** + * @see SiteStore::saveSites + * + * @since 1.25 + * + * @param Site[] $sites + * + * @return bool Success indicator + */ + public function saveSites( array $sites ) { + if ( empty( $sites ) ) { + return true; + } + + $success = $this->siteStore->saveSites( $sites ); + + // purge cache + $this->reset(); + + return $success; + } + + /** + * Purges the internal and external cache of the site list, forcing the list + * of sites to be reloaded. + * + * @since 1.25 + */ + public function reset() { + // purge cache + $this->cache->delete( $this->getCacheKey() ); + $this->sites = null; + } + + /** + * Clears the list of sites stored. + * + * @see SiteStore::clear() + * + * @return bool Success + */ + public function clear() { + $this->reset(); + + return $this->siteStore->clear(); + } + +} diff --git a/includes/site/DBSiteStore.php b/includes/site/DBSiteStore.php new file mode 100644 index 00000000..f167584e --- /dev/null +++ b/includes/site/DBSiteStore.php @@ -0,0 +1,345 @@ +<?php + +/** + * Represents the site configuration of a wiki. + * Holds a list of sites (ie SiteList), stored in the database. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @since 1.25 + * + * @file + * @ingroup Site + * + * @license GNU GPL v2+ + * @author Jeroen De Dauw < jeroendedauw@gmail.com > + */ +class DBSiteStore implements SiteStore { + + /** + * @var SiteList|null + */ + protected $sites = null; + + /** + * @var ORMTable + */ + protected $sitesTable; + + /** + * @since 1.25 + * + * @param ORMTable|null $sitesTable + */ + public function __construct( ORMTable $sitesTable = null ) { + if ( $sitesTable === null ) { + $sitesTable = $this->newSitesTable(); + } + + $this->sitesTable = $sitesTable; + } + + /** + * @see SiteStore::getSites + * + * @since 1.25 + * + * @return SiteList + */ + public function getSites() { + $this->loadSites(); + + return $this->sites; + } + + /** + * Returns a new Site object constructed from the provided ORMRow. + * + * @since 1.25 + * + * @param ORMRow $siteRow + * + * @return Site + */ + protected function siteFromRow( ORMRow $siteRow ) { + + $site = Site::newForType( $siteRow->getField( 'type', Site::TYPE_UNKNOWN ) ); + + $site->setGlobalId( $siteRow->getField( 'global_key' ) ); + + $site->setInternalId( $siteRow->getField( 'id' ) ); + + if ( $siteRow->hasField( 'forward' ) ) { + $site->setForward( $siteRow->getField( 'forward' ) ); + } + + if ( $siteRow->hasField( 'group' ) ) { + $site->setGroup( $siteRow->getField( 'group' ) ); + } + + if ( $siteRow->hasField( 'language' ) ) { + $site->setLanguageCode( $siteRow->getField( 'language' ) === '' + ? null + : $siteRow->getField( 'language' ) + ); + } + + if ( $siteRow->hasField( 'source' ) ) { + $site->setSource( $siteRow->getField( 'source' ) ); + } + + if ( $siteRow->hasField( 'data' ) ) { + $site->setExtraData( $siteRow->getField( 'data' ) ); + } + + if ( $siteRow->hasField( 'config' ) ) { + $site->setExtraConfig( $siteRow->getField( 'config' ) ); + } + + return $site; + } + + /** + * Get a new ORMRow from a Site object + * + * @since 1.25 + * + * @param Site $site + * + * @return ORMRow + */ + protected function getRowFromSite( Site $site ) { + $fields = array( + // Site data + 'global_key' => $site->getGlobalId(), // TODO: check not null + 'type' => $site->getType(), + 'group' => $site->getGroup(), + 'source' => $site->getSource(), + 'language' => $site->getLanguageCode() === null ? '' : $site->getLanguageCode(), + 'protocol' => $site->getProtocol(), + 'domain' => strrev( $site->getDomain() ) . '.', + 'data' => $site->getExtraData(), + + // Site config + 'forward' => $site->shouldForward(), + 'config' => $site->getExtraConfig(), + ); + + if ( $site->getInternalId() !== null ) { + $fields['id'] = $site->getInternalId(); + } + + return new ORMRow( $this->sitesTable, $fields ); + } + + /** + * Fetches the site from the database and loads them into the sites field. + * + * @since 1.25 + */ + protected function loadSites() { + $this->sites = new SiteList(); + + foreach ( $this->sitesTable->select() as $siteRow ) { + $this->sites[] = $this->siteFromRow( $siteRow ); + } + + // Batch load the local site identifiers. + $ids = wfGetDB( $this->sitesTable->getReadDb() )->select( + 'site_identifiers', + array( + 'si_site', + 'si_type', + 'si_key', + ), + array(), + __METHOD__ + ); + + foreach ( $ids as $id ) { + if ( $this->sites->hasInternalId( $id->si_site ) ) { + $site = $this->sites->getSiteByInternalId( $id->si_site ); + $site->addLocalId( $id->si_type, $id->si_key ); + $this->sites->setSite( $site ); + } + } + } + + /** + * @see SiteStore::getSite + * + * @since 1.25 + * + * @param string $globalId + * + * @return Site|null + */ + public function getSite( $globalId ) { + if ( $this->sites === null ) { + $this->sites = $this->getSites(); + } + + return $this->sites->hasSite( $globalId ) ? $this->sites->getSite( $globalId ) : null; + } + + /** + * @see SiteStore::saveSite + * + * @since 1.25 + * + * @param Site $site + * + * @return bool Success indicator + */ + public function saveSite( Site $site ) { + return $this->saveSites( array( $site ) ); + } + + /** + * @see SiteStore::saveSites + * + * @since 1.25 + * + * @param Site[] $sites + * + * @return bool Success indicator + */ + public function saveSites( array $sites ) { + if ( empty( $sites ) ) { + return true; + } + + $dbw = $this->sitesTable->getWriteDbConnection(); + + $dbw->startAtomic( __METHOD__ ); + + $success = true; + + $internalIds = array(); + $localIds = array(); + + foreach ( $sites as $site ) { + if ( $site->getInternalId() !== null ) { + $internalIds[] = $site->getInternalId(); + } + + $siteRow = $this->getRowFromSite( $site ); + $success = $siteRow->save( __METHOD__ ) && $success; + + foreach ( $site->getLocalIds() as $idType => $ids ) { + foreach ( $ids as $id ) { + $localIds[] = array( $siteRow->getId(), $idType, $id ); + } + } + } + + if ( $internalIds !== array() ) { + $dbw->delete( + 'site_identifiers', + array( 'si_site' => $internalIds ), + __METHOD__ + ); + } + + foreach ( $localIds as $localId ) { + $dbw->insert( + 'site_identifiers', + array( + 'si_site' => $localId[0], + 'si_type' => $localId[1], + 'si_key' => $localId[2], + ), + __METHOD__ + ); + } + + $dbw->endAtomic( __METHOD__ ); + + $this->reset(); + + return $success; + } + + /** + * Resets the SiteList + * + * @since 1.25 + */ + public function reset() { + $this->sites = null; + } + + /** + * Clears the list of sites stored in the database. + * + * @see SiteStore::clear() + * + * @return bool Success + */ + public function clear() { + $dbw = $this->sitesTable->getWriteDbConnection(); + + $dbw->startAtomic( __METHOD__ ); + $ok = $dbw->delete( 'sites', '*', __METHOD__ ); + $ok = $dbw->delete( 'site_identifiers', '*', __METHOD__ ) && $ok; + $dbw->endAtomic( __METHOD__ ); + + $this->reset(); + + return $ok; + } + + /** + * @since 1.25 + * + * @return ORMTable + */ + protected function newSitesTable() { + return new ORMTable( + 'sites', + array( + 'id' => 'id', + + // Site data + 'global_key' => 'str', + 'type' => 'str', + 'group' => 'str', + 'source' => 'str', + 'language' => 'str', + 'protocol' => 'str', + 'domain' => 'str', + 'data' => 'array', + + // Site config + 'forward' => 'bool', + 'config' => 'array', + ), + array( + 'type' => Site::TYPE_UNKNOWN, + 'group' => Site::GROUP_NONE, + 'source' => Site::SOURCE_LOCAL, + 'data' => array(), + + 'forward' => false, + 'config' => array(), + 'language' => '', + ), + 'ORMRow', + 'site_' + ); + } + +} diff --git a/includes/site/FileBasedSiteLookup.php b/includes/site/FileBasedSiteLookup.php new file mode 100644 index 00000000..96544403 --- /dev/null +++ b/includes/site/FileBasedSiteLookup.php @@ -0,0 +1,139 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * + * @license GNU GPL v2+ + */ + +/** + * Provides a file-based cache of a SiteStore. The sites are stored in + * a json file. (see docs/sitescache.txt regarding format) + * + * The cache can be built with the rebuildSitesCache.php maintenance script, + * and a MediaWiki instance can be setup to use this by setting the + * 'wgSitesCacheFile' configuration to the cache file location. + * + * @since 1.25 + */ +class FileBasedSiteLookup implements SiteLookup { + + /** + * @var SiteList + */ + private $sites = null; + + /** + * @var string + */ + private $cacheFile; + + /** + * @param string $cacheFile + */ + public function __construct( $cacheFile ) { + $this->cacheFile = $cacheFile; + } + + /** + * @since 1.25 + * + * @return SiteList + */ + public function getSites() { + if ( $this->sites === null ) { + $this->sites = $this->loadSitesFromCache(); + } + + return $this->sites; + } + + /** + * @param string $globalId + * + * @since 1.25 + * + * @return Site|null + */ + public function getSite( $globalId ) { + $sites = $this->getSites(); + + return $sites->hasSite( $globalId ) ? $sites->getSite( $globalId ) : null; + } + + /** + * @return SiteList + */ + private function loadSitesFromCache() { + $data = $this->loadJsonFile(); + + $sites = new SiteList(); + + // @todo lazy initialize the site objects in the site list (e.g. only when needed to access) + foreach ( $data['sites'] as $siteArray ) { + $sites[] = $this->newSiteFromArray( $siteArray ); + } + + return $sites; + } + + /** + * @throws MWException + * @return array see docs/sitescache.txt for format of the array. + */ + private function loadJsonFile() { + if ( !is_readable( $this->cacheFile ) ) { + throw new MWException( 'SiteList cache file not found.' ); + } + + $contents = file_get_contents( $this->cacheFile ); + $data = json_decode( $contents, true ); + + if ( !is_array( $data ) || !is_array( $data['sites'] ) + || !array_key_exists( 'sites', $data ) + ) { + throw new MWException( 'SiteStore json cache data is invalid.' ); + } + + return $data; + } + + /** + * @param array $data + * + * @return Site + */ + private function newSiteFromArray( array $data ) { + $siteType = array_key_exists( 'type', $data ) ? $data['type'] : Site::TYPE_UNKNOWN; + $site = Site::newForType( $siteType ); + + $site->setGlobalId( $data['globalid'] ); + $site->setForward( $data['forward'] ); + $site->setGroup( $data['group'] ); + $site->setLanguageCode( $data['language'] ); + $site->setSource( $data['source'] ); + $site->setExtraData( $data['data'] ); + $site->setExtraConfig( $data['config'] ); + + foreach ( $data['identifiers'] as $identifier ) { + $site->addLocalId( $identifier['type'], $identifier['key'] ); + } + + return $site; + } + +} diff --git a/includes/site/HashSiteStore.php b/includes/site/HashSiteStore.php new file mode 100644 index 00000000..2c254721 --- /dev/null +++ b/includes/site/HashSiteStore.php @@ -0,0 +1,123 @@ +<?php +/** + * In-memory implementation of SiteStore. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * In-memory SiteStore implementation, storing sites in an associative array. + * + * @author Daniel Kinzler + * @author Katie Filbert < aude.wiki@gmail.com > + * + * @since 1.25 + * @ingroup Site + */ +class HashSiteStore implements SiteStore { + + /** + * @var Site[] + */ + private $sites = array(); + + /** + * @param array $sites + */ + public function __construct( $sites = array() ) { + $this->saveSites( $sites ); + } + + /** + * Saves the provided site. + * + * @since 1.25 + * + * @param Site $site + * + * @return boolean Success indicator + */ + public function saveSite( Site $site ) { + $this->sites[$site->getGlobalId()] = $site; + + return true; + } + + /** + * Saves the provided sites. + * + * @since 1.25 + * + * @param Site[] $sites + * + * @return boolean Success indicator + */ + public function saveSites( array $sites ) { + foreach ( $sites as $site ) { + $this->saveSite( $site ); + } + + return true; + } + + /** + * Returns the site with provided global id, or null if there is no such site. + * + * @since 1.25 + * + * @param string $globalId + * @param string $source either 'cache' or 'recache'. + * If 'cache', the values can (but not obliged) come from a cache. + * + * @return Site|null + */ + public function getSite( $globalId, $source = 'cache' ) { + if ( isset( $this->sites[$globalId] ) ) { + return $this->sites[$globalId]; + } else { + return null; + } + } + + /** + * Returns a list of all sites. By default this site is + * fetched from the cache, which can be changed to loading + * the list from the database using the $useCache parameter. + * + * @since 1.25 + * + * @param string $source either 'cache' or 'recache'. + * If 'cache', the values can (but not obliged) come from a cache. + * + * @return SiteList + */ + public function getSites( $source = 'cache' ) { + return new SiteList( $this->sites ); + } + + /** + * Deletes all sites from the database. After calling clear(), getSites() will return an empty + * list and getSite() will return null until saveSite() or saveSites() is called. + */ + public function clear() { + $this->sites = array(); + + return true; + } + +} diff --git a/includes/site/MediaWikiSite.php b/includes/site/MediaWikiSite.php index 9711f042..95631f8e 100644 --- a/includes/site/MediaWikiSite.php +++ b/includes/site/MediaWikiSite.php @@ -116,7 +116,7 @@ class MediaWikiSite extends Site { // Make sure the string is normalized into NFC (due to the bug 40017) // but do nothing to the whitespaces, that should work appropriately. // @see https://bugzilla.wikimedia.org/show_bug.cgi?id=40017 - $pageName = UtfNormal::cleanUp( $pageName ); + $pageName = UtfNormal\Validator::cleanUp( $pageName ); // Build the args for the specific call $args = array( @@ -137,7 +137,7 @@ class MediaWikiSite extends Site { // Go on call the external site // @todo we need a good way to specify a timeout here. - $ret = Http::get( $url ); + $ret = Http::get( $url, array(), __METHOD__ ); } if ( $ret === false ) { diff --git a/includes/site/SiteExporter.php b/includes/site/SiteExporter.php new file mode 100644 index 00000000..62f6ca3c --- /dev/null +++ b/includes/site/SiteExporter.php @@ -0,0 +1,114 @@ +<?php + +/** + * Utility for exporting site entries to XML. + * For the output file format, see docs/sitelist.txt and docs/sitelist-1.0.xsd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @since 1.25 + * + * @file + * @ingroup Site + * + * @license GNU GPL v2+ + * @author Daniel Kinzler + */ +class SiteExporter { + + /** + * @var resource + */ + private $sink; + + /** + * @param resource $sink A file handle open for writing + */ + public function __construct( $sink ) { + if ( !is_resource( $sink ) || get_resource_type( $sink ) !== 'stream' ) { + throw new InvalidArgumentException( '$sink must be a file handle' ); + } + + $this->sink = $sink; + } + + /** + * Writes a <site> tag for each Site object in $sites, and encloses the entire list + * between <sites> tags. + * + * @param Site[]|SiteList $sites + */ + public function exportSites( $sites ) { + $attributes = array( + 'version' => '1.0', + 'xmlns' => 'http://www.mediawiki.org/xml/sitelist-1.0/', + ); + + fwrite( $this->sink, XML::openElement( 'sites', $attributes ) . "\n" ); + + foreach ( $sites as $site ) { + $this->exportSite( $site ); + } + + fwrite( $this->sink, XML::closeElement( 'sites' ) . "\n" ); + fflush( $this->sink ); + } + + /** + * Writes a <site> tag representing the given Site object. + * + * @param Site $site + */ + private function exportSite( Site $site ) { + if ( $site->getType() !== Site::TYPE_UNKNOWN ) { + $siteAttr = array( 'type' => $site->getType() ); + } else { + $siteAttr = null; + } + + fwrite( $this->sink, "\t" . XML::openElement( 'site', $siteAttr ) . "\n" ); + + fwrite( $this->sink, "\t\t" . XML::element( 'globalid', null, $site->getGlobalId() ) . "\n" ); + + if ( $site->getGroup() !== Site::GROUP_NONE ) { + fwrite( $this->sink, "\t\t" . XML::element( 'group', null, $site->getGroup() ) . "\n" ); + } + + if ( $site->getSource() !== Site::SOURCE_LOCAL ) { + fwrite( $this->sink, "\t\t" . XML::element( 'source', null, $site->getSource() ) . "\n" ); + } + + if ( $site->shouldForward() ) { + fwrite( $this->sink, "\t\t" . XML::element( 'forward', null, '' ) . "\n" ); + } + + foreach ( $site->getAllPaths() as $type => $path ) { + fwrite( $this->sink, "\t\t" . XML::element( 'path', array( 'type' => $type ), $path ) . "\n" ); + } + + foreach ( $site->getLocalIds() as $type => $ids ) { + foreach ( $ids as $id ) { + fwrite( $this->sink, "\t\t" . XML::element( 'localid', array( 'type' => $type ), $id ) . "\n" ); + } + } + + //@todo: export <data> + //@todo: export <config> + + fwrite( $this->sink, "\t" . XML::closeElement( 'site' ) . "\n" ); + } + +} diff --git a/includes/site/SiteImporter.php b/includes/site/SiteImporter.php new file mode 100644 index 00000000..a05bad5d --- /dev/null +++ b/includes/site/SiteImporter.php @@ -0,0 +1,263 @@ +<?php + +/** + * Utility for importing site entries from XML. + * For the expected format of the input, see docs/sitelist.txt and docs/sitelist-1.0.xsd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @since 1.25 + * + * @file + * @ingroup Site + * + * @license GNU GPL v2+ + * @author Daniel Kinzler + */ +class SiteImporter { + + /** + * @var SiteStore + */ + private $store; + + /** + * @var callable|null + */ + private $exceptionCallback; + + /** + * @param SiteStore $store + */ + public function __construct( SiteStore $store ) { + $this->store = $store; + } + + /** + * @return callable + */ + public function getExceptionCallback() { + return $this->exceptionCallback; + } + + /** + * @param callable $exceptionCallback + */ + public function setExceptionCallback( $exceptionCallback ) { + $this->exceptionCallback = $exceptionCallback; + } + + /** + * @param string $file + */ + public function importFromFile( $file ) { + $xml = file_get_contents( $file ); + + if ( $xml === false ) { + throw new RuntimeException( 'Failed to read ' . $file . '!' ); + } + + $this->importFromXML( $xml ); + } + + /** + * @param string $xml + * + * @throws InvalidArgumentException + */ + public function importFromXML( $xml ) { + $document = new DOMDocument(); + + $oldLibXmlErrors = libxml_use_internal_errors( true ); + $ok = $document->loadXML( $xml, LIBXML_NONET ); + + if ( !$ok ) { + $errors = libxml_get_errors(); + libxml_use_internal_errors( $oldLibXmlErrors ); + + foreach ( $errors as $error ) { + /** @var LibXMLError $error */ + throw new InvalidArgumentException( + 'Malformed XML: ' . $error->message . ' in line ' . $error->line + ); + } + + throw new InvalidArgumentException( 'Malformed XML!' ); + } + + libxml_use_internal_errors( $oldLibXmlErrors ); + $this->importFromDOM( $document->documentElement ); + } + + /** + * @param DOMElement $root + */ + private function importFromDOM( DOMElement $root ) { + $sites = $this->makeSiteList( $root ); + $this->store->saveSites( $sites ); + } + + /** + * @param DOMElement $root + * + * @return Site[] + */ + private function makeSiteList( DOMElement $root ) { + $sites = array(); + + // Old sites, to get the row IDs that correspond to the global site IDs. + // TODO: Get rid of internal row IDs, they just get in the way. Get rid of ORMRow, too. + $oldSites = $this->store->getSites(); + + $current = $root->firstChild; + while ( $current ) { + if ( $current instanceof DOMElement && $current->tagName === 'site' ) { + try { + $site = $this->makeSite( $current ); + $key = $site->getGlobalId(); + + if ( $oldSites->hasSite( $key ) ) { + $oldSite = $oldSites->getSite( $key ); + $site->setInternalId( $oldSite->getInternalId() ); + } + + $sites[$key] = $site; + } catch ( Exception $ex ) { + $this->handleException( $ex ); + } + } + + $current = $current->nextSibling; + } + + return $sites; + } + + /** + * @param DOMElement $siteElement + * + * @return Site + * @throws InvalidArgumentException + */ + public function makeSite( DOMElement $siteElement ) { + if ( $siteElement->tagName !== 'site' ) { + throw new InvalidArgumentException( 'Expected <site> tag, found ' . $siteElement->tagName ); + } + + $type = $this->getAttributeValue( $siteElement, 'type', Site::TYPE_UNKNOWN ); + $site = Site::newForType( $type ); + + $site->setForward( $this->hasChild( $siteElement, 'forward' ) ); + $site->setGlobalId( $this->getChildText( $siteElement, 'globalid' ) ); + $site->setGroup( $this->getChildText( $siteElement, 'group', Site::GROUP_NONE ) ); + $site->setSource( $this->getChildText( $siteElement, 'source', Site::SOURCE_LOCAL ) ); + + $pathTags = $siteElement->getElementsByTagName( 'path' ); + for ( $i = 0; $i < $pathTags->length; $i++ ) { + $pathElement = $pathTags->item( $i ); + $pathType = $this->getAttributeValue( $pathElement, 'type' ); + $path = $pathElement->textContent; + + $site->setPath( $pathType, $path ); + } + + $idTags = $siteElement->getElementsByTagName( 'localid' ); + for ( $i = 0; $i < $idTags->length; $i++ ) { + $idElement = $idTags->item( $i ); + $idType = $this->getAttributeValue( $idElement, 'type' ); + $id = $idElement->textContent; + + $site->addLocalId( $idType, $id ); + } + + //@todo: import <data> + //@todo: import <config> + + return $site; + } + + /** + * @param DOMElement $element + * @param $name + * @param string|null|bool $default + * + * @return null|string + * @throws MWException If the attribute is not found and no default is provided + */ + private function getAttributeValue( DOMElement $element, $name, $default = false ) { + $node = $element->getAttributeNode( $name ); + + if ( !$node ) { + if ( $default !== false ) { + return $default; + } else { + throw new MWException( + 'Required ' . $name . ' attribute not found in <' . $element->tagName . '> tag' + ); + } + } + + return $node->textContent; + } + + /** + * @param DOMElement $element + * @param string $name + * @param string|null|bool $default + * + * @return null|string + * @throws MWException If the child element is not found and no default is provided + */ + private function getChildText( DOMElement $element, $name, $default = false ) { + $elements = $element->getElementsByTagName( $name ); + + if ( $elements->length < 1 ) { + if ( $default !== false ) { + return $default; + } else { + throw new MWException( + 'Required <' . $name . '> tag not found inside <' . $element->tagName . '> tag' + ); + } + } + + $node = $elements->item( 0 ); + return $node->textContent; + } + + /** + * @param DOMElement $element + * @param string $name + * + * @return bool + * @throws MWException + */ + private function hasChild( DOMElement $element, $name ) { + return $this->getChildText( $element, $name, null ) !== null; + } + + /** + * @param Exception $ex + */ + private function handleException( Exception $ex ) { + if ( $this->exceptionCallback ) { + call_user_func( $this->exceptionCallback, $ex ); + } else { + wfLogWarning( $ex->getMessage() ); + } + } + +} diff --git a/includes/site/SiteLookup.php b/includes/site/SiteLookup.php new file mode 100644 index 00000000..610bf0b7 --- /dev/null +++ b/includes/site/SiteLookup.php @@ -0,0 +1,50 @@ +<?php + +/** + * Interface for service objects providing a lookup of Site objects. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @since 1.25 + * + * @file + * @ingroup Site + * + * @license GNU GPL v2+ + */ +interface SiteLookup { + + /** + * Returns the site with provided global id, or null if there is no such site. + * + * @since 1.25 + * + * @param string $globalId + * + * @return Site|null + */ + public function getSite( $globalId ); + + /** + * Returns a list of all sites. + * + * @since 1.25 + * + * @return SiteList + */ + public function getSites(); + +} diff --git a/includes/site/SiteSQLStore.php b/includes/site/SiteSQLStore.php index d1334680..d77f07be 100644 --- a/includes/site/SiteSQLStore.php +++ b/includes/site/SiteSQLStore.php @@ -28,468 +28,25 @@ * @license GNU GPL v2+ * @author Jeroen De Dauw < jeroendedauw@gmail.com > */ -class SiteSQLStore implements SiteStore { - /** - * @since 1.21 - * - * @var SiteList|null - */ - protected $sites = null; - - /** - * @var ORMTable - */ - protected $sitesTable; - - /** - * @var string|null - */ - private $cacheKey = null; - - /** - * @var int - */ - private $cacheTimeout = 3600; +class SiteSQLStore extends CachingSiteStore { /** * @since 1.21 + * @deprecated 1.25 Construct a SiteStore instance directly instead. * * @param ORMTable|null $sitesTable + * @param BagOStuff|null $cache * * @return SiteStore */ - public static function newInstance( ORMTable $sitesTable = null ) { - return new static( $sitesTable ); - } - - /** - * Constructor. - * - * @since 1.21 - * - * @param ORMTable|null $sitesTable - */ - protected function __construct( ORMTable $sitesTable = null ) { - if ( $sitesTable === null ) { - $sitesTable = $this->newSitesTable(); - } - - $this->sitesTable = $sitesTable; - } - - /** - * Constructs a cache key to use for caching the list of sites. - * - * This includes the concrete class name of the site list as well as a version identifier - * for the list's serialization, to avoid problems when unserializing site lists serialized - * by an older version, e.g. when reading from a cache. - * - * The cache key also includes information about where the sites were loaded from, e.g. - * the name of a database table. - * - * @see SiteList::getSerialVersionId - * - * @return string The cache key. - */ - protected function getCacheKey() { - wfProfileIn( __METHOD__ ); - - if ( $this->cacheKey === null ) { - $type = 'SiteList#' . SiteList::getSerialVersionId(); - $source = $this->sitesTable->getName(); - - if ( $this->sitesTable->getTargetWiki() !== false ) { - $source = $this->sitesTable->getTargetWiki() . '.' . $source; - } - - $this->cacheKey = wfMemcKey( "$source/$type" ); - } - - wfProfileOut( __METHOD__ ); - return $this->cacheKey; - } - - /** - * @see SiteStore::getSites - * - * @since 1.21 - * - * @param string $source Either 'cache' or 'recache' - * - * @return SiteList - */ - public function getSites( $source = 'cache' ) { - wfProfileIn( __METHOD__ ); - - if ( $source === 'cache' ) { - if ( $this->sites === null ) { - $cache = wfGetMainCache(); - $sites = $cache->get( $this->getCacheKey() ); - - if ( is_object( $sites ) ) { - $this->sites = $sites; - } else { - $this->loadSites(); - } - } - } - else { - $this->loadSites(); - } - - wfProfileOut( __METHOD__ ); - return $this->sites; - } - - /** - * Returns a new Site object constructed from the provided ORMRow. - * - * @since 1.21 - * - * @param ORMRow $siteRow - * - * @return Site - */ - protected function siteFromRow( ORMRow $siteRow ) { - wfProfileIn( __METHOD__ ); - - $site = Site::newForType( $siteRow->getField( 'type', Site::TYPE_UNKNOWN ) ); - - $site->setGlobalId( $siteRow->getField( 'global_key' ) ); - - $site->setInternalId( $siteRow->getField( 'id' ) ); - - if ( $siteRow->hasField( 'forward' ) ) { - $site->setForward( $siteRow->getField( 'forward' ) ); - } - - if ( $siteRow->hasField( 'group' ) ) { - $site->setGroup( $siteRow->getField( 'group' ) ); - } - - if ( $siteRow->hasField( 'language' ) ) { - $site->setLanguageCode( $siteRow->getField( 'language' ) === '' - ? null - : $siteRow->getField( 'language' ) - ); - } - - if ( $siteRow->hasField( 'source' ) ) { - $site->setSource( $siteRow->getField( 'source' ) ); - } - - if ( $siteRow->hasField( 'data' ) ) { - $site->setExtraData( $siteRow->getField( 'data' ) ); - } - - if ( $siteRow->hasField( 'config' ) ) { - $site->setExtraConfig( $siteRow->getField( 'config' ) ); - } - - wfProfileOut( __METHOD__ ); - return $site; - } - - /** - * Get a new ORMRow from a Site object - * - * @since 1.22 - * - * @param Site $site - * - * @return ORMRow - */ - protected function getRowFromSite( Site $site ) { - $fields = array( - // Site data - 'global_key' => $site->getGlobalId(), // TODO: check not null - 'type' => $site->getType(), - 'group' => $site->getGroup(), - 'source' => $site->getSource(), - 'language' => $site->getLanguageCode() === null ? '' : $site->getLanguageCode(), - 'protocol' => $site->getProtocol(), - 'domain' => strrev( $site->getDomain() ) . '.', - 'data' => $site->getExtraData(), - - // Site config - 'forward' => $site->shouldForward(), - 'config' => $site->getExtraConfig(), - ); - - if ( $site->getInternalId() !== null ) { - $fields['id'] = $site->getInternalId(); - } - - return new ORMRow( $this->sitesTable, $fields ); - } - - /** - * Fetches the site from the database and loads them into the sites field. - * - * @since 1.21 - */ - protected function loadSites() { - wfProfileIn( __METHOD__ ); - - $this->sites = new SiteList(); - - foreach ( $this->sitesTable->select() as $siteRow ) { - $this->sites[] = $this->siteFromRow( $siteRow ); - } - - // Batch load the local site identifiers. - $ids = wfGetDB( $this->sitesTable->getReadDb() )->select( - 'site_identifiers', - array( - 'si_site', - 'si_type', - 'si_key', - ), - array(), - __METHOD__ - ); - - foreach ( $ids as $id ) { - if ( $this->sites->hasInternalId( $id->si_site ) ) { - $site = $this->sites->getSiteByInternalId( $id->si_site ); - $site->addLocalId( $id->si_type, $id->si_key ); - $this->sites->setSite( $site ); - } - } - - $cache = wfGetMainCache(); - $cache->set( $this->getCacheKey(), $this->sites, $this->cacheTimeout ); - - wfProfileOut( __METHOD__ ); - } - - /** - * @see SiteStore::getSite - * - * @since 1.21 - * - * @param string $globalId - * @param string $source - * - * @return Site|null - */ - public function getSite( $globalId, $source = 'cache' ) { - wfProfileIn( __METHOD__ ); - - $sites = $this->getSites( $source ); - - wfProfileOut( __METHOD__ ); - return $sites->hasSite( $globalId ) ? $sites->getSite( $globalId ) : null; - } - - /** - * @see SiteStore::saveSite - * - * @since 1.21 - * - * @param Site $site - * - * @return bool Success indicator - */ - public function saveSite( Site $site ) { - return $this->saveSites( array( $site ) ); - } - - /** - * @see SiteStore::saveSites - * - * @since 1.21 - * - * @param Site[] $sites - * - * @return bool Success indicator - */ - public function saveSites( array $sites ) { - wfProfileIn( __METHOD__ ); - - if ( empty( $sites ) ) { - wfProfileOut( __METHOD__ ); - return true; - } - - $dbw = $this->sitesTable->getWriteDbConnection(); - - $dbw->startAtomic( __METHOD__ ); - - $success = true; - - $internalIds = array(); - $localIds = array(); - - foreach ( $sites as $site ) { - if ( $site->getInternalId() !== null ) { - $internalIds[] = $site->getInternalId(); - } - - $siteRow = $this->getRowFromSite( $site ); - $success = $siteRow->save( __METHOD__ ) && $success; - - foreach ( $site->getLocalIds() as $idType => $ids ) { - foreach ( $ids as $id ) { - $localIds[] = array( $siteRow->getId(), $idType, $id ); - } - } - } - - if ( $internalIds !== array() ) { - $dbw->delete( - 'site_identifiers', - array( 'si_site' => $internalIds ), - __METHOD__ - ); - } - - foreach ( $localIds as $localId ) { - $dbw->insert( - 'site_identifiers', - array( - 'si_site' => $localId[0], - 'si_type' => $localId[1], - 'si_key' => $localId[2], - ), - __METHOD__ - ); + public static function newInstance( ORMTable $sitesTable = null, BagOStuff $cache = null ) { + if ( $cache === null ) { + $cache = wfGetMainCache(); } - $dbw->endAtomic( __METHOD__ ); + $siteStore = new DBSiteStore(); - // purge cache - $this->reset(); - - wfProfileOut( __METHOD__ ); - return $success; + return new static( $siteStore, $cache ); } - /** - * Purges the internal and external cache of the site list, forcing the list - * of sites to be re-read from the database. - * - * @since 1.21 - */ - public function reset() { - wfProfileIn( __METHOD__ ); - // purge cache - $cache = wfGetMainCache(); - $cache->delete( $this->getCacheKey() ); - $this->sites = null; - - wfProfileOut( __METHOD__ ); - } - - /** - * Clears the list of sites stored in the database. - * - * @see SiteStore::clear() - * - * @return bool Success - */ - public function clear() { - wfProfileIn( __METHOD__ ); - $dbw = $this->sitesTable->getWriteDbConnection(); - - $dbw->startAtomic( __METHOD__ ); - $ok = $dbw->delete( 'sites', '*', __METHOD__ ); - $ok = $dbw->delete( 'site_identifiers', '*', __METHOD__ ) && $ok; - $dbw->endAtomic( __METHOD__); - - $this->reset(); - - wfProfileOut( __METHOD__ ); - return $ok; - } - - /** - * @since 1.21 - * - * @return ORMTable - */ - protected function newSitesTable() { - return new ORMTable( - 'sites', - array( - 'id' => 'id', - - // Site data - 'global_key' => 'str', - 'type' => 'str', - 'group' => 'str', - 'source' => 'str', - 'language' => 'str', - 'protocol' => 'str', - 'domain' => 'str', - 'data' => 'array', - - // Site config - 'forward' => 'bool', - 'config' => 'array', - ), - array( - 'type' => Site::TYPE_UNKNOWN, - 'group' => Site::GROUP_NONE, - 'source' => Site::SOURCE_LOCAL, - 'data' => array(), - - 'forward' => false, - 'config' => array(), - 'language' => '', - ), - 'ORMRow', - 'site_' - ); - } - -} - -/** - * @deprecated since 1.21 - */ -class Sites extends SiteSQLStore { - - /** - * Factory for creating new site objects. - * - * @since 1.21 - * @deprecated since 1.21 - * - * @param string|bool $globalId - * - * @return Site - */ - public static function newSite( $globalId = false ) { - $site = new Site(); - - if ( $globalId !== false ) { - $site->setGlobalId( $globalId ); - } - - return $site; - } - - /** - * @deprecated since 1.21 - * @return SiteStore - */ - public static function singleton() { - static $singleton; - - if ( $singleton === null ) { - $singleton = new static(); - } - - return $singleton; - } - - /** - * @deprecated since 1.21 - * @param string $group - * @return SiteList - */ - public function getSiteGroup( $group ) { - return $this->getSites()->getGroup( $group ); - } } diff --git a/includes/site/SiteStore.php b/includes/site/SiteStore.php index 537f1ccb..10e0c1b9 100644 --- a/includes/site/SiteStore.php +++ b/includes/site/SiteStore.php @@ -26,7 +26,7 @@ * @license GNU GPL v2+ * @author Jeroen De Dauw < jeroendedauw@gmail.com > */ -interface SiteStore { +interface SiteStore extends SiteLookup { /** * Saves the provided site. @@ -51,33 +51,6 @@ interface SiteStore { public function saveSites( array $sites ); /** - * Returns the site with provided global id, or null if there is no such site. - * - * @since 1.21 - * - * @param string $globalId - * @param string $source Either 'cache' or 'recache'. - * If 'cache', the values are allowed (but not obliged) to come from a cache. - * - * @return Site|null - */ - public function getSite( $globalId, $source = 'cache' ); - - /** - * Returns a list of all sites. By default this site is - * fetched from the cache, which can be changed to loading - * the list from the database using the $useCache parameter. - * - * @since 1.21 - * - * @param string $source Either 'cache' or 'recache'. - * If 'cache', the values are allowed (but not obliged) to come from a cache. - * - * @return SiteList - */ - public function getSites( $source = 'cache' ); - - /** * Deletes all sites from the database. After calling clear(), getSites() will return an empty * list and getSite() will return null until saveSite() or saveSites() is called. */ diff --git a/includes/site/SitesCacheFileBuilder.php b/includes/site/SitesCacheFileBuilder.php new file mode 100644 index 00000000..2e420409 --- /dev/null +++ b/includes/site/SitesCacheFileBuilder.php @@ -0,0 +1,113 @@ +<?php + +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @since 1.25 + * + * @file + * + * @license GNU GPL v2+ + */ +class SitesCacheFileBuilder { + + /** + * @var SiteLookup + */ + private $siteLookup; + + /** + * @var string + */ + private $cacheFile; + + /** + * @param SiteLookup $siteLookup + * @param string $cacheFile + */ + public function __construct( SiteLookup $siteLookup, $cacheFile ) { + $this->siteLookup = $siteLookup; + $this->cacheFile = $cacheFile; + } + + public function build() { + $this->sites = $this->siteLookup->getSites(); + $this->cacheSites( $this->sites->getArrayCopy() ); + } + + /** + * @param Site[] $sites + * + * @throws MWException if in manualRecache mode + * @return bool + */ + private function cacheSites( array $sites ) { + $sitesArray = array(); + + foreach ( $sites as $site ) { + $globalId = $site->getGlobalId(); + $sitesArray[$globalId] = $this->getSiteAsArray( $site ); + } + + $json = json_encode( array( + 'sites' => $sitesArray + ) ); + + $result = file_put_contents( $this->cacheFile, $json ); + + return $result !== false; + } + + /** + * @param Site $site + * + * @return array + */ + private function getSiteAsArray( Site $site ) { + $siteEntry = unserialize( $site->serialize() ); + $siteIdentifiers = $this->buildLocalIdentifiers( $site ); + $identifiersArray = array(); + + foreach ( $siteIdentifiers as $identifier ) { + $identifiersArray[] = $identifier; + } + + $siteEntry['identifiers'] = $identifiersArray; + + return $siteEntry; + } + + /** + * @param Site $site + * + * @return array Site local identifiers + */ + private function buildLocalIdentifiers( Site $site ) { + $localIds = array(); + + foreach ( $site->getLocalIds() as $idType => $ids ) { + foreach ( $ids as $id ) { + $localIds[] = array( + 'type' => $idType, + 'key' => $id + ); + } + } + + return $localIds; + } + +} |