diff options
| -rw-r--r-- | README | 31 | ||||
| -rw-r--r-- | actions/noticesearch.php | 2 | ||||
| -rw-r--r-- | actions/noticesearchrss.php | 2 | ||||
| -rw-r--r-- | actions/peoplesearch.php | 2 | ||||
| -rw-r--r-- | actions/twitapisearchatom.php | 2 | ||||
| -rw-r--r-- | actions/twitapisearchjson.php | 2 | ||||
| -rw-r--r-- | classes/Memcached_DataObject.php | 29 | ||||
| -rw-r--r-- | classes/Status_network.php | 28 | ||||
| -rw-r--r-- | lib/default.php | 4 | ||||
| -rw-r--r-- | lib/search_engines.php | 71 | ||||
| -rw-r--r-- | plugins/SphinxSearch/README | 45 | ||||
| -rw-r--r-- | plugins/SphinxSearch/SphinxSearchPlugin.php | 100 | ||||
| -rwxr-xr-x | plugins/SphinxSearch/scripts/gen_config.php | 126 | ||||
| -rwxr-xr-x | plugins/SphinxSearch/scripts/index_update.php | 61 | ||||
| -rw-r--r-- | plugins/SphinxSearch/scripts/sphinx-utils.php | 63 | ||||
| -rwxr-xr-x | plugins/SphinxSearch/scripts/sphinx.sh (renamed from scripts/sphinx.sh) | 0 | ||||
| -rw-r--r-- | plugins/SphinxSearch/sphinx.conf.sample (renamed from sphinx.conf.sample) | 0 | ||||
| -rw-r--r-- | plugins/SphinxSearch/sphinxsearch.php | 96 | ||||
| -rwxr-xr-x | scripts/sphinx-cron.sh | 24 | ||||
| -rwxr-xr-x | scripts/sphinx-indexer.sh | 24 |
20 files changed, 539 insertions, 173 deletions
@@ -389,20 +389,16 @@ the server first. Sphinx ------ -To use a Sphinx server to search users and notices, you also need -to install, compile and enable the sphinx pecl extension for php on the -client side, which itself depends on the sphinx development files. -"pecl install sphinx" should take care of that. Add "extension=sphinx.so" -to your php.ini and reload apache to enable it. +To use a Sphinx server to search users and notices, you'll need to +enable the SphinxSearch plugin. Add to your config.php: -You can update your MySQL or Postgresql databases to drop their fulltext -search indexes, since they're now provided by sphinx. + addPlugin('SphinxSearch'); + $config['sphinx']['server'] = 'searchhost.local'; -On the sphinx server side, a script reads the main database and build -the keyword index. A cron job reads the database and keeps the sphinx -indexes up to date. scripts/sphinx-cron.sh should be called by cron -every 5 minutes, for example. scripts/sphinx.sh is an init.d script -to start and stop the sphinx search daemon. +You also need to install, compile and enable the sphinx pecl extension for +php on the client side, which itself depends on the sphinx development files. + +See plugins/SphinxSearch/README for more details and server setup. SMS --- @@ -1168,17 +1164,6 @@ base: memcached uses key-value pairs to store data. We build long, StatusNet site using your memcached server. port: Port to connect to; defaults to 11211. -sphinx ------- - -You can get a significant boost in performance using Sphinx Search -instead of your database server to search for users and notices. -<http://sphinxsearch.com/>. - -enabled: Set to true to enable. Default false. -server: a string with the hostname of the sphinx server. -port: an integer with the port number of the sphinx server. - emailpost --------- diff --git a/actions/noticesearch.php b/actions/noticesearch.php index 79cf572cc..1e5a69180 100644 --- a/actions/noticesearch.php +++ b/actions/noticesearch.php @@ -104,7 +104,7 @@ class NoticesearchAction extends SearchAction { $notice = new Notice(); - $search_engine = $notice->getSearchEngine('identica_notices'); + $search_engine = $notice->getSearchEngine('notice'); $search_engine->set_sort_mode('chron'); // Ask for an extra to see if there's more. $search_engine->limit((($page-1)*NOTICES_PER_PAGE), NOTICES_PER_PAGE + 1); diff --git a/actions/noticesearchrss.php b/actions/noticesearchrss.php index f59ad7962..18f07f855 100644 --- a/actions/noticesearchrss.php +++ b/actions/noticesearchrss.php @@ -62,7 +62,7 @@ class NoticesearchrssAction extends Rss10Action $notice = new Notice(); - $search_engine = $notice->getSearchEngine('identica_notices'); + $search_engine = $notice->getSearchEngine('notice'); $search_engine->set_sort_mode('chron'); if (!$limit) $limit = 20; diff --git a/actions/peoplesearch.php b/actions/peoplesearch.php index 38135ecbd..69de44859 100644 --- a/actions/peoplesearch.php +++ b/actions/peoplesearch.php @@ -61,7 +61,7 @@ class PeoplesearchAction extends SearchAction function showResults($q, $page) { $profile = new Profile(); - $search_engine = $profile->getSearchEngine('identica_people'); + $search_engine = $profile->getSearchEngine('profile'); $search_engine->set_sort_mode('chron'); // Ask for an extra to see if there's more. $search_engine->limit((($page-1)*PROFILES_PER_PAGE), PROFILES_PER_PAGE + 1); diff --git a/actions/twitapisearchatom.php b/actions/twitapisearchatom.php index 7d618c471..526ca2ae8 100644 --- a/actions/twitapisearchatom.php +++ b/actions/twitapisearchatom.php @@ -161,7 +161,7 @@ class TwitapisearchatomAction extends ApiAction // lcase it for comparison $q = strtolower($this->query); - $search_engine = $notice->getSearchEngine('identica_notices'); + $search_engine = $notice->getSearchEngine('notice'); $search_engine->set_sort_mode('chron'); $search_engine->limit(($this->page - 1) * $this->rpp, $this->rpp + 1, true); diff --git a/actions/twitapisearchjson.php b/actions/twitapisearchjson.php index c7fa741a0..741ed78d6 100644 --- a/actions/twitapisearchjson.php +++ b/actions/twitapisearchjson.php @@ -121,7 +121,7 @@ class TwitapisearchjsonAction extends ApiAction // lcase it for comparison $q = strtolower($this->query); - $search_engine = $notice->getSearchEngine('identica_notices'); + $search_engine = $notice->getSearchEngine('notice'); $search_engine->set_sort_mode('chron'); $search_engine->limit(($this->page - 1) * $this->rpp, $this->rpp + 1, true); if (false === $search_engine->query($q)) { diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 9c2ac3e01..753fe954e 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -184,27 +184,20 @@ class Memcached_DataObject extends DB_DataObject require_once INSTALLDIR.'/lib/search_engines.php'; static $search_engine; if (!isset($search_engine)) { - $connected = false; - if (common_config('sphinx', 'enabled')) { - $search_engine = new SphinxSearch($this, $table); - $connected = $search_engine->is_connected(); - } - - // unable to connect to sphinx' search daemon - if (!$connected) { - if ('mysql' === common_config('db', 'type')) { - $type = common_config('search', 'type'); - if ($type == 'like') { - $search_engine = new MySQLLikeSearch($this, $table); - } else if ($type == 'fulltext') { - $search_engine = new MySQLSearch($this, $table); - } else { - throw new ServerException('Unknown search type: ' . $type); - } + if (Event::handle('GetSearchEngine', array($this, $table, &$search_engine))) { + if ('mysql' === common_config('db', 'type')) { + $type = common_config('search', 'type'); + if ($type == 'like') { + $search_engine = new MySQLLikeSearch($this, $table); + } else if ($type == 'fulltext') { + $search_engine = new MySQLSearch($this, $table); } else { - $search_engine = new PGSearch($this, $table); + throw new ServerException('Unknown search type: ' . $type); } + } else { + $search_engine = new PGSearch($this, $table); } + } } return $search_engine; } diff --git a/classes/Status_network.php b/classes/Status_network.php index fe4f0b0c5..b3117640d 100644 --- a/classes/Status_network.php +++ b/classes/Status_network.php @@ -57,14 +57,16 @@ class Status_network extends DB_DataObject $config['db']['ini_'.$dbname] = INSTALLDIR.'/classes/status_network.ini'; $config['db']['table_status_network'] = $dbname; - self::$cache = new Memcache(); + if (class_exists('Memcache')) { + self::$cache = new Memcache(); - if (is_array($servers)) { - foreach($servers as $server) { - self::$cache->addServer($server); + if (is_array($servers)) { + foreach($servers as $server) { + self::$cache->addServer($server); + } + } else { + self::$cache->addServer($servers); } - } else { - self::$cache->addServer($servers); } self::$base = $dbname; @@ -76,6 +78,10 @@ class Status_network extends DB_DataObject static function memGet($k, $v) { + if (!self::$cache) { + return self::staticGet($k, $v); + } + $ck = self::cacheKey($k, $v); $sn = self::$cache->get($ck); @@ -92,10 +98,12 @@ class Status_network extends DB_DataObject function decache() { - $keys = array('nickname', 'hostname', 'pathname'); - foreach ($keys as $k) { - $ck = self::cacheKey($k, $this->$k); - self::$cache->delete($ck); + if (self::$cache) { + $keys = array('nickname', 'hostname', 'pathname'); + foreach ($keys as $k) { + $ck = self::cacheKey($k, $this->$k); + self::$cache->delete($ck); + } } } diff --git a/lib/default.php b/lib/default.php index f6cc4b725..95366e0b3 100644 --- a/lib/default.php +++ b/lib/default.php @@ -125,10 +125,6 @@ $default = 'public' => array()), # JIDs of users who want to receive the public stream 'invite' => array('enabled' => true), - 'sphinx' => - array('enabled' => false, - 'server' => 'localhost', - 'port' => 3312), 'tag' => array('dropoff' => 864000.0), 'popular' => diff --git a/lib/search_engines.php b/lib/search_engines.php index 69f6ff468..332db3f89 100644 --- a/lib/search_engines.php +++ b/lib/search_engines.php @@ -46,70 +46,11 @@ class SearchEngine } } -class SphinxSearch extends SearchEngine -{ - private $sphinx; - private $connected; - - function __construct($target, $table) - { - $fp = @fsockopen(common_config('sphinx', 'server'), common_config('sphinx', 'port')); - if (!$fp) { - $this->connected = false; - return; - } - fclose($fp); - parent::__construct($target, $table); - $this->sphinx = new SphinxClient; - $this->sphinx->setServer(common_config('sphinx', 'server'), common_config('sphinx', 'port')); - $this->connected = true; - } - - function is_connected() - { - return $this->connected; - } - - function limit($offset, $count, $rss = false) - { - //FIXME without LARGEST_POSSIBLE, the most recent results aren't returned - // this probably has a large impact on performance - $LARGEST_POSSIBLE = 1e6; - - if ($rss) { - $this->sphinx->setLimits($offset, $count, $count, $LARGEST_POSSIBLE); - } - else { - // return at most 50 pages of results - $this->sphinx->setLimits($offset, $count, 50 * ($count - 1), $LARGEST_POSSIBLE); - } - - return $this->target->limit(0, $count); - } - - function query($q) - { - $result = $this->sphinx->query($q, $this->table); - if (!isset($result['matches'])) return false; - $id_set = join(', ', array_keys($result['matches'])); - $this->target->whereAdd("id in ($id_set)"); - return true; - } - - function set_sort_mode($mode) - { - if ('chron' === $mode) { - $this->sphinx->SetSortMode(SPH_SORT_ATTR_DESC, 'created_ts'); - return $this->target->orderBy('created desc'); - } - } -} - class MySQLSearch extends SearchEngine { function query($q) { - if ('identica_people' === $this->table) { + if ('profile' === $this->table) { $this->target->whereAdd('MATCH(nickname, fullname, location, bio, homepage) ' . 'AGAINST (\''.addslashes($q).'\' IN BOOLEAN MODE)'); if (strtolower($q) != $q) { @@ -117,7 +58,7 @@ class MySQLSearch extends SearchEngine 'AGAINST (\''.addslashes(strtolower($q)).'\' IN BOOLEAN MODE)', 'OR'); } return true; - } else if ('identica_notices' === $this->table) { + } else if ('notice' === $this->table) { // Don't show imported notices $this->target->whereAdd('notice.is_local != ' . Notice::GATEWAY); @@ -143,13 +84,13 @@ class MySQLLikeSearch extends SearchEngine { function query($q) { - if ('identica_people' === $this->table) { + if ('profile' === $this->table) { $qry = sprintf('(nickname LIKE "%%%1$s%%" OR '. ' fullname LIKE "%%%1$s%%" OR '. ' location LIKE "%%%1$s%%" OR '. ' bio LIKE "%%%1$s%%" OR '. ' homepage LIKE "%%%1$s%%")', addslashes($q)); - } else if ('identica_notices' === $this->table) { + } else if ('notice' === $this->table) { $qry = sprintf('content LIKE "%%%1$s%%"', addslashes($q)); } else { throw new ServerException('Unknown table: ' . $this->table); @@ -165,9 +106,9 @@ class PGSearch extends SearchEngine { function query($q) { - if ('identica_people' === $this->table) { + if ('profile' === $this->table) { return $this->target->whereAdd('textsearch @@ plainto_tsquery(\''.addslashes($q).'\')'); - } else if ('identica_notices' === $this->table) { + } else if ('notice' === $this->table) { // XXX: We need to filter out gateway notices (notice.is_local = -2) --Zach diff --git a/plugins/SphinxSearch/README b/plugins/SphinxSearch/README new file mode 100644 index 000000000..5a2c063bd --- /dev/null +++ b/plugins/SphinxSearch/README @@ -0,0 +1,45 @@ +You can get a significant boost in performance using Sphinx Search +instead of your database server to search for users and notices. +<http://sphinxsearch.com/>. + +Configuration +------------- + +In StatusNet's configuration, you can adjust the following settings +under 'sphinx': + +enabled: Set to true to enable. Default false. +server: a string with the hostname of the sphinx server. +port: an integer with the port number of the sphinx server. + + +Requirements +------------ + +To use a Sphinx server to search users and notices, you also need +to install, compile and enable the sphinx pecl extension for php on the +client side, which itself depends on the sphinx development files. +"pecl install sphinx" should take care of that. Add "extension=sphinx.so" +to your php.ini and reload apache to enable it. + +You can update your MySQL or Postgresql databases to drop their fulltext +search indexes, since they're now provided by sphinx. + + +You will also need a Sphinx server to serve the search queries. + +On the sphinx server side, a script reads the main database and build +the keyword index. A cron job reads the database and keeps the sphinx +indexes up to date. scripts/sphinx-cron.sh should be called by cron +every 5 minutes, for example. scripts/sphinx.sh is an init.d script +to start and stop the sphinx search daemon. + + +Server configuration +-------------------- +scripts/gen_config.php can generate a sphinx.conf file listing MySQL +data sources for your databases. You may need to tweak paths afterwards. + + $ plugins/SphinxSearch/scripts/gen_config.php > sphinx.conf + +If you wish, you can build a full config yourself based on sphinx.conf.sample diff --git a/plugins/SphinxSearch/SphinxSearchPlugin.php b/plugins/SphinxSearch/SphinxSearchPlugin.php new file mode 100644 index 000000000..7a27a4c04 --- /dev/null +++ b/plugins/SphinxSearch/SphinxSearchPlugin.php @@ -0,0 +1,100 @@ +<?php +/** + * StatusNet, the distributed open-source microblogging tool + * + * PHP version 5 + * + * LICENCE: This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @category Plugin + * @package StatusNet + * @author Brion Vibber <brion@status.net> + * @copyright 2009 Control Yourself, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +// Set defaults if not already set in the config array... +global $config; +$sphinxDefaults = + array('enabled' => true, + 'server' => 'localhost', + 'port' => 3312); +foreach($sphinxDefaults as $key => $val) { + if (!isset($config['sphinx'][$key])) { + $config['sphinx'][$key] = $val; + } +} + + + +/** + * Plugin for Sphinx search backend. + * + * @category Plugin + * @package StatusNet + * @author Brion Vibber <brion@status.net> + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + * @link http://twitter.com/ + */ + +class SphinxSearchPlugin extends Plugin +{ + /** + * Automatically load any classes used + * + * @param string $cls the class + * @return boolean hook return + */ + function onAutoload($cls) + { + switch ($cls) { + case 'SphinxSearch': + include_once INSTALLDIR . '/plugins/SphinxSearch/' . + strtolower($cls) . '.php'; + return false; + default: + return true; + } + } + + /** + * Create sphinx search engine object for the given table type. + * + * @param Memcached_DataObject $target + * @param string $table + * @param out &$search_engine SearchEngine object on output if successful + * @ return boolean hook return + */ + function onGetSearchEngine(Memcached_DataObject $target, $table, &$search_engine) + { + if (common_config('sphinx', 'enabled')) { + if (!class_exists('SphinxClient')) { + throw new ServerException('Sphinx PHP extension must be installed.'); + } + $engine = new SphinxSearch($target, $table); + if ($engine->is_connected()) { + $search_engine = $engine; + return false; + } + } + // Sphinx disabled or disconnected + return true; + } +} diff --git a/plugins/SphinxSearch/scripts/gen_config.php b/plugins/SphinxSearch/scripts/gen_config.php new file mode 100755 index 000000000..d5a00b6b6 --- /dev/null +++ b/plugins/SphinxSearch/scripts/gen_config.php @@ -0,0 +1,126 @@ +#!/usr/bin/env php +<?php +/* + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2009, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..')); + +$longoptions = array('base=', 'network'); + +$helptext = <<<END_OF_TRIM_HELP +Generates sphinx.conf file based on StatusNet configuration. + --base Base dir to Sphinx install + (default /usr/local) + --network Use status_network global config table + (non-functional at present) + + +END_OF_TRIM_HELP; + +require_once INSTALLDIR . '/scripts/commandline.inc'; +require dirname(__FILE__) . '/sphinx-utils.php'; + + +$timestamp = date('r'); +print <<<END +# +# Sphinx configuration for StatusNet +# Generated {$timestamp} +# + +END; + +sphinx_iterate_sites('sphinx_site_template'); + +print <<<END + +indexer +{ + mem_limit = 300M +} + +searchd +{ + port = 3312 + log = {$base}/log/searchd.log + query_log = {$base}/log/query.log + read_timeout = 5 + max_children = 30 + pid_file = {$base}/log/searchd.pid + max_matches = 1000 + seamless_rotate = 1 + preopen_indexes = 0 + unlink_old = 1 +} + +END; + + + +/** + * Build config entries for a single site + * @fixme we only seem to have master DB currently available... + */ +function sphinx_site_template($sn) +{ + return + sphinx_template($sn, + 'profile', + 'SELECT id, UNIX_TIMESTAMP(created) as created_ts, nickname, fullname, location, bio, homepage FROM profile', + 'SELECT * FROM profile where id = $id') . + sphinx_template($sn, + 'notice', + 'SELECT id, UNIX_TIMESTAMP(created) as created_ts, content FROM notice', + 'SELECT * FROM notice where notice.id = $id AND notice.is_local != -2'); +} + +function sphinx_template($sn, $table, $query, $query_info) +{ + $base = sphinx_base(); + $dbtype = common_config('db', 'type'); + + print <<<END + +# +# {$sn->sitename} +# +source {$sn->dbname}_src_{$table} +{ + type = {$dbtype} + sql_host = {$sn->dbhost} + sql_user = {$sn->dbuser} + sql_pass = {$sn->dbpass} + sql_db = {$sn->dbname} + sql_query_pre = SET NAMES utf8; + sql_query = {$query} + sql_query_info = {$query_info} + sql_attr_timestamp = created_ts +} + +index {$sn->dbname}_{$table} +{ + source = {$sn->dbname}_src_{$table} + path = {$base}/data/{$sn->dbname}_{$table} + docinfo = extern + charset_type = utf-8 + min_word_len = 3 +} + + +END; +} diff --git a/plugins/SphinxSearch/scripts/index_update.php b/plugins/SphinxSearch/scripts/index_update.php new file mode 100755 index 000000000..23c60ced7 --- /dev/null +++ b/plugins/SphinxSearch/scripts/index_update.php @@ -0,0 +1,61 @@ +#!/usr/bin/env php +<?php +/* + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2009, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..')); + +$longoptions = array('base=', 'network'); + +$helptext = <<<END_OF_TRIM_HELP +Runs Sphinx search indexer. + --rotate Have Sphinx run index update in background and + rotate updated indexes into place as they finish. + --base Base dir to Sphinx install + (default /usr/local) + --network Use status_network global config table for site list + (non-functional at present) + + +END_OF_TRIM_HELP; + +require_once INSTALLDIR . '/scripts/commandline.inc'; +require dirname(__FILE__) . '/sphinx-utils.php'; + +sphinx_iterate_sites('sphinx_index_update'); + +function sphinx_index_update($sn) +{ + $base = sphinx_base(); + + $baseIndexes = array('notice', 'profile'); + $params = array(); + + if (have_option('rotate')) { + $params[] = '--rotate'; + } + foreach ($baseIndexes as $index) { + $params[] = "{$sn->dbname}_{$index}"; + } + + $params = implode(' ', $params); + $cmd = "$base/bin/indexer --config $base/etc/sphinx.conf $params"; + + print "$cmd\n"; + system($cmd); +} diff --git a/plugins/SphinxSearch/scripts/sphinx-utils.php b/plugins/SphinxSearch/scripts/sphinx-utils.php new file mode 100644 index 000000000..7bbc25270 --- /dev/null +++ b/plugins/SphinxSearch/scripts/sphinx-utils.php @@ -0,0 +1,63 @@ +<?php +/* + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2009, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +function sphinx_use_network() +{ + return have_option('network'); +} + +function sphinx_base() +{ + if (have_option('base')) { + return get_option_value('base'); + } else { + return "/usr/local/sphinx"; + } +} + +function sphinx_iterate_sites($callback) +{ + if (sphinx_use_network()) { + // @fixme this should use, like, some kind of config + Status_network::setupDB('localhost', 'statusnet', 'statuspass', 'statusnet'); + $sn = new Status_network(); + if (!$sn->find()) { + die("Confused... no sites in status_network table or lookup failed.\n"); + } + while ($sn->fetch()) { + $callback($sn); + } + } else { + if (preg_match('!^(mysqli?|pgsql)://(.*?):(.*?)@(.*?)/(.*?)$!', + common_config('db', 'database'), $matches)) { + list(/*all*/, $dbtype, $dbuser, $dbpass, $dbhost, $dbname) = $matches; + $sn = (object)array( + 'sitename' => common_config('site', 'name'), + 'dbhost' => $dbhost, + 'dbuser' => $dbuser, + 'dbpass' => $dbpass, + 'dbname' => $dbname); + $callback($sn); + } else { + print "Unrecognized database configuration string in config.php\n"; + exit(1); + } + } +} + diff --git a/scripts/sphinx.sh b/plugins/SphinxSearch/scripts/sphinx.sh index b8edeb302..b8edeb302 100755 --- a/scripts/sphinx.sh +++ b/plugins/SphinxSearch/scripts/sphinx.sh diff --git a/sphinx.conf.sample b/plugins/SphinxSearch/sphinx.conf.sample index 3de62f637..3de62f637 100644 --- a/sphinx.conf.sample +++ b/plugins/SphinxSearch/sphinx.conf.sample diff --git a/plugins/SphinxSearch/sphinxsearch.php b/plugins/SphinxSearch/sphinxsearch.php new file mode 100644 index 000000000..71f330828 --- /dev/null +++ b/plugins/SphinxSearch/sphinxsearch.php @@ -0,0 +1,96 @@ +<?php +/* + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2008, 2009, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +class SphinxSearch extends SearchEngine +{ + private $sphinx; + private $connected; + + function __construct($target, $table) + { + $fp = @fsockopen(common_config('sphinx', 'server'), common_config('sphinx', 'port')); + if (!$fp) { + $this->connected = false; + return; + } + fclose($fp); + parent::__construct($target, $table); + $this->sphinx = new SphinxClient; + $this->sphinx->setServer(common_config('sphinx', 'server'), common_config('sphinx', 'port')); + $this->connected = true; + } + + function is_connected() + { + return $this->connected; + } + + function limit($offset, $count, $rss = false) + { + //FIXME without LARGEST_POSSIBLE, the most recent results aren't returned + // this probably has a large impact on performance + $LARGEST_POSSIBLE = 1e6; + + if ($rss) { + $this->sphinx->setLimits($offset, $count, $count, $LARGEST_POSSIBLE); + } + else { + // return at most 50 pages of results + $this->sphinx->setLimits($offset, $count, 50 * ($count - 1), $LARGEST_POSSIBLE); + } + + return $this->target->limit(0, $count); + } + + function query($q) + { + $result = $this->sphinx->query($q, $this->remote_table()); + if (!isset($result['matches'])) return false; + $id_set = join(', ', array_keys($result['matches'])); + $this->target->whereAdd("id in ($id_set)"); + return true; + } + + function set_sort_mode($mode) + { + if ('chron' === $mode) { + $this->sphinx->SetSortMode(SPH_SORT_ATTR_DESC, 'created_ts'); + return $this->target->orderBy('created desc'); + } + } + + function remote_table() + { + return $this->dbname() . '_' . $this->table; + } + + function dbname() + { + // @fixme there should be a less dreadful way to do this. + // DB objects won't give database back until they connect, it's confusing + if (preg_match('!^.*?://.*?:.*?@.*?/(.*?)$!', common_config('db', 'database'), $matches)) { + return $matches[1]; + } + throw new ServerException("Sphinx search could not identify database name"); + } +} diff --git a/scripts/sphinx-cron.sh b/scripts/sphinx-cron.sh deleted file mode 100755 index bc537af1a..000000000 --- a/scripts/sphinx-cron.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - -# StatusNet - a distributed open-source microblogging tool - -# Copyright (C) 2008, 2009, StatusNet, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -# This program tries to start the daemons for StatusNet. -# Note that the 'maildaemon' needs to run as a mail filter. - -/usr/local/bin/indexer --config /usr/local/etc/sphinx.conf --all --rotate - diff --git a/scripts/sphinx-indexer.sh b/scripts/sphinx-indexer.sh deleted file mode 100755 index 1ec0826be..000000000 --- a/scripts/sphinx-indexer.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - -# StatusNet - a distributed open-source microblogging tool - -# Copyright (C) 2008, 2009, StatusNet, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -# This program tries to start the daemons for StatusNet. -# Note that the 'maildaemon' needs to run as a mail filter. - -/usr/local/bin/indexer --config /usr/local/etc/sphinx.conf --all - |