From ec145b73fc91dd54695dd374c8a71a11e233b8c0 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 12 Jan 2010 19:57:15 -0800 Subject: Major refactoring of queue handlers to support running multiple sites in one daemon. Key changes: * Initialization code moved from common.php to StatusNet class; can now switch configurations during runtime. * As a consequence, configuration files must now be idempotent... Be careful with constant, function or class definitions. * Control structure for daemons/QueueManager/QueueHandler has been refactored; the run loop is now managed by IoMaster run via scripts/queuedaemon.php IoManager subclasses are woken to handle socket input or polling, and may cover multiple sites. * Plugins can implement notice queue handlers more easily by registering a QueueHandler class; no more need to add a daemon. The new QueueDaemon runs from scripts/queuedaemon.php: * This replaces most of the old *handler.php scripts; they've been refactored to the bare handler classes. * Spawns multiple child processes to spread load; defaults to CPU count on Linux and Mac OS X systems, or override with --threads=N * When multithreaded, child processes are automatically respawned on failure. * Threads gracefully shut down and restart when passing a soft memory limit (defaults to 90% of memory_limit), limiting damage from memory leaks. * Support for UDP-based monitoring: http://www.gitorious.org/snqmon Rough control flow diagram: QueueDaemon -> IoMaster -> IoManager QueueManager [listen or poll] -> QueueHandler XmppManager [ping & keepalive] XmppConfirmManager [poll updates] Todo: * Respawning features not currently available running single-threaded. * When running single-site, configuration changes aren't picked up. * New sites or config changes affecting queue subscriptions are not yet handled without a daemon restart. * SNMP monitoring output to integrate with general tools (nagios, ganglia) * Convert XMPP confirmation message sends to use stomp queue instead of polling * Convert xmppdaemon.php to IoManager? * Convert Twitter status, friends import polling daemons to IoManager * Clean up some error reporting and failure modes * May need to adjust queue priorities for best perf in backlog/flood cases Detailed code history available in my daemon-work branch: http://www.gitorious.org/~brion/statusnet/brion-fixes/commits/daemon-work --- lib/iomanager.php | 193 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 lib/iomanager.php (limited to 'lib/iomanager.php') diff --git a/lib/iomanager.php b/lib/iomanager.php new file mode 100644 index 000000000..ee2ff958b --- /dev/null +++ b/lib/iomanager.php @@ -0,0 +1,193 @@ +. + * + * @category QueueManager + * @package StatusNet + * @author Evan Prodromou + * @author Sarven Capadisli + * @author Brion Vibber + * @copyright 2009-2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +abstract class IoManager +{ + const SINGLE_ONLY = 0; + const INSTANCE_PER_SITE = 1; + const INSTANCE_PER_PROCESS = 2; + + /** + * Factory function to get an appropriate subclass. + */ + public abstract static function get(); + + /** + * Tell the i/o queue master if and how we can handle multi-site + * processes. + * + * Return one of: + * IoManager::SINGLE_ONLY + * IoManager::INSTANCE_PER_SITE + * IoManager::INSTANCE_PER_PROCESS + */ + public static function multiSite() + { + return IoManager::SINGLE_ONLY; + } + + /** + * If in a multisite configuration, the i/o master will tell + * your manager about each site you'll have to handle so you + * can do any necessary per-site setup. + * + * @param string $site target site server name + */ + public function addSite($site) + { + /* no-op */ + } + + /** + * This method is called when data is available on one of your + * i/o manager's sockets. The socket with data is passed in, + * in case you have multiple sockets. + * + * If your i/o manager is based on polling during idle processing, + * you don't need to implement this. + * + * @param resource $socket + * @return boolean true on success, false on failure + */ + public function handleInput($socket) + { + return true; + } + + /** + * Return any open sockets that the run loop should listen + * for input on. If input comes in on a listed socket, + * the matching manager's handleInput method will be called. + * + * @return array of resources + */ + function getSockets() + { + return array(); + } + + /** + * Maximum planned time between poll() calls when input isn't waiting. + * Actual time may vary! + * + * When we get a polling hit, the timeout will be cut down to 0 while + * input is coming in, then will back off to this amount if no further + * input shows up. + * + * By default polling is disabled; you must override this to enable + * polling for this manager. + * + * @return int max poll interval in seconds, or 0 to disable polling + */ + function pollInterval() + { + return 0; + } + + /** + * Request a maximum timeout for listeners before the next idle period. + * Actual wait may be shorter, so don't go crazy in your idle()! + * Wait could be longer if other handlers performed some slow activity. + * + * Return 0 to request that listeners return immediately if there's no + * i/o and speed up the idle as much as possible; but don't do that all + * the time as this will burn CPU. + * + * @return int seconds + */ + function timeout() + { + return 60; + } + + /** + * Called by IoManager after each handled item or empty polling cycle. + * This is a good time to e.g. service your XMPP connection. + * + * Doesn't need to be overridden if there's no maintenance to do. + */ + function idle() + { + return true; + } + + /** + * The meat of a polling manager... check for something to do + * and do it! Note that you should not take too long, as other + * i/o managers may need to do some work too! + * + * On a successful hit, the next poll() call will come as soon + * as possible followed by exponential backoff up to pollInterval() + * if no more data is available. + * + * @return boolean true if events were hit + */ + public function poll() + { + return false; + } + + /** + * Initialization, run when the queue manager starts. + * If this function indicates failure, the handler run will be aborted. + * + * @param IoMaster $master process/event controller + * @return boolean true on success, false on failure + */ + public function start($master) + { + $this->master = $master; + return true; + } + + /** + * Cleanup, run when the queue manager ends. + * If this function indicates failure, a warning will be logged. + * + * @return boolean true on success, false on failure + */ + public function finish() + { + return true; + } + + /** + * Ping iomaster's queue status monitor with a stats update. + * Only valid during input loop! + * + * @param string $counter keyword for counter to increment + */ + public function stats($counter, $owners=array()) + { + $this->master->stats($counter, $owners); + } +} + -- cgit v1.2.3-54-g00ecf