From ec145b73fc91dd54695dd374c8a71a11e233b8c0 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 12 Jan 2010 19:57:15 -0800 Subject: Major refactoring of queue handlers to support running multiple sites in one daemon. Key changes: * Initialization code moved from common.php to StatusNet class; can now switch configurations during runtime. * As a consequence, configuration files must now be idempotent... Be careful with constant, function or class definitions. * Control structure for daemons/QueueManager/QueueHandler has been refactored; the run loop is now managed by IoMaster run via scripts/queuedaemon.php IoManager subclasses are woken to handle socket input or polling, and may cover multiple sites. * Plugins can implement notice queue handlers more easily by registering a QueueHandler class; no more need to add a daemon. The new QueueDaemon runs from scripts/queuedaemon.php: * This replaces most of the old *handler.php scripts; they've been refactored to the bare handler classes. * Spawns multiple child processes to spread load; defaults to CPU count on Linux and Mac OS X systems, or override with --threads=N * When multithreaded, child processes are automatically respawned on failure. * Threads gracefully shut down and restart when passing a soft memory limit (defaults to 90% of memory_limit), limiting damage from memory leaks. * Support for UDP-based monitoring: http://www.gitorious.org/snqmon Rough control flow diagram: QueueDaemon -> IoMaster -> IoManager QueueManager [listen or poll] -> QueueHandler XmppManager [ping & keepalive] XmppConfirmManager [poll updates] Todo: * Respawning features not currently available running single-threaded. * When running single-site, configuration changes aren't picked up. * New sites or config changes affecting queue subscriptions are not yet handled without a daemon restart. * SNMP monitoring output to integrate with general tools (nagios, ganglia) * Convert XMPP confirmation message sends to use stomp queue instead of polling * Convert xmppdaemon.php to IoManager? * Convert Twitter status, friends import polling daemons to IoManager * Clean up some error reporting and failure modes * May need to adjust queue priorities for best perf in backlog/flood cases Detailed code history available in my daemon-work branch: http://www.gitorious.org/~brion/statusnet/brion-fixes/commits/daemon-work --- lib/queuehandler.php | 100 ++++++++++++--------------------------------------- 1 file changed, 23 insertions(+), 77 deletions(-) (limited to 'lib/queuehandler.php') diff --git a/lib/queuehandler.php b/lib/queuehandler.php index cd43b1e09..613be6e33 100644 --- a/lib/queuehandler.php +++ b/lib/queuehandler.php @@ -19,14 +19,6 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } -require_once(INSTALLDIR.'/lib/daemon.php'); -require_once(INSTALLDIR.'/classes/Queue_item.php'); -require_once(INSTALLDIR.'/classes/Notice.php'); - -define('CLAIM_TIMEOUT', 1200); -define('QUEUE_HANDLER_MISS_IDLE', 10); -define('QUEUE_HANDLER_HIT_IDLE', 0); - /** * Base class for queue handlers. * @@ -36,24 +28,20 @@ define('QUEUE_HANDLER_HIT_IDLE', 0); * * Subclasses must override at least the following methods: * - transport - * - start - * - finish * - handle_notice - * - * Some subclasses will also want to override the idle handler: - * - idle */ -class QueueHandler extends Daemon +#class QueueHandler extends Daemon +class QueueHandler { - function __construct($id=null, $daemonize=true) - { - parent::__construct($daemonize); - - if ($id) { - $this->set_id($id); - } - } +# function __construct($id=null, $daemonize=true) +# { +# parent::__construct($daemonize); +# +# if ($id) { +# $this->set_id($id); +# } +# } /** * How many seconds a polling-based queue manager should wait between @@ -61,22 +49,23 @@ class QueueHandler extends Daemon * * Defaults to 60 seconds; override to speed up or slow down. * + * @fixme not really compatible with global queue manager * @return int timeout in seconds */ - function timeout() - { - return 60; - } +# function timeout() +# { +# return 60; +# } - function class_name() - { - return ucfirst($this->transport()) . 'Handler'; - } +# function class_name() +# { +# return ucfirst($this->transport()) . 'Handler'; +# } - function name() - { - return strtolower($this->class_name().'.'.$this->get_id()); - } +# function name() +# { +# return strtolower($this->class_name().'.'.$this->get_id()); +# } /** * Return transport keyword which identifies items this queue handler @@ -92,30 +81,6 @@ class QueueHandler extends Daemon return null; } - /** - * Initialization, run when the queue handler starts. - * If this function indicates failure, the handler run will be aborted. - * - * @fixme run() will abort if this doesn't return true, - * but some subclasses don't bother. - * @return boolean true on success, false on failure - */ - function start() - { - } - - /** - * Cleanup, run when the queue handler ends. - * If this function indicates failure, a warning will be logged. - * - * @fixme run() will throw warnings if this doesn't return true, - * but many subclasses don't bother. - * @return boolean true on success, false on failure - */ - function finish() - { - } - /** * Here's the meat of your queue handler -- you're handed a Notice * object, which you may do as you will with. @@ -169,29 +134,10 @@ class QueueHandler extends Daemon return true; } - /** - * Called by QueueHandler after each handled item or empty polling cycle. - * This is a good time to e.g. service your XMPP connection. - * - * Doesn't need to be overridden if there's no maintenance to do. - * - * @param int $timeout seconds to sleep if there's nothing to do - */ - function idle($timeout=0) - { - if ($timeout > 0) { - sleep($timeout); - } - } function log($level, $msg) { common_log($level, $this->class_name() . ' ('. $this->get_id() .'): '.$msg); } - - function getSockets() - { - return array(); - } } -- cgit v1.2.3-54-g00ecf