summaryrefslogtreecommitdiff
path: root/lib/dbqueuemanager.php
diff options
context:
space:
mode:
authorBrion Vibber <brion@pobox.com>2010-01-22 12:52:36 -0800
committerBrion Vibber <brion@pobox.com>2010-01-22 12:52:36 -0800
commitc7507e7e9dafa6d6e054978e720e4fce3abc9929 (patch)
treed756f2a2c7c9b487b6ef0f066a87f3cde40bc65d /lib/dbqueuemanager.php
parenta3e484a0e898bb94dd45cd7807bea1a931d7c6a9 (diff)
XMPP queued output & initial retooling of DB queue manager to support non-Notice objects.
Queue handlers for XMPP individual & firehose output now send their XML stanzas to another output queue instead of connecting directly to the chat server. This lets us have as many general processing threads as we need, while all actual XMPP input and output go through a single daemon with a single connection open. This avoids problems with multiple connected resources: * multiple windows shown in some chat clients (psi, gajim, kopete) * extra load on server * incoming message delivery forwarding issues Database changes: * queue_item drops 'notice_id' in favor of a 'frame' blob. This is based on Craig Andrews' work branch to generalize queues to take any object, but conservatively leaving out the serialization for now. Table updater (preserves any existing queued items) in db/rc3to09.sql Code changes to watch out for: * Queue handlers should now define a handle() method instead of handle_notice() * QueueDaemon and XmppDaemon now share common i/o (IoMaster) and respawning thread management (RespawningDaemon) infrastructure. * The polling XmppConfirmManager has been dropped, as the message is queued directly when saving IM settings. * Enable $config['queue']['debug_memory'] to output current memory usage at each run through the event loop to watch for memory leaks To do: * Adapt XMPP i/o to component connection mode for multi-site support. * XMPP input can also be broken out to a queue, which would allow the actual notice save etc to be handled by general queue threads. * Make sure there are no problems with simply pushing serialized Notice objects to queues. * Find a way to improve interactive performance of the database-backed queue handler; polling is pretty painful to XMPP. * Possibly redo the way QueueHandlers are injected into a QueueManager. The grouping used to split out the XMPP output queue is a bit awkward. Conflicts: scripts/xmppdaemon.php
Diffstat (limited to 'lib/dbqueuemanager.php')
-rw-r--r--lib/dbqueuemanager.php140
1 files changed, 42 insertions, 98 deletions
diff --git a/lib/dbqueuemanager.php b/lib/dbqueuemanager.php
index 889365b64..c6350fc66 100644
--- a/lib/dbqueuemanager.php
+++ b/lib/dbqueuemanager.php
@@ -31,19 +31,17 @@
class DBQueueManager extends QueueManager
{
/**
- * Saves a notice object reference into the queue item table.
+ * Saves an object reference into the queue item table.
* @return boolean true on success
* @throws ServerException on failure
*/
public function enqueue($object, $queue)
{
- $notice = $object;
-
$qi = new Queue_item();
- $qi->notice_id = $notice->id;
+ $qi->frame = $this->encode($object);
$qi->transport = $queue;
- $qi->created = $notice->created;
+ $qi->created = common_sql_now();
$result = $qi->insert();
if (!$result) {
@@ -57,146 +55,92 @@ class DBQueueManager extends QueueManager
}
/**
- * Poll every minute for new events during idle periods.
+ * Poll every 10 seconds for new events during idle periods.
* We'll look in more often when there's data available.
*
* @return int seconds
*/
public function pollInterval()
{
- return 60;
+ return 10;
}
/**
* Run a polling cycle during idle processing in the input loop.
- * @return boolean true if we had a hit
+ * @return boolean true if we should poll again for more data immediately
*/
public function poll()
{
$this->_log(LOG_DEBUG, 'Checking for notices...');
- $item = $this->_nextItem();
- if ($item === false) {
+ $qi = Queue_item::top($this->getQueues());
+ if (empty($qi)) {
$this->_log(LOG_DEBUG, 'No notices waiting; idling.');
return false;
}
- if ($item === true) {
- // We dequeued an entry for a deleted or invalid notice.
- // Consider it a hit for poll rate purposes.
- return true;
- }
- list($queue, $notice) = $item;
- $this->_log(LOG_INFO, 'Got notice '. $notice->id . ' for transport ' . $queue);
-
- // Yay! Got one!
- $handler = $this->getHandler($queue);
- if ($handler) {
- if ($handler->handle_notice($notice)) {
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] Successfully handled notice");
- $this->_done($notice, $queue);
+ $queue = $qi->transport;
+ $item = $this->decode($qi->frame);
+
+ if ($item) {
+ $rep = $this->logrep($item);
+ $this->_log(LOG_INFO, "Got $rep for transport $queue");
+
+ $handler = $this->getHandler($queue);
+ if ($handler) {
+ if ($handler->handle($item)) {
+ $this->_log(LOG_INFO, "[$queue:$rep] Successfully handled item");
+ $this->_done($qi);
+ } else {
+ $this->_log(LOG_INFO, "[$queue:$rep] Failed to handle item");
+ $this->_fail($qi);
+ }
} else {
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] Failed to handle notice");
- $this->_fail($notice, $queue);
+ $this->_log(LOG_INFO, "[$queue:$rep] No handler for queue $queue; discarding.");
+ $this->_done($qi);
}
} else {
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] No handler for queue $queue; discarding.");
- $this->_done($notice, $queue);
+ $this->_log(LOG_INFO, "[$queue] Got empty/deleted item, discarding");
+ $this->_fail($qi);
}
return true;
}
/**
- * Pop the oldest unclaimed item off the queue set and claim it.
- *
- * @return mixed false if no items; true if bogus hit; otherwise array(string, Notice)
- * giving the queue transport name.
- */
- protected function _nextItem()
- {
- $start = time();
- $result = null;
-
- $qi = Queue_item::top();
- if (empty($qi)) {
- return false;
- }
-
- $queue = $qi->transport;
- $notice = Notice::staticGet('id', $qi->notice_id);
- if (empty($notice)) {
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] dequeued non-existent notice");
- $qi->delete();
- return true;
- }
-
- $result = $notice;
- return array($queue, $notice);
- }
-
- /**
* Delete our claimed item from the queue after successful processing.
*
- * @param Notice $object
- * @param string $queue
+ * @param QueueItem $qi
*/
- protected function _done($object, $queue)
+ protected function _done($qi)
{
- // XXX: right now, we only handle notices
-
- $notice = $object;
-
- $qi = Queue_item::pkeyGet(array('notice_id' => $notice->id,
- 'transport' => $queue));
+ $queue = $qi->transport;
- if (empty($qi)) {
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] Cannot find queue item");
- } else {
- if (empty($qi->claimed)) {
- $this->_log(LOG_WARNING, "[$queue:notice $notice->id] Reluctantly releasing unclaimed queue item");
- }
- $qi->delete();
- $qi->free();
+ if (empty($qi->claimed)) {
+ $this->_log(LOG_WARNING, "Reluctantly releasing unclaimed queue item $qi->id from $qi->queue");
}
+ $qi->delete();
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] done with item");
$this->stats('handled', $queue);
-
- $notice->free();
}
/**
* Free our claimed queue item for later reprocessing in case of
* temporary failure.
*
- * @param Notice $object
- * @param string $queue
+ * @param QueueItem $qi
*/
- protected function _fail($object, $queue)
+ protected function _fail($qi)
{
- // XXX: right now, we only handle notices
-
- $notice = $object;
-
- $qi = Queue_item::pkeyGet(array('notice_id' => $notice->id,
- 'transport' => $queue));
+ $queue = $qi->transport;
- if (empty($qi)) {
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] Cannot find queue item");
+ if (empty($qi->claimed)) {
+ $this->_log(LOG_WARNING, "[$queue:item $qi->id] Ignoring failure for unclaimed queue item");
} else {
- if (empty($qi->claimed)) {
- $this->_log(LOG_WARNING, "[$queue:notice $notice->id] Ignoring failure for unclaimed queue item");
- } else {
- $orig = clone($qi);
- $qi->claimed = null;
- $qi->update($orig);
- $qi = null;
- }
+ $orig = clone($qi);
+ $qi->claimed = null;
+ $qi->update($orig);
}
- $this->_log(LOG_INFO, "[$queue:notice $notice->id] done with queue item");
$this->stats('error', $queue);
-
- $notice->free();
}
protected function _log($level, $msg)