Control channel for queue daemons to request graceful shutdown, restart, or update to listen to a newly added or reconfigured site.

  queuectl.php --update -s<site>
  queuectl.php --stop
  queuectl.php --restart

Default control channel is /topic/statusnet-control. For external utilities to send a site update ping direct to the queue server, connect via Stomp and send a message formatted thus:

  update:<nickname>

(Nickname here, *not* server hostname! The rest of the queues will be updated to use nicknames later.)

Note that all currently-connected queue daemons will get these notifications, including both queuedaemon.php and xmppdaemon.php. (XMPP will ignore site update requests for sites that it's not handling.)

Limitations:
* only implemented for stomp queue manager so far
* --update may not yet handle a changed server name properly
* --restart won't reload PHP code files that were already loaded at startup. Still need to stop and restart the daemons from 'outside' when updating code base.

1 files changed, 42 insertions, 11 deletions

diff --git a/lib/iomaster.php b/lib/iomaster.php
index 3bf82bc6b..bcab3542b 100644
--- a/lib/iomaster.php
+++ b/lib/iomaster.php
@@ -38,6 +38,9 @@ abstract class IoMaster
     protected $pollTimeouts = array();
     protected $lastPoll = array();
 
+    public $shutdown = false; // Did we do a graceful shutdown?
+    public $respawn = true; // Should we respawn after shutdown?
+
     /**
      * @param string $id process ID to use in logging/monitoring
      */
@@ -144,7 +147,7 @@ abstract class IoMaster
         $this->logState('init');
         $this->start();
 
-        while (true) {
+        while (!$this->shutdown) {
             $timeouts = array_values($this->pollTimeouts);
             $timeouts[] = 60; // default max timeout
 
@@ -196,16 +199,7 @@ abstract class IoMaster
             $this->logState('idle');
             $this->idle();
 
-            $memoryLimit = $this->softMemoryLimit();
-            if ($memoryLimit > 0) {
-                $usage = memory_get_usage();
-                if ($usage > $memoryLimit) {
-                    common_log(LOG_INFO, "Queue thread hit soft memory limit ($usage > $memoryLimit); gracefully restarting.");
-                    break;
-                } else if (common_config('queue', 'debug_memory')) {
-                    common_log(LOG_DEBUG, "Memory usage $usage");
-                }
-            }
+            $this->checkMemory();
         }
 
         $this->logState('shutdown');
@@ -213,6 +207,24 @@ abstract class IoMaster
     }
 
     /**
+     * Check runtime memory usage, possibly triggering a graceful shutdown
+     * and thread respawn if we've crossed the soft limit.
+     */
+    protected function checkMemory()
+    {
+        $memoryLimit = $this->softMemoryLimit();
+        if ($memoryLimit > 0) {
+            $usage = memory_get_usage();
+            if ($usage > $memoryLimit) {
+                common_log(LOG_INFO, "Queue thread hit soft memory limit ($usage > $memoryLimit); gracefully restarting.");
+                $this->requestRestart();
+            } else if (common_config('queue', 'debug_memory')) {
+                common_log(LOG_DEBUG, "Memory usage $usage");
+            }
+        }
+    }
+
+    /**
      * Return fully-parsed soft memory limit in bytes.
      * @return intval 0 or -1 if not set
      */
@@ -354,5 +366,24 @@ abstract class IoMaster
         $owners[] = "thread:" . $this->id;
         $this->monitor->stats($key, $owners);
     }
+
+    /**
+     * For IoManagers to request a graceful shutdown at end of event loop.
+     */
+    public function requestShutdown()
+    {
+        $this->shutdown = true;
+        $this->respawn = false;
+    }
+
+    /**
+     * For IoManagers to request a graceful restart at end of event loop.
+     */
+    public function requestRestart()
+    {
+        $this->shutdown = true;
+        $this->respawn = true;
+    }
+
 }