Stomp queue restructuring for mass scalability:

- Multiplexing queues into groups and for multiple sites.
- Sharing vs breakout configurable per site and per queue via $config['queue']['breakout']
- Detect how many times a message is redelivered, discard if it's killed too many daemons
 - count configurable with $config['queue']['max_retries']
 - can dump the items to files in $config['queue']['dead_letter_dir']

Queue daemon memory & resource leak fixes:
- avoid unnecessary reconnections to memcached server (switch persistent connections back in on second initialization, assuming it's child process)
- monkey-patch for leaky .ini loads in DB_DataObject::databaseStructure() - was leaking 200k per active switch
- applied leak fixes to Status_network as well, using intermediate base Safe_DataObject for both it and Memcache_DataObject

Misc queue fixes:
- correct handling of child processes exiting due to signal termination instead of regular exit
- shutdown instead of infinite respawn loop if we're already past the soft memory limit at startup
- Added --all option for xmppdaemon... still opens one xmpp connection per site that has xmpp active

Cache updates:
- add Cache::increment() method with native support for memcached atomic increment

3 files changed, 263 insertions, 55 deletions

diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php
index ab65c30ce..16c3d906c 100644
--- a/classes/Memcached_DataObject.php
+++ b/classes/Memcached_DataObject.php
@@ -19,58 +19,9 @@
 
 if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
 
-class Memcached_DataObject extends DB_DataObject
+class Memcached_DataObject extends Safe_DataObject
 {
     /**
-     * Destructor to free global memory resources associated with
-     * this data object when it's unset or goes out of scope.
-     * DB_DataObject doesn't do this yet by itself.
-     */
-
-    function __destruct()
-    {
-        $this->free();
-        if (method_exists('DB_DataObject', '__destruct')) {
-            parent::__destruct();
-        }
-    }
-
-    /**
-     * Magic function called at serialize() time.
-     *
-     * We use this to drop a couple process-specific references
-     * from DB_DataObject which can cause trouble in future
-     * processes.
-     *
-     * @return array of variable names to include in serialization.
-     */
-    function __sleep()
-    {
-        $vars = array_keys(get_object_vars($this));
-        $skip = array('_DB_resultid', '_link_loaded');
-        return array_diff($vars, $skip);
-    }
-
-    /**
-     * Magic function called at unserialize() time.
-     *
-     * Clean out some process-specific variables which might
-     * be floating around from a previous process's cached
-     * objects.
-     *
-     * Old cached objects may still have them.
-     */
-    function __wakeup()
-    {
-        // Refers to global state info from a previous process.
-        // Clear this out so we don't accidentally break global
-        // state in *this* process.
-        $this->_DB_resultid = null;
-        // We don't have any local DBO refs, so clear these out.
-        $this->_link_loaded = false;
-    }
-
-    /**
      * Wrapper for DB_DataObject's static lookup using memcached
      * as backing instead of an in-process cache array.
      *
@@ -579,3 +530,4 @@ class Memcached_DataObject extends DB_DataObject
         return $c->set($cacheKey, $value);
     }
 }
+
diff --git a/classes/Safe_DataObject.php b/classes/Safe_DataObject.php
new file mode 100644
index 000000000..021f7b506
--- /dev/null
+++ b/classes/Safe_DataObject.php
@@ -0,0 +1,250 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
+
+/**
+ * Extended DB_DataObject to improve a few things:
+ * - free global resources from destructor
+ * - remove bogus global references from serialized objects
+ * - don't leak memory when loading already-used .ini files
+ *   (eg when using the same schema on thousands of databases)
+ */
+class Safe_DataObject extends DB_DataObject
+{
+    /**
+     * Destructor to free global memory resources associated with
+     * this data object when it's unset or goes out of scope.
+     * DB_DataObject doesn't do this yet by itself.
+     */
+
+    function __destruct()
+    {
+        $this->free();
+        if (method_exists('DB_DataObject', '__destruct')) {
+            parent::__destruct();
+        }
+    }
+
+    /**
+     * Magic function called at serialize() time.
+     *
+     * We use this to drop a couple process-specific references
+     * from DB_DataObject which can cause trouble in future
+     * processes.
+     *
+     * @return array of variable names to include in serialization.
+     */
+    function __sleep()
+    {
+        $vars = array_keys(get_object_vars($this));
+        $skip = array('_DB_resultid', '_link_loaded');
+        return array_diff($vars, $skip);
+    }
+
+    /**
+     * Magic function called at unserialize() time.
+     *
+     * Clean out some process-specific variables which might
+     * be floating around from a previous process's cached
+     * objects.
+     *
+     * Old cached objects may still have them.
+     */
+    function __wakeup()
+    {
+        // Refers to global state info from a previous process.
+        // Clear this out so we don't accidentally break global
+        // state in *this* process.
+        $this->_DB_resultid = null;
+        // We don't have any local DBO refs, so clear these out.
+        $this->_link_loaded = false;
+    }
+
+
+    /**
+     * Work around memory-leak bugs...
+     * Had to copy-paste the whole function in order to patch a couple lines of it.
+     * Would be nice if this code was better factored.
+     *     
+     * @param optional string  name of database to assign / read
+     * @param optional array   structure of database, and keys
+     * @param optional array  table links
+     *
+     * @access public
+     * @return true or PEAR:error on wrong paramenters.. or false if no file exists..
+     *              or the array(tablename => array(column_name=>type)) if called with 1 argument.. (databasename)
+     */
+    function databaseStructure()
+    {
+
+        global $_DB_DATAOBJECT;
+        
+        // Assignment code 
+        
+        if ($args = func_get_args()) {
+        
+            if (count($args) == 1) {
+                
+                // this returns all the tables and their structure..
+                if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) {
+                    $this->debug("Loading Generator as databaseStructure called with args",1);
+                }
+                
+                $x = new DB_DataObject;
+                $x->_database = $args[0];
+                $this->_connect();
+                $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5];
+       
+                $tables = $DB->getListOf('tables');
+                class_exists('DB_DataObject_Generator') ? '' : 
+                    require_once 'DB/DataObject/Generator.php';
+                    
+                foreach($tables as $table) {
+                    $y = new DB_DataObject_Generator;
+                    $y->fillTableSchema($x->_database,$table);
+                }
+                return $_DB_DATAOBJECT['INI'][$x->_database];            
+            } else {
+        
+                $_DB_DATAOBJECT['INI'][$args[0]] = isset($_DB_DATAOBJECT['INI'][$args[0]]) ?
+                    $_DB_DATAOBJECT['INI'][$args[0]] + $args[1] : $args[1];
+                
+                if (isset($args[1])) {
+                    $_DB_DATAOBJECT['LINKS'][$args[0]] = isset($_DB_DATAOBJECT['LINKS'][$args[0]]) ?
+                        $_DB_DATAOBJECT['LINKS'][$args[0]] + $args[2] : $args[2];
+                }
+                return true;
+            }
+          
+        }
+        
+        
+        
+        if (!$this->_database) {
+            $this->_connect();
+        }
+        
+        // loaded already?
+        if (!empty($_DB_DATAOBJECT['INI'][$this->_database])) {
+            
+            // database loaded - but this is table is not available..
+            if (
+                    empty($_DB_DATAOBJECT['INI'][$this->_database][$this->__table]) 
+                    && !empty($_DB_DATAOBJECT['CONFIG']['proxy'])
+                ) {
+                if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) {
+                    $this->debug("Loading Generator to fetch Schema",1);
+                }
+                class_exists('DB_DataObject_Generator') ? '' : 
+                    require_once 'DB/DataObject/Generator.php';
+                    
+                
+                $x = new DB_DataObject_Generator;
+                $x->fillTableSchema($this->_database,$this->__table);
+            }
+            return true;
+        }
+        
+        
+        if (empty($_DB_DATAOBJECT['CONFIG'])) {
+            DB_DataObject::_loadConfig();
+        }
+        
+        // if you supply this with arguments, then it will take those
+        // as the database and links array...
+         
+        $schemas = isset($_DB_DATAOBJECT['CONFIG']['schema_location']) ?
+            array("{$_DB_DATAOBJECT['CONFIG']['schema_location']}/{$this->_database}.ini") :
+            array() ;
+                 
+        if (isset($_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"])) {
+            $schemas = is_array($_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"]) ?
+                $_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"] :
+                explode(PATH_SEPARATOR,$_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"]);
+        }
+                    
+         
+        /* BEGIN CHANGED FROM UPSTREAM */
+        $_DB_DATAOBJECT['INI'][$this->_database] = $this->parseIniFiles($schemas);
+        /* END CHANGED FROM UPSTREAM */
+
+        // now have we loaded the structure.. 
+        
+        if (!empty($_DB_DATAOBJECT['INI'][$this->_database][$this->__table])) {
+            return true;
+        }
+        // - if not try building it..
+        if (!empty($_DB_DATAOBJECT['CONFIG']['proxy'])) {
+            class_exists('DB_DataObject_Generator') ? '' : 
+                require_once 'DB/DataObject/Generator.php';
+                
+            $x = new DB_DataObject_Generator;
+            $x->fillTableSchema($this->_database,$this->__table);
+            // should this fail!!!???
+            return true;
+        }
+        $this->debug("Cant find database schema: {$this->_database}/{$this->__table} \n".
+                    "in links file data: " . print_r($_DB_DATAOBJECT['INI'],true),"databaseStructure",5);
+        // we have to die here!! - it causes chaos if we dont (including looping forever!)
+        $this->raiseError( "Unable to load schema for database and table (turn debugging up to 5 for full error message)", DB_DATAOBJECT_ERROR_INVALIDARGS, PEAR_ERROR_DIE);
+        return false;
+    }
+
+    /** For parseIniFiles */
+    protected static $iniCache = array();
+
+    /**
+     * When switching site configurations, DB_DataObject was loading its
+     * .ini files over and over, leaking gobs of memory.
+     * This refactored helper function uses a local cache of .ini files
+     * to minimize the leaks.
+     *
+     * @param array of .ini file names $schemas
+     * @return array
+     */
+    protected function parseIniFiles($schemas)
+    {
+        $key = implode("|", $schemas);
+        if (!isset(Safe_DataObject::$iniCache[$key])) {
+            $data = array();
+            foreach ($schemas as $ini) {
+                if (file_exists($ini) && is_file($ini)) {
+                    $data = array_merge($data, parse_ini_file($ini, true));
+
+                    if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { 
+                        if (!is_readable ($ini)) {
+                            $this->debug("ini file is not readable: $ini","databaseStructure",1);
+                        } else {
+                            $this->debug("Loaded ini file: $ini","databaseStructure",1);
+                        }
+                    }
+                } else {
+                    if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) {
+                        $this->debug("Missing ini file: $ini","databaseStructure",1);
+                    }
+                }
+            }
+            Safe_DataObject::$iniCache[$key] = $data;
+        }
+
+        return Safe_DataObject::$iniCache[$key];
+    }
+}
+
diff --git a/classes/Status_network.php b/classes/Status_network.php
index 4bda24b6a..a452c32ce 100644
--- a/classes/Status_network.php
+++ b/classes/Status_network.php
@@ -21,7 +21,7 @@
 
 if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
 
-class Status_network extends DB_DataObject
+class Status_network extends Safe_DataObject
 {
     ###START_AUTOCODE
     /* the code below is auto generated do not remove the above tag */
@@ -57,6 +57,7 @@ class Status_network extends DB_DataObject
     ###END_AUTOCODE
 
     static $cache = null;
+    static $cacheInitialized = false;
     static $base = null;
     static $wildcard = null;
 
@@ -78,11 +79,15 @@ class Status_network extends DB_DataObject
         if (class_exists('Memcache')) {
             self::$cache = new Memcache();
 
-            // Can't close persistent connections, making forking painful.
+            // If we're a parent command-line process we need
+            // to be able to close out the connection after
+            // forking, so disable persistence.
             //
-            // @fixme only do this in *parent* CLI processes.
-            // single-process and child-processes *should* use persistent.
-            $persist = php_sapi_name() != 'cli';
+            // We'll turn it back on again the second time
+            // through which will either be in a child process,
+            // or a single-process script which is switching
+            // configurations.
+            $persist = php_sapi_name() != 'cli' || self::$cacheInitialized;
             if (is_array($servers)) {
                 foreach($servers as $server) {
                     self::$cache->addServer($server, 11211, $persist);
@@ -90,6 +95,7 @@ class Status_network extends DB_DataObject
             } else {
                 self::$cache->addServer($servers, 11211, $persist);
             }
+            self::$cacheInitialized = true;
         }
 
         self::$base = $dbname;