From c74aea589d5a79d7048470d44e457dffc8919ad3 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 16 Feb 2010 09:01:59 -0800 Subject: Stomp queue restructuring for mass scalability: - Multiplexing queues into groups and for multiple sites. - Sharing vs breakout configurable per site and per queue via $config['queue']['breakout'] - Detect how many times a message is redelivered, discard if it's killed too many daemons - count configurable with $config['queue']['max_retries'] - can dump the items to files in $config['queue']['dead_letter_dir'] Queue daemon memory & resource leak fixes: - avoid unnecessary reconnections to memcached server (switch persistent connections back in on second initialization, assuming it's child process) - monkey-patch for leaky .ini loads in DB_DataObject::databaseStructure() - was leaking 200k per active switch - applied leak fixes to Status_network as well, using intermediate base Safe_DataObject for both it and Memcache_DataObject Misc queue fixes: - correct handling of child processes exiting due to signal termination instead of regular exit - shutdown instead of infinite respawn loop if we're already past the soft memory limit at startup - Added --all option for xmppdaemon... still opens one xmpp connection per site that has xmpp active Cache updates: - add Cache::increment() method with native support for memcached atomic increment --- lib/default.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib/default.php') diff --git a/lib/default.php b/lib/default.php index bf4b83718..a74cccae1 100644 --- a/lib/default.php +++ b/lib/default.php @@ -81,7 +81,7 @@ $default = 'subsystem' => 'db', # default to database, or 'stomp' 'stomp_server' => null, 'queue_basename' => '/queue/statusnet/', - 'control_channel' => '/topic/statusnet-control', // broadcasts to all queue daemons + 'control_channel' => '/topic/statusnet/control', // broadcasts to all queue daemons 'stomp_username' => null, 'stomp_password' => null, 'stomp_persistent' => true, // keep items across queue server restart, if persistence is enabled @@ -91,6 +91,12 @@ $default = 'spawndelay' => 1, // Wait at least N seconds between (re)spawns of child processes to avoid slamming the queue server with subscription startup 'debug_memory' => false, // true to spit memory usage to log 'inboxes' => true, // true to do inbox distribution & output queueing from in background via 'distrib' queue + 'breakout' => array('*' => 'shared'), // set global or per-handler queue breakout + // 'shared': use a shared queue for all sites + // 'handler': share each/this handler over multiple sites + // 'site': break out for each/this handler on this site + 'max_retries' => 10, // drop messages after N failed attempts to process (Stomp) + 'dead_letter_dir' => false, // set to directory to save dropped messages into (Stomp) ), 'license' => array('type' => 'cc', # can be 'cc', 'allrightsreserved', 'private' -- cgit v1.2.3-54-g00ecf