From 640628de2d593933e810b4785dfe38923b979713 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:03:19 -0400 Subject: A queuehandler for blowing caches offline We add a queuehandler for blowing the memcached caches off-line. This should speed up the processing of new notices. --- lib/util.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/util.php b/lib/util.php index e0eda0114..12797891c 100644 --- a/lib/util.php +++ b/lib/util.php @@ -879,7 +879,17 @@ function common_broadcast_notice($notice, $remote=false) function common_enqueue_notice($notice) { - foreach (array('jabber', 'omb', 'sms', 'public', 'twitter', 'facebook', 'ping') as $transport) { + $transports = array('omb', 'sms', 'twitter', 'facebook', 'ping'); + + if (common_config('xmpp', 'enabled')) { + $transports = array_merge($transports, array('jabber', 'public')); + } + + if (common_config('memcached', 'enabled')) { + $transports[] = 'memcached'; + } + + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; $qi->transport = $transport; @@ -1332,7 +1342,7 @@ function common_compatible_license($from, $to) */ function common_database_tablename($tablename) { - + if(common_config('db','quote_identifiers')) { $tablename = '"'. $tablename .'"'; } -- cgit v1.2.3-54-g00ecf From aee45ea91dcec4736d8b9befe17e030b873d9226 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:08:48 -0400 Subject: Add an inbox queue handler Handle distributing a notice to multiple inboxes in a queue handler rather than in the Web action. --- classes/Notice.php | 5 +++- lib/util.php | 4 +++ scripts/inboxqueuehandler.php | 57 +++++++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 2 +- scripts/stopdaemons.sh | 3 ++- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 scripts/inboxqueuehandler.php (limited to 'lib') diff --git a/classes/Notice.php b/classes/Notice.php index fbfeb9489..ff00f2a94 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,10 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - $notice->addToInboxes(); + if (!common_config('queues', 'enabled')) { + $notice->addToInboxes(); + } + $notice->query('COMMIT'); Event::handle('EndNoticeSave', array($notice)); diff --git a/lib/util.php b/lib/util.php index 12797891c..9b6d2941a 100644 --- a/lib/util.php +++ b/lib/util.php @@ -889,6 +889,10 @@ function common_enqueue_notice($notice) $transports[] = 'memcached'; } + if (common_config('queues', 'enabled')) { + $transports[] = 'inbox'; + } + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php new file mode 100644 index 000000000..16e334b83 --- /dev/null +++ b/scripts/inboxqueuehandler.php @@ -0,0 +1,57 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class InboxQueueHandler extends QueueHandler +{ + function transport() + { + return 'inbox'; + } + + function handle_notice($notice) + { + common_log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); + $notice->addToInboxes(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new InboxQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index 08de6d954..66f9ed4e0 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -25,7 +25,7 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ - memcachedqueuehandler.php; do + memcachedqueuehandler.php inboxqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index e5a181cd1..196991de0 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,8 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ + memcachedhandler inboxhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From ece70bf326fa680ec7acdef00751e2bb71bceee1 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 09:35:10 +0000 Subject: incorrect config setting for inboxes --- lib/util.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/util.php b/lib/util.php index af4db4f02..d77039b74 100644 --- a/lib/util.php +++ b/lib/util.php @@ -889,7 +889,8 @@ function common_enqueue_notice($notice) $transports[] = 'memcached'; } - if (common_config('queues', 'enabled')) { + if (common_config('inboxes', 'enabled') === true || + common_config('inboxes', 'enabled') === 'transitional') { $transports[] = 'inbox'; } -- cgit v1.2.3-54-g00ecf From 7c383dc1d46cae8abc6d1e5c7eb93ad7cdd91f63 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 10:08:26 +0000 Subject: alert to what transport we're checking for --- lib/queuehandler.php | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/queuehandler.php b/lib/queuehandler.php index 9ce9e32b3..fde650d9e 100644 --- a/lib/queuehandler.php +++ b/lib/queuehandler.php @@ -36,7 +36,7 @@ class QueueHandler extends Daemon $this->set_id($id); } } - + function class_name() { return ucfirst($this->transport()) . 'Handler'; @@ -46,7 +46,7 @@ class QueueHandler extends Daemon { return strtolower($this->class_name().'.'.$this->get_id()); } - + function get_id() { return $this->_id; @@ -56,16 +56,16 @@ class QueueHandler extends Daemon { $this->_id = $id; } - + function transport() { return null; } - + function start() { } - + function finish() { } @@ -74,14 +74,14 @@ class QueueHandler extends Daemon { return true; } - + function run() { if (!$this->start()) { return false; } - $this->log(LOG_INFO, 'checking for queued notices'); $transport = $this->transport(); + $this->log(LOG_INFO, 'checking for queued notices for "' . $transport . '"'); do { $qi = Queue_item::top($transport); if ($qi) { @@ -113,7 +113,7 @@ class QueueHandler extends Daemon } else { $this->clear_old_claims(); $this->idle(5); - } + } } while (true); if (!$this->finish()) { return false; @@ -127,7 +127,7 @@ class QueueHandler extends Daemon sleep($timeout); } } - + function clear_old_claims() { $qi = new Queue_item(); @@ -137,10 +137,9 @@ class QueueHandler extends Daemon $qi->free(); unset($qi); } - + function log($level, $msg) { common_log($level, $this->class_name() . ' ('. $this->get_id() .'): '.$msg); } } - \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 1c0d82de3bb7f75649a017a7d5632a6e070876c2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 10:09:08 +0000 Subject: 8-char limit on transports --- lib/util.php | 3 ++- scripts/memcachedqueuehandler.php | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/util.php b/lib/util.php index d77039b74..f862d7fcc 100644 --- a/lib/util.php +++ b/lib/util.php @@ -886,7 +886,8 @@ function common_enqueue_notice($notice) } if (common_config('memcached', 'enabled')) { - $transports[] = 'memcached'; + // Note: limited to 8 chars + $transports[] = 'memcache'; } if (common_config('inboxes', 'enabled') === true || diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php index 6e819b41f..185b781f7 100755 --- a/scripts/memcachedqueuehandler.php +++ b/scripts/memcachedqueuehandler.php @@ -37,7 +37,7 @@ class MemcachedQueueHandler extends QueueHandler { function transport() { - return 'memcached'; + return 'memcache'; } function start() { -- cgit v1.2.3-54-g00ecf