From 3081b2e31708520e6ed83041ce3ba04329d3ec90 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 08:44:48 -0400 Subject: start of querybyid --- classes/User.php | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/User.php b/classes/User.php index 3b9b5cd83..098381f73 100644 --- a/classes/User.php +++ b/classes/User.php @@ -444,21 +444,42 @@ class User extends Memcached_DataObject 'SELECT notice.* ' . 'FROM notice JOIN subscription ON notice.profile_id = subscription.subscribed ' . 'WHERE subscription.subscriber = %d '; - $order = null; + return Notice::getStream(sprintf($qry, $this->id), + 'user:notices_with_friends:' . $this->id, + $offset, $limit, $since_id, $before_id, + $order, $since); } else if ($enabled === true || ($enabled == 'transitional' && $this->inboxed == 1)) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN notice_inbox ON notice.id = notice_inbox.notice_id ' . - 'WHERE notice_inbox.user_id = %d '; - // NOTE: we override ORDER - $order = null; + $cache = common_memcache(); + + if (!empty($cache)) { + + # Get the notices out of the cache + + $notices = $cache->get(common_cache_key($cachekey)); + + # On a cache hit, return a DB-object-like wrapper + + if ($notices !== false) { + $wrapper = new ArrayWrapper(array_slice($notices, $offset, $limit)); + return $wrapper; + } + } + + $inbox = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); + + $ids = array(); + + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + + $inbox->free(); + unset($inbox); + + return Notice::getStreamByIds($ids, 'user:notices_with_friends:' . $this->id); } - return Notice::getStream(sprintf($qry, $this->id), - 'user:notices_with_friends:' . $this->id, - $offset, $limit, $since_id, $before_id, - $order, $since); } function blowFavesCache() -- cgit v1.2.3-54-g00ecf From 640628de2d593933e810b4785dfe38923b979713 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:03:19 -0400 Subject: A queuehandler for blowing caches offline We add a queuehandler for blowing the memcached caches off-line. This should speed up the processing of new notices. --- classes/Notice.php | 25 ++++++++++++++++- lib/util.php | 14 ++++++++-- scripts/memcachedqueuehandler.php | 58 +++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 3 +- scripts/stopdaemons.sh | 2 +- 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 scripts/memcachedqueuehandler.php (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 5fa0d79a1..fbfeb9489 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -207,7 +207,11 @@ class Notice extends Memcached_DataObject # XXX: someone clever could prepend instead of clearing the cache if (common_config('memcached', 'enabled')) { - $notice->blowCaches(); + if (common_config('queues', 'enabled')) { + $notice->blowAuthorCaches(); + } else { + $notice->blowCaches(); + } } return $notice; @@ -271,6 +275,25 @@ class Notice extends Memcached_DataObject $this->blowGroupCache($blowLast); } + function blowAuthorCaches($blowLast=false) + { + // Clear the user's cache + $cache = common_memcache(); + if ($cache) { + $user = User::staticGet($this->profile_id); + if (!empty($user)) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + if ($blowLast) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + } + } + $user->free(); + unset($user); + } + $this->blowNoticeCache($blowLast); + $this->blowPublicCache($blowLast); + } + function blowGroupCache($blowLast=false) { $cache = common_memcache(); diff --git a/lib/util.php b/lib/util.php index e0eda0114..12797891c 100644 --- a/lib/util.php +++ b/lib/util.php @@ -879,7 +879,17 @@ function common_broadcast_notice($notice, $remote=false) function common_enqueue_notice($notice) { - foreach (array('jabber', 'omb', 'sms', 'public', 'twitter', 'facebook', 'ping') as $transport) { + $transports = array('omb', 'sms', 'twitter', 'facebook', 'ping'); + + if (common_config('xmpp', 'enabled')) { + $transports = array_merge($transports, array('jabber', 'public')); + } + + if (common_config('memcached', 'enabled')) { + $transports[] = 'memcached'; + } + + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; $qi->transport = $transport; @@ -1332,7 +1342,7 @@ function common_compatible_license($from, $to) */ function common_database_tablename($tablename) { - + if(common_config('db','quote_identifiers')) { $tablename = '"'. $tablename .'"'; } diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php new file mode 100644 index 000000000..3fcebcfc3 --- /dev/null +++ b/scripts/memcachedqueuehandler.php @@ -0,0 +1,58 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class MemcachedQueueHandler extends QueueHandler +{ + function transport() + { + return 'memcached'; + } + + function handle_notice($notice) + { + // XXX: fork here + common_log(LOG_INFO, "Blowing memcached for $notice->id\n"; + $notice->blowCaches(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new MemcachedQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index c3729761d..08de6d954 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -24,7 +24,8 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ - twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php; do + twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ + memcachedqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index 2bb8f9ecb..e5a181cd1 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,7 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler ; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From aee45ea91dcec4736d8b9befe17e030b873d9226 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:08:48 -0400 Subject: Add an inbox queue handler Handle distributing a notice to multiple inboxes in a queue handler rather than in the Web action. --- classes/Notice.php | 5 +++- lib/util.php | 4 +++ scripts/inboxqueuehandler.php | 57 +++++++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 2 +- scripts/stopdaemons.sh | 3 ++- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 scripts/inboxqueuehandler.php (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index fbfeb9489..ff00f2a94 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,10 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - $notice->addToInboxes(); + if (!common_config('queues', 'enabled')) { + $notice->addToInboxes(); + } + $notice->query('COMMIT'); Event::handle('EndNoticeSave', array($notice)); diff --git a/lib/util.php b/lib/util.php index 12797891c..9b6d2941a 100644 --- a/lib/util.php +++ b/lib/util.php @@ -889,6 +889,10 @@ function common_enqueue_notice($notice) $transports[] = 'memcached'; } + if (common_config('queues', 'enabled')) { + $transports[] = 'inbox'; + } + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php new file mode 100644 index 000000000..16e334b83 --- /dev/null +++ b/scripts/inboxqueuehandler.php @@ -0,0 +1,57 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class InboxQueueHandler extends QueueHandler +{ + function transport() + { + return 'inbox'; + } + + function handle_notice($notice) + { + common_log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); + $notice->addToInboxes(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new InboxQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index 08de6d954..66f9ed4e0 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -25,7 +25,7 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ - memcachedqueuehandler.php; do + memcachedqueuehandler.php inboxqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index e5a181cd1..196991de0 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,8 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ + memcachedhandler inboxhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From 31b04220bd9fbadbfac01b60276a9c14896b6a45 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 09:34:56 +0000 Subject: incorrect config setting for queues --- classes/Notice.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index ff00f2a94..ebed0b8af 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,7 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - if (!common_config('queues', 'enabled')) { + if (!common_config('queue', 'enabled')) { $notice->addToInboxes(); } @@ -210,7 +210,7 @@ class Notice extends Memcached_DataObject # XXX: someone clever could prepend instead of clearing the cache if (common_config('memcached', 'enabled')) { - if (common_config('queues', 'enabled')) { + if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { $notice->blowCaches(); -- cgit v1.2.3-54-g00ecf From a9df5eab100bce97da3e028851d224ea2e9fff80 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 10:38:51 +0000 Subject: insert into user's inbox at Web time --- classes/Notice.php | 76 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 25 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index ebed0b8af..27b98de1c 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,9 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - if (!common_config('queue', 'enabled')) { + if (common_config('queue', 'enabled')) { + $notice->addToAuthorInbox(); + } else { $notice->addToInboxes(); } @@ -282,16 +284,8 @@ class Notice extends Memcached_DataObject { // Clear the user's cache $cache = common_memcache(); - if ($cache) { - $user = User::staticGet($this->profile_id); - if (!empty($user)) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); - if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); - } - } - $user->free(); - unset($user); + if (!empty($cache)) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $this->profile_id)); } $this->blowNoticeCache($blowLast); $this->blowPublicCache($blowLast); @@ -665,6 +659,33 @@ class Notice extends Memcached_DataObject return; } + function addToAuthorInbox() + { + $enabled = common_config('inboxes', 'enabled'); + + if ($enabled === true || $enabled === 'transitional') { + $user = User::staticGet('id', $this->profile_id); + if (empty($user)) { + return; + } + $inbox = new Notice_inbox(); + $UT = common_config('db','type')=='pgsql'?'"user"':'user'; + $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created) ' . + "SELECT $UT.id, " . $this->id . ", '" . $this->created . "' " . + "FROM $UT " . + "WHERE $UT.id = " . $this->profile_id . ' ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + "WHERE user_id = " . $this->profile_id . ' '. + 'AND notice_id = ' . $this->id . ' )'; + if ($enabled === 'transitional') { + $qry .= " AND $UT.inboxed = 1"; + } + $inbox->query($qry); + } + return; + } + function saveGroups() { $enabled = common_config('inboxes', 'enabled'); @@ -717,24 +738,29 @@ class Notice extends Memcached_DataObject // FIXME: do this in an offline daemon - $inbox = new Notice_inbox(); - $UT = common_config('db','type')=='pgsql'?'"user"':'user'; - $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created, source) ' . - "SELECT $UT.id, " . $this->id . ", '" . $this->created . "', 2 " . - "FROM $UT JOIN group_member ON $UT.id = group_member.profile_id " . - 'WHERE group_member.group_id = ' . $group->id . ' ' . - 'AND NOT EXISTS (SELECT user_id, notice_id ' . - 'FROM notice_inbox ' . - "WHERE user_id = $UT.id " . - 'AND notice_id = ' . $this->id . ' )'; - if ($enabled === 'transitional') { - $qry .= " AND $UT.inboxed = 1"; - } - $result = $inbox->query($qry); + $this->addToGroupInboxes($group); } } } + function addToGroupInboxes($group) + { + $inbox = new Notice_inbox(); + $UT = common_config('db','type')=='pgsql'?'"user"':'user'; + $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created, source) ' . + "SELECT $UT.id, " . $this->id . ", '" . $this->created . "', 2 " . + "FROM $UT JOIN group_member ON $UT.id = group_member.profile_id " . + 'WHERE group_member.group_id = ' . $group->id . ' ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + "WHERE user_id = $UT.id " . + 'AND notice_id = ' . $this->id . ' )'; + if ($enabled === 'transitional') { + $qry .= " AND $UT.inboxed = 1"; + } + $result = $inbox->query($qry); + } + function saveReplies() { // Alternative reply format -- cgit v1.2.3-54-g00ecf From 84072aa5cf6124d59a06a7f0a7945c00ee2836da Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 12:13:49 -0400 Subject: run 'set names' after each connection to deal with UTF8 correctly --- classes/Memcached_DataObject.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'classes') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 5f71f716b..877bbf2e0 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,4 +227,20 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } + + // We overload so that 'SET NAMES "utf8"' is called for + // each connection + + function _connect() + { + global $_DB_DATAOBJECT; + $exists = !empty($this->_database_dsn_md5) && + isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); + $result = parent::_connect(); + if (!$exists) { + $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; + $DB->query('SET NAMES "utf8"'); + } + return $result; + } } -- cgit v1.2.3-54-g00ecf From 068f6801cc59488bfc50ef399a2a4d22b1b7e9c2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 12:27:32 -0400 Subject: Revert "run 'set names' after each connection to deal with UTF8 correctly" This reverts commit 84072aa5cf6124d59a06a7f0a7945c00ee2836da. This commit caused grievous harm to old notices on identi.ca. Reverting until we figure out how to convert the old notices. --- classes/Memcached_DataObject.php | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'classes') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 877bbf2e0..5f71f716b 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,20 +227,4 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } - - // We overload so that 'SET NAMES "utf8"' is called for - // each connection - - function _connect() - { - global $_DB_DATAOBJECT; - $exists = !empty($this->_database_dsn_md5) && - isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); - $result = parent::_connect(); - if (!$exists) { - $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; - $DB->query('SET NAMES "utf8"'); - } - return $result; - } } -- cgit v1.2.3-54-g00ecf From c7105c2af1ebe3cddd477265c6fea59a61d0c7e5 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:07:05 -0400 Subject: Change to avoid a join in notice inbox The join in notice_inbox is causing temp-table sorting on identi.ca, so I'm trying a finer-tuned approach. --- classes/Notice.php | 31 ++++++++++++--- classes/Notice_inbox.php | 101 ++++++++++++++++++++++++++++++++++++++++++++++- classes/User.php | 29 +------------- 3 files changed, 127 insertions(+), 34 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 27b98de1c..b087c94bc 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,6 +215,7 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { + common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -285,7 +286,7 @@ class Notice extends Memcached_DataObject // Clear the user's cache $cache = common_memcache(); if (!empty($cache)) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $this->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$this->profile_id)); } $this->blowNoticeCache($blowLast); $this->blowPublicCache($blowLast); @@ -307,9 +308,9 @@ class Notice extends Memcached_DataObject $member->group_id = $group_inbox->group_id; if ($member->find()) { while ($member->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id . ';last')); } } } @@ -352,9 +353,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); } } $user->free(); @@ -613,6 +614,26 @@ class Notice extends Memcached_DataObject return $wrapper; } + function getStreamByIds($ids) + { + $cache = common_memcache(); + + if (!empty($cache)) { + $notices = array(); + foreach ($ids as $id) { + $notices[] = Notice::staticGet('id', $id); + } + return new ArrayWrapper($notices); + } else { + $notice = new Notice(); + $notice->whereAdd('id in (' . implode(', ', $ids) . ')'); + $notice->orderBy('id DESC'); + + $notice->find(); + return $notice; + } + } + function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 81ddb4538..162da74fe 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -1,7 +1,7 @@ INBOX_CACHE_WINDOW) { + common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); + return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); + } + + $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + common_debug('Cache hit for notice_inbox.'); + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + common_debug('Cache hit for notice_inbox on last item.'); + + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + $last_id, null, null); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + null, null, null); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } + + function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) + { + $inbox = new Notice_inbox(); + + $inbox->user_id = $user_id; + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $inbox->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; + } } diff --git a/classes/User.php b/classes/User.php index 098381f73..ce7ea1464 100644 --- a/classes/User.php +++ b/classes/User.php @@ -451,34 +451,9 @@ class User extends Memcached_DataObject } else if ($enabled === true || ($enabled == 'transitional' && $this->inboxed == 1)) { - $cache = common_memcache(); + $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - if (!empty($cache)) { - - # Get the notices out of the cache - - $notices = $cache->get(common_cache_key($cachekey)); - - # On a cache hit, return a DB-object-like wrapper - - if ($notices !== false) { - $wrapper = new ArrayWrapper(array_slice($notices, $offset, $limit)); - return $wrapper; - } - } - - $inbox = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - - $ids = array(); - - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - - $inbox->free(); - unset($inbox); - - return Notice::getStreamByIds($ids, 'user:notices_with_friends:' . $this->id); + return Notice::getStreamByIds($ids); } } -- cgit v1.2.3-54-g00ecf From fe53e780be5db4ceb2831a1d69faec6130a10deb Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:31:56 -0400 Subject: Remove some debug comments in query-by-id --- classes/Notice.php | 5 ++--- classes/Notice_inbox.php | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index b087c94bc..49d0939c1 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,7 +215,6 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { - common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -353,9 +352,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id)); if ($blowLast) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id.';last')); } } $user->free(); diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 162da74fe..f32137038 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -50,7 +50,6 @@ class Notice_inbox extends Memcached_DataObject if (empty($cache) || $since_id != 0 || $before_id != 0 || !is_null($since) || ($offset + $limit) > INBOX_CACHE_WINDOW) { - common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); } @@ -60,7 +59,6 @@ class Notice_inbox extends Memcached_DataObject if (!empty($idstr)) { // Cache hit! Woohoo! - common_debug('Cache hit for notice_inbox.'); $window = explode(',', $idstr); $ids = array_slice($window, $offset, $limit); return $ids; @@ -69,8 +67,6 @@ class Notice_inbox extends Memcached_DataObject $laststr = common_cache_key($idkey.';last'); if (!empty($laststr)) { - common_debug('Cache hit for notice_inbox on last item.'); - $window = explode(',', $laststr); $last_id = $window[0]; $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, -- cgit v1.2.3-54-g00ecf From 10ef8a2f7112a83188e9702d480abd3c6062c26c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 11:27:45 -0400 Subject: Move algorithm for caching to Notice class Moved the algorithm for notice stream caching to the Notice class. --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++++++ classes/Notice_inbox.php | 56 +++++------------------------------------------- 2 files changed, 60 insertions(+), 51 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 49d0939c1..faabb1e14 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -967,4 +967,59 @@ class Notice extends Memcached_DataObject array('notice' => $this->id)); } } + + function stream($fn, $args, $cachekey, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $cache = common_memcache(); + + if (empty($cache) || + $since_id != 0 || $before_id != 0 || !is_null($since) || + ($offset + $limit) > NOTICE_CACHE_WINDOW) { + return call_user_func_array($fn, array_merge($args, array($offset, $limit, $since_id, + $before_id, $since))); + } + + $idkey = common_cache_key($cachekey); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + $last_id, 0, null))); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + 0, 0, null))); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } } diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index f32137038..dec14b0d1 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -43,58 +43,12 @@ class Notice_inbox extends Memcached_DataObject /* the code above is auto generated do not remove the tag below */ ###END_AUTOCODE - function stream($user_id, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + function stream($user_id, $offset, $limit, $since_id, $before_id, $since) { - $cache = common_memcache(); - - if (empty($cache) || - $since_id != 0 || $before_id != 0 || !is_null($since) || - ($offset + $limit) > INBOX_CACHE_WINDOW) { - return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); - } - - $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); - - $idstr = $cache->get($idkey); - - if (!empty($idstr)) { - // Cache hit! Woohoo! - $window = explode(',', $idstr); - $ids = array_slice($window, $offset, $limit); - return $ids; - } - - $laststr = common_cache_key($idkey.';last'); - - if (!empty($laststr)) { - $window = explode(',', $laststr); - $last_id = $window[0]; - $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - $last_id, null, null); - - $new_window = array_merge($new_ids, $window); - - $new_windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $new_windowstr); - $result = $cache->set($idkey . ';last', $new_windowstr); - - $ids = array_slice($new_window, $offset, $limit); - - return $ids; - } - - $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - null, null, null); - - $windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $windowstr); - $result = $cache->set($idkey . ';last', $windowstr); - - $ids = array_slice($window, $offset, $limit); - - return $ids; + return Notice::stream(array('Notice_inbox', '_streamDirect'), + array($user_id), + 'notice_inbox:by_user:'.$user_id, + $offset, $limit, $since_id, $before_id, $since); } function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) -- cgit v1.2.3-54-g00ecf From a4d959b8a2254d173bdf45f418f6add9f6f62cda Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 12:05:31 -0400 Subject: Public stream uses IDs method Public stream now uses IDs method --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index faabb1e14..6a7522bd3 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -635,25 +635,58 @@ class Notice extends Memcached_DataObject function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { + $ids = Notice::stream(array('Notice', '_publicStreamDirect'), + array(), + 'public', + $offset, $limit, $since_id, $before_id, $since); - $parts = array(); + return Notice::getStreamByIds($ids); + } + + function _publicStreamDirect($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $notice = new Notice(); + + $notice->selectAdd(); // clears it + $notice->selectAdd('id'); - $qry = 'SELECT * FROM notice '; + $notice->orderBy('id DESC'); + + if (!is_null($offset)) { + $notice->limit($offset, $limit); + } if (common_config('public', 'localonly')) { - $parts[] = 'is_local = 1'; + $notice->whereAdd('is_local = 1'); } else { # -1 == blacklisted - $parts[] = 'is_local != -1'; + $notice->whereAdd('is_local != -1'); } - if ($parts) { - $qry .= ' WHERE ' . implode(' AND ', $parts); + if ($since_id != 0) { + $notice->whereAdd('id > ' . $since_id); } - return Notice::getStream($qry, - 'public', - $offset, $limit, $since_id, $before_id, null, $since); + if ($before_id != 0) { + $notice->whereAdd('id < ' . $before_id); + } + + if (!is_null($since)) { + $notice->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $ids = array(); + + if ($notice->find()) { + while ($notice->fetch()) { + $ids[] = $notice->id; + } + } + + $notice->free(); + $notice = NULL; + + return $ids; } function addToInboxes() @@ -990,7 +1023,7 @@ class Notice extends Memcached_DataObject return $ids; } - $laststr = common_cache_key($idkey.';last'); + $laststr = $cache->get($idkey.';last'); if (!empty($laststr)) { $window = explode(',', $laststr); @@ -1013,7 +1046,7 @@ class Notice extends Memcached_DataObject $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, 0, 0, null))); - $windowstr = implode(',', $new_window); + $windowstr = implode(',', $window); $result = $cache->set($idkey, $windowstr); $result = $cache->set($idkey . ';last', $windowstr); -- cgit v1.2.3-54-g00ecf