From 640628de2d593933e810b4785dfe38923b979713 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:03:19 -0400 Subject: A queuehandler for blowing caches offline We add a queuehandler for blowing the memcached caches off-line. This should speed up the processing of new notices. --- classes/Notice.php | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index 5fa0d79a1..fbfeb9489 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -207,7 +207,11 @@ class Notice extends Memcached_DataObject # XXX: someone clever could prepend instead of clearing the cache if (common_config('memcached', 'enabled')) { - $notice->blowCaches(); + if (common_config('queues', 'enabled')) { + $notice->blowAuthorCaches(); + } else { + $notice->blowCaches(); + } } return $notice; @@ -271,6 +275,25 @@ class Notice extends Memcached_DataObject $this->blowGroupCache($blowLast); } + function blowAuthorCaches($blowLast=false) + { + // Clear the user's cache + $cache = common_memcache(); + if ($cache) { + $user = User::staticGet($this->profile_id); + if (!empty($user)) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + if ($blowLast) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + } + } + $user->free(); + unset($user); + } + $this->blowNoticeCache($blowLast); + $this->blowPublicCache($blowLast); + } + function blowGroupCache($blowLast=false) { $cache = common_memcache(); -- cgit v1.2.3-54-g00ecf From aee45ea91dcec4736d8b9befe17e030b873d9226 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:08:48 -0400 Subject: Add an inbox queue handler Handle distributing a notice to multiple inboxes in a queue handler rather than in the Web action. --- classes/Notice.php | 5 +++- lib/util.php | 4 +++ scripts/inboxqueuehandler.php | 57 +++++++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 2 +- scripts/stopdaemons.sh | 3 ++- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 scripts/inboxqueuehandler.php (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index fbfeb9489..ff00f2a94 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,10 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - $notice->addToInboxes(); + if (!common_config('queues', 'enabled')) { + $notice->addToInboxes(); + } + $notice->query('COMMIT'); Event::handle('EndNoticeSave', array($notice)); diff --git a/lib/util.php b/lib/util.php index 12797891c..9b6d2941a 100644 --- a/lib/util.php +++ b/lib/util.php @@ -889,6 +889,10 @@ function common_enqueue_notice($notice) $transports[] = 'memcached'; } + if (common_config('queues', 'enabled')) { + $transports[] = 'inbox'; + } + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php new file mode 100644 index 000000000..16e334b83 --- /dev/null +++ b/scripts/inboxqueuehandler.php @@ -0,0 +1,57 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class InboxQueueHandler extends QueueHandler +{ + function transport() + { + return 'inbox'; + } + + function handle_notice($notice) + { + common_log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); + $notice->addToInboxes(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new InboxQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index 08de6d954..66f9ed4e0 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -25,7 +25,7 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ - memcachedqueuehandler.php; do + memcachedqueuehandler.php inboxqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index e5a181cd1..196991de0 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,8 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ + memcachedhandler inboxhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From 31b04220bd9fbadbfac01b60276a9c14896b6a45 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 09:34:56 +0000 Subject: incorrect config setting for queues --- classes/Notice.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index ff00f2a94..ebed0b8af 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,7 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - if (!common_config('queues', 'enabled')) { + if (!common_config('queue', 'enabled')) { $notice->addToInboxes(); } @@ -210,7 +210,7 @@ class Notice extends Memcached_DataObject # XXX: someone clever could prepend instead of clearing the cache if (common_config('memcached', 'enabled')) { - if (common_config('queues', 'enabled')) { + if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { $notice->blowCaches(); -- cgit v1.2.3-54-g00ecf From a9df5eab100bce97da3e028851d224ea2e9fff80 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 10:38:51 +0000 Subject: insert into user's inbox at Web time --- classes/Notice.php | 76 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 25 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index ebed0b8af..27b98de1c 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,9 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - if (!common_config('queue', 'enabled')) { + if (common_config('queue', 'enabled')) { + $notice->addToAuthorInbox(); + } else { $notice->addToInboxes(); } @@ -282,16 +284,8 @@ class Notice extends Memcached_DataObject { // Clear the user's cache $cache = common_memcache(); - if ($cache) { - $user = User::staticGet($this->profile_id); - if (!empty($user)) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); - if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); - } - } - $user->free(); - unset($user); + if (!empty($cache)) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $this->profile_id)); } $this->blowNoticeCache($blowLast); $this->blowPublicCache($blowLast); @@ -665,6 +659,33 @@ class Notice extends Memcached_DataObject return; } + function addToAuthorInbox() + { + $enabled = common_config('inboxes', 'enabled'); + + if ($enabled === true || $enabled === 'transitional') { + $user = User::staticGet('id', $this->profile_id); + if (empty($user)) { + return; + } + $inbox = new Notice_inbox(); + $UT = common_config('db','type')=='pgsql'?'"user"':'user'; + $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created) ' . + "SELECT $UT.id, " . $this->id . ", '" . $this->created . "' " . + "FROM $UT " . + "WHERE $UT.id = " . $this->profile_id . ' ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + "WHERE user_id = " . $this->profile_id . ' '. + 'AND notice_id = ' . $this->id . ' )'; + if ($enabled === 'transitional') { + $qry .= " AND $UT.inboxed = 1"; + } + $inbox->query($qry); + } + return; + } + function saveGroups() { $enabled = common_config('inboxes', 'enabled'); @@ -717,24 +738,29 @@ class Notice extends Memcached_DataObject // FIXME: do this in an offline daemon - $inbox = new Notice_inbox(); - $UT = common_config('db','type')=='pgsql'?'"user"':'user'; - $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created, source) ' . - "SELECT $UT.id, " . $this->id . ", '" . $this->created . "', 2 " . - "FROM $UT JOIN group_member ON $UT.id = group_member.profile_id " . - 'WHERE group_member.group_id = ' . $group->id . ' ' . - 'AND NOT EXISTS (SELECT user_id, notice_id ' . - 'FROM notice_inbox ' . - "WHERE user_id = $UT.id " . - 'AND notice_id = ' . $this->id . ' )'; - if ($enabled === 'transitional') { - $qry .= " AND $UT.inboxed = 1"; - } - $result = $inbox->query($qry); + $this->addToGroupInboxes($group); } } } + function addToGroupInboxes($group) + { + $inbox = new Notice_inbox(); + $UT = common_config('db','type')=='pgsql'?'"user"':'user'; + $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created, source) ' . + "SELECT $UT.id, " . $this->id . ", '" . $this->created . "', 2 " . + "FROM $UT JOIN group_member ON $UT.id = group_member.profile_id " . + 'WHERE group_member.group_id = ' . $group->id . ' ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + "WHERE user_id = $UT.id " . + 'AND notice_id = ' . $this->id . ' )'; + if ($enabled === 'transitional') { + $qry .= " AND $UT.inboxed = 1"; + } + $result = $inbox->query($qry); + } + function saveReplies() { // Alternative reply format -- cgit v1.2.3-54-g00ecf From c7105c2af1ebe3cddd477265c6fea59a61d0c7e5 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:07:05 -0400 Subject: Change to avoid a join in notice inbox The join in notice_inbox is causing temp-table sorting on identi.ca, so I'm trying a finer-tuned approach. --- classes/Notice.php | 31 ++++++++++++--- classes/Notice_inbox.php | 101 ++++++++++++++++++++++++++++++++++++++++++++++- classes/User.php | 29 +------------- 3 files changed, 127 insertions(+), 34 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index 27b98de1c..b087c94bc 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,6 +215,7 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { + common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -285,7 +286,7 @@ class Notice extends Memcached_DataObject // Clear the user's cache $cache = common_memcache(); if (!empty($cache)) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $this->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$this->profile_id)); } $this->blowNoticeCache($blowLast); $this->blowPublicCache($blowLast); @@ -307,9 +308,9 @@ class Notice extends Memcached_DataObject $member->group_id = $group_inbox->group_id; if ($member->find()) { while ($member->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id . ';last')); } } } @@ -352,9 +353,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); } } $user->free(); @@ -613,6 +614,26 @@ class Notice extends Memcached_DataObject return $wrapper; } + function getStreamByIds($ids) + { + $cache = common_memcache(); + + if (!empty($cache)) { + $notices = array(); + foreach ($ids as $id) { + $notices[] = Notice::staticGet('id', $id); + } + return new ArrayWrapper($notices); + } else { + $notice = new Notice(); + $notice->whereAdd('id in (' . implode(', ', $ids) . ')'); + $notice->orderBy('id DESC'); + + $notice->find(); + return $notice; + } + } + function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 81ddb4538..162da74fe 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -1,7 +1,7 @@ INBOX_CACHE_WINDOW) { + common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); + return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); + } + + $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + common_debug('Cache hit for notice_inbox.'); + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + common_debug('Cache hit for notice_inbox on last item.'); + + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + $last_id, null, null); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + null, null, null); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } + + function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) + { + $inbox = new Notice_inbox(); + + $inbox->user_id = $user_id; + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $inbox->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; + } } diff --git a/classes/User.php b/classes/User.php index 098381f73..ce7ea1464 100644 --- a/classes/User.php +++ b/classes/User.php @@ -451,34 +451,9 @@ class User extends Memcached_DataObject } else if ($enabled === true || ($enabled == 'transitional' && $this->inboxed == 1)) { - $cache = common_memcache(); + $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - if (!empty($cache)) { - - # Get the notices out of the cache - - $notices = $cache->get(common_cache_key($cachekey)); - - # On a cache hit, return a DB-object-like wrapper - - if ($notices !== false) { - $wrapper = new ArrayWrapper(array_slice($notices, $offset, $limit)); - return $wrapper; - } - } - - $inbox = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - - $ids = array(); - - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - - $inbox->free(); - unset($inbox); - - return Notice::getStreamByIds($ids, 'user:notices_with_friends:' . $this->id); + return Notice::getStreamByIds($ids); } } -- cgit v1.2.3-54-g00ecf From fe53e780be5db4ceb2831a1d69faec6130a10deb Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:31:56 -0400 Subject: Remove some debug comments in query-by-id --- classes/Notice.php | 5 ++--- classes/Notice_inbox.php | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index b087c94bc..49d0939c1 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,7 +215,6 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { - common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -353,9 +352,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id)); if ($blowLast) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id.';last')); } } $user->free(); diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 162da74fe..f32137038 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -50,7 +50,6 @@ class Notice_inbox extends Memcached_DataObject if (empty($cache) || $since_id != 0 || $before_id != 0 || !is_null($since) || ($offset + $limit) > INBOX_CACHE_WINDOW) { - common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); } @@ -60,7 +59,6 @@ class Notice_inbox extends Memcached_DataObject if (!empty($idstr)) { // Cache hit! Woohoo! - common_debug('Cache hit for notice_inbox.'); $window = explode(',', $idstr); $ids = array_slice($window, $offset, $limit); return $ids; @@ -69,8 +67,6 @@ class Notice_inbox extends Memcached_DataObject $laststr = common_cache_key($idkey.';last'); if (!empty($laststr)) { - common_debug('Cache hit for notice_inbox on last item.'); - $window = explode(',', $laststr); $last_id = $window[0]; $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, -- cgit v1.2.3-54-g00ecf From 10ef8a2f7112a83188e9702d480abd3c6062c26c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 11:27:45 -0400 Subject: Move algorithm for caching to Notice class Moved the algorithm for notice stream caching to the Notice class. --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++++++ classes/Notice_inbox.php | 56 +++++------------------------------------------- 2 files changed, 60 insertions(+), 51 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index 49d0939c1..faabb1e14 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -967,4 +967,59 @@ class Notice extends Memcached_DataObject array('notice' => $this->id)); } } + + function stream($fn, $args, $cachekey, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $cache = common_memcache(); + + if (empty($cache) || + $since_id != 0 || $before_id != 0 || !is_null($since) || + ($offset + $limit) > NOTICE_CACHE_WINDOW) { + return call_user_func_array($fn, array_merge($args, array($offset, $limit, $since_id, + $before_id, $since))); + } + + $idkey = common_cache_key($cachekey); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + $last_id, 0, null))); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + 0, 0, null))); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } } diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index f32137038..dec14b0d1 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -43,58 +43,12 @@ class Notice_inbox extends Memcached_DataObject /* the code above is auto generated do not remove the tag below */ ###END_AUTOCODE - function stream($user_id, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + function stream($user_id, $offset, $limit, $since_id, $before_id, $since) { - $cache = common_memcache(); - - if (empty($cache) || - $since_id != 0 || $before_id != 0 || !is_null($since) || - ($offset + $limit) > INBOX_CACHE_WINDOW) { - return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); - } - - $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); - - $idstr = $cache->get($idkey); - - if (!empty($idstr)) { - // Cache hit! Woohoo! - $window = explode(',', $idstr); - $ids = array_slice($window, $offset, $limit); - return $ids; - } - - $laststr = common_cache_key($idkey.';last'); - - if (!empty($laststr)) { - $window = explode(',', $laststr); - $last_id = $window[0]; - $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - $last_id, null, null); - - $new_window = array_merge($new_ids, $window); - - $new_windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $new_windowstr); - $result = $cache->set($idkey . ';last', $new_windowstr); - - $ids = array_slice($new_window, $offset, $limit); - - return $ids; - } - - $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - null, null, null); - - $windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $windowstr); - $result = $cache->set($idkey . ';last', $windowstr); - - $ids = array_slice($window, $offset, $limit); - - return $ids; + return Notice::stream(array('Notice_inbox', '_streamDirect'), + array($user_id), + 'notice_inbox:by_user:'.$user_id, + $offset, $limit, $since_id, $before_id, $since); } function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) -- cgit v1.2.3-54-g00ecf From a4d959b8a2254d173bdf45f418f6add9f6f62cda Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 12:05:31 -0400 Subject: Public stream uses IDs method Public stream now uses IDs method --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index faabb1e14..6a7522bd3 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -635,25 +635,58 @@ class Notice extends Memcached_DataObject function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { + $ids = Notice::stream(array('Notice', '_publicStreamDirect'), + array(), + 'public', + $offset, $limit, $since_id, $before_id, $since); - $parts = array(); + return Notice::getStreamByIds($ids); + } + + function _publicStreamDirect($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $notice = new Notice(); + + $notice->selectAdd(); // clears it + $notice->selectAdd('id'); - $qry = 'SELECT * FROM notice '; + $notice->orderBy('id DESC'); + + if (!is_null($offset)) { + $notice->limit($offset, $limit); + } if (common_config('public', 'localonly')) { - $parts[] = 'is_local = 1'; + $notice->whereAdd('is_local = 1'); } else { # -1 == blacklisted - $parts[] = 'is_local != -1'; + $notice->whereAdd('is_local != -1'); } - if ($parts) { - $qry .= ' WHERE ' . implode(' AND ', $parts); + if ($since_id != 0) { + $notice->whereAdd('id > ' . $since_id); } - return Notice::getStream($qry, - 'public', - $offset, $limit, $since_id, $before_id, null, $since); + if ($before_id != 0) { + $notice->whereAdd('id < ' . $before_id); + } + + if (!is_null($since)) { + $notice->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $ids = array(); + + if ($notice->find()) { + while ($notice->fetch()) { + $ids[] = $notice->id; + } + } + + $notice->free(); + $notice = NULL; + + return $ids; } function addToInboxes() @@ -990,7 +1023,7 @@ class Notice extends Memcached_DataObject return $ids; } - $laststr = common_cache_key($idkey.';last'); + $laststr = $cache->get($idkey.';last'); if (!empty($laststr)) { $window = explode(',', $laststr); @@ -1013,7 +1046,7 @@ class Notice extends Memcached_DataObject $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, 0, 0, null))); - $windowstr = implode(',', $new_window); + $windowstr = implode(',', $window); $result = $cache->set($idkey, $windowstr); $result = $cache->set($idkey . ';last', $windowstr); -- cgit v1.2.3-54-g00ecf From 1e8ea1eb460b163176c4d7d1e7dffa500024ef91 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 16:09:03 -0400 Subject: Make the tag stream use ID mechanism --- classes/Notice.php | 5 +---- classes/Notice_tag.php | 61 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 14 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index 6a7522bd3..2bb466155 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -328,10 +328,7 @@ class Notice extends Memcached_DataObject $tag->notice_id = $this->id; if ($tag->find()) { while ($tag->fetch()) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $tag->tag)); - if ($blowLast) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $tag->tag . ';last')); - } + $tag->blowCache($blowLast); } } $tag->free(); diff --git a/classes/Notice_tag.php b/classes/Notice_tag.php index f2247299a..e5b772243 100644 --- a/classes/Notice_tag.php +++ b/classes/Notice_tag.php @@ -37,21 +37,62 @@ class Notice_tag extends Memcached_DataObject ###END_AUTOCODE static function getStream($tag, $offset=0, $limit=20) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN notice_tag ON notice.id = notice_tag.notice_id ' . - "WHERE notice_tag.tag = '%s' "; - - return Notice::getStream(sprintf($qry, $tag), - 'notice_tag:notice_stream:' . common_keyize($tag), - $offset, $limit); + + $ids = Notice::stream(array('Notice_tag', '_streamDirect'), + array($tag), + 'notice_tag:notice_ids:' . common_keyize($tag), + $offset, $limit); + + return Notice::getStreamByIds($ids); + } + + function _streamDirect($tag, $offset, $limit, $since_id, $before_id, $since) + { + $nt = new Notice_tag(); + + $nt->tag = $tag; + + $nt->selectAdd(); + $nt->selectAdd('notice_id'); + + if ($since_id != 0) { + $nt->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $nt->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $nt->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $nt->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $nt->limit($offset, $limit); + } + + $ids = array(); + + if ($nt->find()) { + while ($nt->fetch()) { + $ids[] = $nt->notice_id; + } + } + + return $ids; } - function blowCache() + function blowCache($blowLast=false) { $cache = common_memcache(); if ($cache) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $this->tag)); + $idkey = common_cache_key('notice_tag:notice_ids:' . common_keyize($this->tag)); + $cache->delete($idkey); + if ($blowLast) { + $cache->delete($idkey.';last'); + } } } -- cgit v1.2.3-54-g00ecf From aee641ee1e311fb0af0f9f6d75ca7fae2c7d8477 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 20:45:33 -0400 Subject: make replies use new query format --- classes/Notice.php | 4 ++-- classes/Reply.php | 47 +++++++++++++++++++++++++++++++++++++++++++++-- classes/User.php | 10 +++------- 3 files changed, 50 insertions(+), 11 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index 2bb466155..808631f4d 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -380,9 +380,9 @@ class Notice extends Memcached_DataObject $reply->notice_id = $this->id; if ($reply->find()) { while ($reply->fetch()) { - $cache->delete(common_cache_key('user:replies:'.$reply->profile_id)); + $cache->delete(common_cache_key('reply:stream:'.$reply->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:replies:'.$reply->profile_id.';last')); + $cache->delete(common_cache_key('reply:stream:'.$reply->profile_id.';last')); } } } diff --git a/classes/Reply.php b/classes/Reply.php index af86aaf87..4439053b4 100644 --- a/classes/Reply.php +++ b/classes/Reply.php @@ -4,7 +4,7 @@ */ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; -class Reply extends Memcached_DataObject +class Reply extends Memcached_DataObject { ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -13,7 +13,7 @@ class Reply extends Memcached_DataObject public $notice_id; // int(4) primary_key not_null public $profile_id; // int(4) primary_key not_null public $modified; // timestamp() not_null default_CURRENT_TIMESTAMP - public $replied_id; // int(4) + public $replied_id; // int(4) /* Static get */ function staticGet($k,$v=null) @@ -21,4 +21,47 @@ class Reply extends Memcached_DataObject /* the code above is auto generated do not remove the tag below */ ###END_AUTOCODE + + function stream($user_id, $offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) + { + $ids = Notice::stream(array('Reply', '_streamDirect'), + array($user_id), + 'reply:stream:' . $user_id, + $offset, $limit, $since_id, $before_id, $since); + return $ids; + } + + function _streamDirect($user_id, $offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) + { + $reply = new Reply(); + $reply->profile_id = $user_id; + + if ($since_id != 0) { + $reply->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $reply->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $reply->whereAdd('modified > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $reply->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $reply->limit($offset, $limit); + } + + $ids = array(); + + if ($reply->find()) { + while ($reply->fetch()) { + $ids[] = $reply->notice_id; + } + } + + return $ids; + } } diff --git a/classes/User.php b/classes/User.php index ce7ea1464..b76e45c33 100644 --- a/classes/User.php +++ b/classes/User.php @@ -401,13 +401,9 @@ class User extends Memcached_DataObject function getReplies($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN reply ON notice.id = reply.notice_id ' . - 'WHERE reply.profile_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'user:replies:'.$this->id, - $offset, $limit, $since_id, $before_id, null, $since); + $ids = Reply::stream($this->id, $offset, $limit, $since_id, $before_id, $since); + common_debug("Ids = " . implode(',', $ids)); + return Notice::getStreamByIds($ids); } function getNotices($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) -- cgit v1.2.3-54-g00ecf From 3328ec545c36fc2408cdb9d048effe24feafe218 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 11:27:57 -0700 Subject: make profile notice getting use ids --- classes/Notice.php | 6 +++--- classes/Profile.php | 53 +++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 11 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index 808631f4d..eceed325b 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -363,10 +363,10 @@ class Notice extends Memcached_DataObject { if ($this->is_local) { $cache = common_memcache(); - if ($cache) { - $cache->delete(common_cache_key('profile:notices:'.$this->profile_id)); + if (!empty($cache)) { + $cache->delete(common_cache_key('profile:notice_ids:'.$this->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('profile:notices:'.$this->profile_id.';last')); + $cache->delete(common_cache_key('profile:notice_ids:'.$this->profile_id.';last')); } } } diff --git a/classes/Profile.php b/classes/Profile.php index f3bfe299c..ae5641d79 100644 --- a/classes/Profile.php +++ b/classes/Profile.php @@ -155,14 +155,51 @@ class Profile extends Memcached_DataObject function getNotices($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0) { - $qry = - 'SELECT * ' . - 'FROM notice ' . - 'WHERE profile_id = %d '; - - return Notice::getStream(sprintf($qry, $this->id), - 'profile:notices:'.$this->id, - $offset, $limit, $since_id, $before_id); + // XXX: I'm not sure this is going to be any faster. It probably isn't. + $ids = Notice::stream(array($this, '_streamDirect'), + array(), + 'profile:notice_ids:' . $this->id, + $offset, $limit, $since_id, $before_id); + + return Notice::getStreamByIds($ids); + } + + function _streamDirect($offset, $limit, $since_id, $before_id, $since) + { + $notice = new Notice(); + + $notice->profile_id = $this->id; + + $notice->selectAdd(); + $notice->selectAdd('id'); + + if ($since_id != 0) { + $notice->whereAdd('id > ' . $since_id); + } + + if ($before_id != 0) { + $notice->whereAdd('id < ' . $before_id); + } + + if (!is_null($since)) { + $notice->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $notice->orderBy('id DESC'); + + if (!is_null($offset)) { + $notice->limit($offset, $limit); + } + + $ids = array(); + + if ($notice->find()) { + while ($notice->fetch()) { + $ids[] = $notice->id; + } + } + + return $ids; } function isMember($group) -- cgit v1.2.3-54-g00ecf From 021b520a11d3449a1476182e1ad117582999d364 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 11:38:50 -0700 Subject: Make user group stream use IDs --- classes/Notice.php | 4 ++-- classes/User_group.php | 53 ++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 10 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Notice.php b/classes/Notice.php index eceed325b..c036e6e9e 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -299,9 +299,9 @@ class Notice extends Memcached_DataObject $group_inbox->notice_id = $this->id; if ($group_inbox->find()) { while ($group_inbox->fetch()) { - $cache->delete(common_cache_key('group:notices:'.$group_inbox->group_id)); + $cache->delete(common_cache_key('user_group:notice_ids:' . $group_inbox->group_id)); if ($blowLast) { - $cache->delete(common_cache_key('group:notices:'.$group_inbox->group_id.';last')); + $cache->delete(common_cache_key('user_group:notice_ids:' . $group_inbox->group_id.';last')); } $member = new Group_member(); $member->group_id = $group_inbox->group_id; diff --git a/classes/User_group.php b/classes/User_group.php index d152f9d56..7cc31e702 100755 --- a/classes/User_group.php +++ b/classes/User_group.php @@ -50,13 +50,50 @@ class User_group extends Memcached_DataObject function getNotices($offset, $limit) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN group_inbox ON notice.id = group_inbox.notice_id ' . - 'WHERE group_inbox.group_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'group:notices:'.$this->id, - $offset, $limit); + $ids = Notice::stream(array($this, '_streamDirect'), + array(), + 'user_group:notice_ids:' . $this->id, + $offset, $limit); + + return Notice::getStreamByIds($ids); + } + + function _streamDirect($offset, $limit, $since_id, $before_id, $since) + { + $inbox = new Group_inbox(); + + $inbox->group_id = $this->id; + + $inbox->selectAdd(); + $inbox->selectAdd('notice_id'); + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $inbox->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; } function allowedNickname($nickname) @@ -91,7 +128,7 @@ class User_group extends Memcached_DataObject function setOriginal($filename) { $imagefile = new ImageFile($this->id, Avatar::path($filename)); - + $orig = clone($this); $this->original_logo = Avatar::url($filename); $this->homepage_logo = Avatar::url($imagefile->resize(AVATAR_PROFILE_SIZE)); -- cgit v1.2.3-54-g00ecf From 5314d9b2cfaef3f2fd0ead262e18d1776fd99c8d Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 12:01:28 -0700 Subject: make faves work with ids --- classes/Fave.php | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- classes/Notice.php | 4 ++-- classes/User.php | 38 +++++++++++++++++--------------------- 3 files changed, 70 insertions(+), 25 deletions(-) (limited to 'classes/Notice.php') diff --git a/classes/Fave.php b/classes/Fave.php index 24df5938c..915b4572f 100644 --- a/classes/Fave.php +++ b/classes/Fave.php @@ -4,7 +4,7 @@ */ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; -class Fave extends Memcached_DataObject +class Fave extends Memcached_DataObject { ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -31,9 +31,58 @@ class Fave extends Memcached_DataObject } return $fave; } - + function &pkeyGet($kv) { return Memcached_DataObject::pkeyGet('Fave', $kv); } + + function stream($user_id, $offset=0, $limit=NOTICES_PER_PAGE) + { + $ids = Notice::stream(array('Fave', '_streamDirect'), + array($user_id), + 'fave:ids_by_user:'.$user_id, + $offset, $limit); + return $ids; + } + + function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) + { + $fav = new Fave(); + + $fav->user_id = $user_id; + + $fav->selectAdd(); + $fav->selectAdd('notice_id'); + + if ($since_id != 0) { + $fav->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $fav->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $fav->whereAdd('modified > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + // NOTE: we sort by fave time, not by notice time! + + $fav->orderBy('modified DESC'); + + if (!is_null($offset)) { + $fav->limit($offset, $limit); + } + + $ids = array(); + + if ($fav->find()) { + while ($fav->fetch()) { + $ids[] = $fav->notice_id; + } + } + + return $ids; + } } diff --git a/classes/Notice.php b/classes/Notice.php index c036e6e9e..771a4e715 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -412,9 +412,9 @@ class Notice extends Memcached_DataObject $fave->notice_id = $this->id; if ($fave->find()) { while ($fave->fetch()) { - $cache->delete(common_cache_key('user:faves:'.$fave->user_id)); + $cache->delete(common_cache_key('fave:ids_by_user:'.$fave->user_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:faves:'.$fave->user_id.';last')); + $cache->delete(common_cache_key('fave:ids_by_user:'.$fave->user_id.';last')); } } } diff --git a/classes/User.php b/classes/User.php index b76e45c33..b5ac7b220 100644 --- a/classes/User.php +++ b/classes/User.php @@ -349,30 +349,31 @@ class User extends Memcached_DataObject $cache = common_memcache(); // XXX: Kind of a hack. + if ($cache) { // This is the stream of favorite notices, in rev chron // order. This forces it into cache. - $faves = $this->favoriteNotices(0, NOTICE_CACHE_WINDOW); - $cnt = 0; - while ($faves->fetch()) { - if ($faves->id < $notice->id) { - // If we passed it, it's not a fave - return false; - } else if ($faves->id == $notice->id) { - // If it matches a cached notice, then it's a fave - return true; - } - $cnt++; + + $ids = Fave::stream($this->id, 0, NOTICE_CACHE_WINDOW); + + // If it's in the list, then it's a fave + + if (in_array($notice->id, $ids)) { + return true; } + // If we're not past the end of the cache window, // then the cache has all available faves, so this one // is not a fave. - if ($cnt < NOTICE_CACHE_WINDOW) { + + if (count($ids) < NOTICE_CACHE_WINDOW) { return false; } + // Otherwise, cache doesn't have all faves; // fall through to the default } + $fave = Fave::pkeyGet(array('user_id' => $this->id, 'notice_id' => $notice->id)); return ((is_null($fave)) ? false : true); @@ -418,13 +419,8 @@ class User extends Memcached_DataObject function favoriteNotices($offset=0, $limit=NOTICES_PER_PAGE) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN fave ON notice.id = fave.notice_id ' . - 'WHERE fave.user_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'user:faves:'.$this->id, - $offset, $limit); + $ids = Fave::stream($this->id, $offset, $limit); + return Notice::getStreamByIds($ids); } function noticesWithFriends($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) @@ -459,8 +455,8 @@ class User extends Memcached_DataObject if ($cache) { // Faves don't happen chronologically, so we need to blow // ;last cache, too - $cache->delete(common_cache_key('user:faves:'.$this->id)); - $cache->delete(common_cache_key('user:faves:'.$this->id).';last'); + $cache->delete(common_cache_key('fave:ids_by_user:'.$this->id)); + $cache->delete(common_cache_key('fave:ids_by_user:'.$this->id.';last')); } } -- cgit v1.2.3-54-g00ecf