From 3081b2e31708520e6ed83041ce3ba04329d3ec90 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 08:44:48 -0400 Subject: start of querybyid --- classes/User.php | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/User.php b/classes/User.php index 3b9b5cd83..098381f73 100644 --- a/classes/User.php +++ b/classes/User.php @@ -444,21 +444,42 @@ class User extends Memcached_DataObject 'SELECT notice.* ' . 'FROM notice JOIN subscription ON notice.profile_id = subscription.subscribed ' . 'WHERE subscription.subscriber = %d '; - $order = null; + return Notice::getStream(sprintf($qry, $this->id), + 'user:notices_with_friends:' . $this->id, + $offset, $limit, $since_id, $before_id, + $order, $since); } else if ($enabled === true || ($enabled == 'transitional' && $this->inboxed == 1)) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN notice_inbox ON notice.id = notice_inbox.notice_id ' . - 'WHERE notice_inbox.user_id = %d '; - // NOTE: we override ORDER - $order = null; + $cache = common_memcache(); + + if (!empty($cache)) { + + # Get the notices out of the cache + + $notices = $cache->get(common_cache_key($cachekey)); + + # On a cache hit, return a DB-object-like wrapper + + if ($notices !== false) { + $wrapper = new ArrayWrapper(array_slice($notices, $offset, $limit)); + return $wrapper; + } + } + + $inbox = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); + + $ids = array(); + + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + + $inbox->free(); + unset($inbox); + + return Notice::getStreamByIds($ids, 'user:notices_with_friends:' . $this->id); } - return Notice::getStream(sprintf($qry, $this->id), - 'user:notices_with_friends:' . $this->id, - $offset, $limit, $since_id, $before_id, - $order, $since); } function blowFavesCache() -- cgit v1.2.3-54-g00ecf From 640628de2d593933e810b4785dfe38923b979713 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:03:19 -0400 Subject: A queuehandler for blowing caches offline We add a queuehandler for blowing the memcached caches off-line. This should speed up the processing of new notices. --- classes/Notice.php | 25 ++++++++++++++++- lib/util.php | 14 ++++++++-- scripts/memcachedqueuehandler.php | 58 +++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 3 +- scripts/stopdaemons.sh | 2 +- 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 scripts/memcachedqueuehandler.php (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 5fa0d79a1..fbfeb9489 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -207,7 +207,11 @@ class Notice extends Memcached_DataObject # XXX: someone clever could prepend instead of clearing the cache if (common_config('memcached', 'enabled')) { - $notice->blowCaches(); + if (common_config('queues', 'enabled')) { + $notice->blowAuthorCaches(); + } else { + $notice->blowCaches(); + } } return $notice; @@ -271,6 +275,25 @@ class Notice extends Memcached_DataObject $this->blowGroupCache($blowLast); } + function blowAuthorCaches($blowLast=false) + { + // Clear the user's cache + $cache = common_memcache(); + if ($cache) { + $user = User::staticGet($this->profile_id); + if (!empty($user)) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + if ($blowLast) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + } + } + $user->free(); + unset($user); + } + $this->blowNoticeCache($blowLast); + $this->blowPublicCache($blowLast); + } + function blowGroupCache($blowLast=false) { $cache = common_memcache(); diff --git a/lib/util.php b/lib/util.php index e0eda0114..12797891c 100644 --- a/lib/util.php +++ b/lib/util.php @@ -879,7 +879,17 @@ function common_broadcast_notice($notice, $remote=false) function common_enqueue_notice($notice) { - foreach (array('jabber', 'omb', 'sms', 'public', 'twitter', 'facebook', 'ping') as $transport) { + $transports = array('omb', 'sms', 'twitter', 'facebook', 'ping'); + + if (common_config('xmpp', 'enabled')) { + $transports = array_merge($transports, array('jabber', 'public')); + } + + if (common_config('memcached', 'enabled')) { + $transports[] = 'memcached'; + } + + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; $qi->transport = $transport; @@ -1332,7 +1342,7 @@ function common_compatible_license($from, $to) */ function common_database_tablename($tablename) { - + if(common_config('db','quote_identifiers')) { $tablename = '"'. $tablename .'"'; } diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php new file mode 100644 index 000000000..3fcebcfc3 --- /dev/null +++ b/scripts/memcachedqueuehandler.php @@ -0,0 +1,58 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class MemcachedQueueHandler extends QueueHandler +{ + function transport() + { + return 'memcached'; + } + + function handle_notice($notice) + { + // XXX: fork here + common_log(LOG_INFO, "Blowing memcached for $notice->id\n"; + $notice->blowCaches(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new MemcachedQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index c3729761d..08de6d954 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -24,7 +24,8 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ - twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php; do + twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ + memcachedqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index 2bb8f9ecb..e5a181cd1 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,7 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler ; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From aee45ea91dcec4736d8b9befe17e030b873d9226 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:08:48 -0400 Subject: Add an inbox queue handler Handle distributing a notice to multiple inboxes in a queue handler rather than in the Web action. --- classes/Notice.php | 5 +++- lib/util.php | 4 +++ scripts/inboxqueuehandler.php | 57 +++++++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 2 +- scripts/stopdaemons.sh | 3 ++- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 scripts/inboxqueuehandler.php (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index fbfeb9489..ff00f2a94 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,10 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - $notice->addToInboxes(); + if (!common_config('queues', 'enabled')) { + $notice->addToInboxes(); + } + $notice->query('COMMIT'); Event::handle('EndNoticeSave', array($notice)); diff --git a/lib/util.php b/lib/util.php index 12797891c..9b6d2941a 100644 --- a/lib/util.php +++ b/lib/util.php @@ -889,6 +889,10 @@ function common_enqueue_notice($notice) $transports[] = 'memcached'; } + if (common_config('queues', 'enabled')) { + $transports[] = 'inbox'; + } + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php new file mode 100644 index 000000000..16e334b83 --- /dev/null +++ b/scripts/inboxqueuehandler.php @@ -0,0 +1,57 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class InboxQueueHandler extends QueueHandler +{ + function transport() + { + return 'inbox'; + } + + function handle_notice($notice) + { + common_log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); + $notice->addToInboxes(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new InboxQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index 08de6d954..66f9ed4e0 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -25,7 +25,7 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ - memcachedqueuehandler.php; do + memcachedqueuehandler.php inboxqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index e5a181cd1..196991de0 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,8 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ + memcachedhandler inboxhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From 31b04220bd9fbadbfac01b60276a9c14896b6a45 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 09:34:56 +0000 Subject: incorrect config setting for queues --- classes/Notice.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index ff00f2a94..ebed0b8af 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,7 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - if (!common_config('queues', 'enabled')) { + if (!common_config('queue', 'enabled')) { $notice->addToInboxes(); } @@ -210,7 +210,7 @@ class Notice extends Memcached_DataObject # XXX: someone clever could prepend instead of clearing the cache if (common_config('memcached', 'enabled')) { - if (common_config('queues', 'enabled')) { + if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { $notice->blowCaches(); -- cgit v1.2.3-54-g00ecf From a9df5eab100bce97da3e028851d224ea2e9fff80 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 10:38:51 +0000 Subject: insert into user's inbox at Web time --- classes/Notice.php | 76 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 25 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index ebed0b8af..27b98de1c 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,9 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - if (!common_config('queue', 'enabled')) { + if (common_config('queue', 'enabled')) { + $notice->addToAuthorInbox(); + } else { $notice->addToInboxes(); } @@ -282,16 +284,8 @@ class Notice extends Memcached_DataObject { // Clear the user's cache $cache = common_memcache(); - if ($cache) { - $user = User::staticGet($this->profile_id); - if (!empty($user)) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); - if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); - } - } - $user->free(); - unset($user); + if (!empty($cache)) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $this->profile_id)); } $this->blowNoticeCache($blowLast); $this->blowPublicCache($blowLast); @@ -665,6 +659,33 @@ class Notice extends Memcached_DataObject return; } + function addToAuthorInbox() + { + $enabled = common_config('inboxes', 'enabled'); + + if ($enabled === true || $enabled === 'transitional') { + $user = User::staticGet('id', $this->profile_id); + if (empty($user)) { + return; + } + $inbox = new Notice_inbox(); + $UT = common_config('db','type')=='pgsql'?'"user"':'user'; + $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created) ' . + "SELECT $UT.id, " . $this->id . ", '" . $this->created . "' " . + "FROM $UT " . + "WHERE $UT.id = " . $this->profile_id . ' ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + "WHERE user_id = " . $this->profile_id . ' '. + 'AND notice_id = ' . $this->id . ' )'; + if ($enabled === 'transitional') { + $qry .= " AND $UT.inboxed = 1"; + } + $inbox->query($qry); + } + return; + } + function saveGroups() { $enabled = common_config('inboxes', 'enabled'); @@ -717,24 +738,29 @@ class Notice extends Memcached_DataObject // FIXME: do this in an offline daemon - $inbox = new Notice_inbox(); - $UT = common_config('db','type')=='pgsql'?'"user"':'user'; - $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created, source) ' . - "SELECT $UT.id, " . $this->id . ", '" . $this->created . "', 2 " . - "FROM $UT JOIN group_member ON $UT.id = group_member.profile_id " . - 'WHERE group_member.group_id = ' . $group->id . ' ' . - 'AND NOT EXISTS (SELECT user_id, notice_id ' . - 'FROM notice_inbox ' . - "WHERE user_id = $UT.id " . - 'AND notice_id = ' . $this->id . ' )'; - if ($enabled === 'transitional') { - $qry .= " AND $UT.inboxed = 1"; - } - $result = $inbox->query($qry); + $this->addToGroupInboxes($group); } } } + function addToGroupInboxes($group) + { + $inbox = new Notice_inbox(); + $UT = common_config('db','type')=='pgsql'?'"user"':'user'; + $qry = 'INSERT INTO notice_inbox (user_id, notice_id, created, source) ' . + "SELECT $UT.id, " . $this->id . ", '" . $this->created . "', 2 " . + "FROM $UT JOIN group_member ON $UT.id = group_member.profile_id " . + 'WHERE group_member.group_id = ' . $group->id . ' ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + "WHERE user_id = $UT.id " . + 'AND notice_id = ' . $this->id . ' )'; + if ($enabled === 'transitional') { + $qry .= " AND $UT.inboxed = 1"; + } + $result = $inbox->query($qry); + } + function saveReplies() { // Alternative reply format -- cgit v1.2.3-54-g00ecf From 84072aa5cf6124d59a06a7f0a7945c00ee2836da Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 12:13:49 -0400 Subject: run 'set names' after each connection to deal with UTF8 correctly --- classes/Memcached_DataObject.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'classes') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 5f71f716b..877bbf2e0 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,4 +227,20 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } + + // We overload so that 'SET NAMES "utf8"' is called for + // each connection + + function _connect() + { + global $_DB_DATAOBJECT; + $exists = !empty($this->_database_dsn_md5) && + isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); + $result = parent::_connect(); + if (!$exists) { + $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; + $DB->query('SET NAMES "utf8"'); + } + return $result; + } } -- cgit v1.2.3-54-g00ecf From 068f6801cc59488bfc50ef399a2a4d22b1b7e9c2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 12:27:32 -0400 Subject: Revert "run 'set names' after each connection to deal with UTF8 correctly" This reverts commit 84072aa5cf6124d59a06a7f0a7945c00ee2836da. This commit caused grievous harm to old notices on identi.ca. Reverting until we figure out how to convert the old notices. --- classes/Memcached_DataObject.php | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'classes') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 877bbf2e0..5f71f716b 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,20 +227,4 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } - - // We overload so that 'SET NAMES "utf8"' is called for - // each connection - - function _connect() - { - global $_DB_DATAOBJECT; - $exists = !empty($this->_database_dsn_md5) && - isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); - $result = parent::_connect(); - if (!$exists) { - $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; - $DB->query('SET NAMES "utf8"'); - } - return $result; - } } -- cgit v1.2.3-54-g00ecf From c7105c2af1ebe3cddd477265c6fea59a61d0c7e5 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:07:05 -0400 Subject: Change to avoid a join in notice inbox The join in notice_inbox is causing temp-table sorting on identi.ca, so I'm trying a finer-tuned approach. --- classes/Notice.php | 31 ++++++++++++--- classes/Notice_inbox.php | 101 ++++++++++++++++++++++++++++++++++++++++++++++- classes/User.php | 29 +------------- 3 files changed, 127 insertions(+), 34 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 27b98de1c..b087c94bc 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,6 +215,7 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { + common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -285,7 +286,7 @@ class Notice extends Memcached_DataObject // Clear the user's cache $cache = common_memcache(); if (!empty($cache)) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $this->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$this->profile_id)); } $this->blowNoticeCache($blowLast); $this->blowPublicCache($blowLast); @@ -307,9 +308,9 @@ class Notice extends Memcached_DataObject $member->group_id = $group_inbox->group_id; if ($member->find()) { while ($member->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id . ';last')); } } } @@ -352,9 +353,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); } } $user->free(); @@ -613,6 +614,26 @@ class Notice extends Memcached_DataObject return $wrapper; } + function getStreamByIds($ids) + { + $cache = common_memcache(); + + if (!empty($cache)) { + $notices = array(); + foreach ($ids as $id) { + $notices[] = Notice::staticGet('id', $id); + } + return new ArrayWrapper($notices); + } else { + $notice = new Notice(); + $notice->whereAdd('id in (' . implode(', ', $ids) . ')'); + $notice->orderBy('id DESC'); + + $notice->find(); + return $notice; + } + } + function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 81ddb4538..162da74fe 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -1,7 +1,7 @@ INBOX_CACHE_WINDOW) { + common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); + return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); + } + + $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + common_debug('Cache hit for notice_inbox.'); + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + common_debug('Cache hit for notice_inbox on last item.'); + + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + $last_id, null, null); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + null, null, null); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } + + function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) + { + $inbox = new Notice_inbox(); + + $inbox->user_id = $user_id; + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $inbox->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; + } } diff --git a/classes/User.php b/classes/User.php index 098381f73..ce7ea1464 100644 --- a/classes/User.php +++ b/classes/User.php @@ -451,34 +451,9 @@ class User extends Memcached_DataObject } else if ($enabled === true || ($enabled == 'transitional' && $this->inboxed == 1)) { - $cache = common_memcache(); + $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - if (!empty($cache)) { - - # Get the notices out of the cache - - $notices = $cache->get(common_cache_key($cachekey)); - - # On a cache hit, return a DB-object-like wrapper - - if ($notices !== false) { - $wrapper = new ArrayWrapper(array_slice($notices, $offset, $limit)); - return $wrapper; - } - } - - $inbox = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - - $ids = array(); - - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - - $inbox->free(); - unset($inbox); - - return Notice::getStreamByIds($ids, 'user:notices_with_friends:' . $this->id); + return Notice::getStreamByIds($ids); } } -- cgit v1.2.3-54-g00ecf From fe53e780be5db4ceb2831a1d69faec6130a10deb Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:31:56 -0400 Subject: Remove some debug comments in query-by-id --- classes/Notice.php | 5 ++--- classes/Notice_inbox.php | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index b087c94bc..49d0939c1 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,7 +215,6 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { - common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -353,9 +352,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id)); if ($blowLast) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id.';last')); } } $user->free(); diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 162da74fe..f32137038 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -50,7 +50,6 @@ class Notice_inbox extends Memcached_DataObject if (empty($cache) || $since_id != 0 || $before_id != 0 || !is_null($since) || ($offset + $limit) > INBOX_CACHE_WINDOW) { - common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); } @@ -60,7 +59,6 @@ class Notice_inbox extends Memcached_DataObject if (!empty($idstr)) { // Cache hit! Woohoo! - common_debug('Cache hit for notice_inbox.'); $window = explode(',', $idstr); $ids = array_slice($window, $offset, $limit); return $ids; @@ -69,8 +67,6 @@ class Notice_inbox extends Memcached_DataObject $laststr = common_cache_key($idkey.';last'); if (!empty($laststr)) { - common_debug('Cache hit for notice_inbox on last item.'); - $window = explode(',', $laststr); $last_id = $window[0]; $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, -- cgit v1.2.3-54-g00ecf From 10ef8a2f7112a83188e9702d480abd3c6062c26c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 11:27:45 -0400 Subject: Move algorithm for caching to Notice class Moved the algorithm for notice stream caching to the Notice class. --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++++++ classes/Notice_inbox.php | 56 +++++------------------------------------------- 2 files changed, 60 insertions(+), 51 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 49d0939c1..faabb1e14 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -967,4 +967,59 @@ class Notice extends Memcached_DataObject array('notice' => $this->id)); } } + + function stream($fn, $args, $cachekey, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $cache = common_memcache(); + + if (empty($cache) || + $since_id != 0 || $before_id != 0 || !is_null($since) || + ($offset + $limit) > NOTICE_CACHE_WINDOW) { + return call_user_func_array($fn, array_merge($args, array($offset, $limit, $since_id, + $before_id, $since))); + } + + $idkey = common_cache_key($cachekey); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + $last_id, 0, null))); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + 0, 0, null))); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } } diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index f32137038..dec14b0d1 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -43,58 +43,12 @@ class Notice_inbox extends Memcached_DataObject /* the code above is auto generated do not remove the tag below */ ###END_AUTOCODE - function stream($user_id, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + function stream($user_id, $offset, $limit, $since_id, $before_id, $since) { - $cache = common_memcache(); - - if (empty($cache) || - $since_id != 0 || $before_id != 0 || !is_null($since) || - ($offset + $limit) > INBOX_CACHE_WINDOW) { - return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); - } - - $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); - - $idstr = $cache->get($idkey); - - if (!empty($idstr)) { - // Cache hit! Woohoo! - $window = explode(',', $idstr); - $ids = array_slice($window, $offset, $limit); - return $ids; - } - - $laststr = common_cache_key($idkey.';last'); - - if (!empty($laststr)) { - $window = explode(',', $laststr); - $last_id = $window[0]; - $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - $last_id, null, null); - - $new_window = array_merge($new_ids, $window); - - $new_windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $new_windowstr); - $result = $cache->set($idkey . ';last', $new_windowstr); - - $ids = array_slice($new_window, $offset, $limit); - - return $ids; - } - - $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - null, null, null); - - $windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $windowstr); - $result = $cache->set($idkey . ';last', $windowstr); - - $ids = array_slice($window, $offset, $limit); - - return $ids; + return Notice::stream(array('Notice_inbox', '_streamDirect'), + array($user_id), + 'notice_inbox:by_user:'.$user_id, + $offset, $limit, $since_id, $before_id, $since); } function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) -- cgit v1.2.3-54-g00ecf From a4d959b8a2254d173bdf45f418f6add9f6f62cda Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 12:05:31 -0400 Subject: Public stream uses IDs method Public stream now uses IDs method --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index faabb1e14..6a7522bd3 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -635,25 +635,58 @@ class Notice extends Memcached_DataObject function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { + $ids = Notice::stream(array('Notice', '_publicStreamDirect'), + array(), + 'public', + $offset, $limit, $since_id, $before_id, $since); - $parts = array(); + return Notice::getStreamByIds($ids); + } + + function _publicStreamDirect($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $notice = new Notice(); + + $notice->selectAdd(); // clears it + $notice->selectAdd('id'); - $qry = 'SELECT * FROM notice '; + $notice->orderBy('id DESC'); + + if (!is_null($offset)) { + $notice->limit($offset, $limit); + } if (common_config('public', 'localonly')) { - $parts[] = 'is_local = 1'; + $notice->whereAdd('is_local = 1'); } else { # -1 == blacklisted - $parts[] = 'is_local != -1'; + $notice->whereAdd('is_local != -1'); } - if ($parts) { - $qry .= ' WHERE ' . implode(' AND ', $parts); + if ($since_id != 0) { + $notice->whereAdd('id > ' . $since_id); } - return Notice::getStream($qry, - 'public', - $offset, $limit, $since_id, $before_id, null, $since); + if ($before_id != 0) { + $notice->whereAdd('id < ' . $before_id); + } + + if (!is_null($since)) { + $notice->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $ids = array(); + + if ($notice->find()) { + while ($notice->fetch()) { + $ids[] = $notice->id; + } + } + + $notice->free(); + $notice = NULL; + + return $ids; } function addToInboxes() @@ -990,7 +1023,7 @@ class Notice extends Memcached_DataObject return $ids; } - $laststr = common_cache_key($idkey.';last'); + $laststr = $cache->get($idkey.';last'); if (!empty($laststr)) { $window = explode(',', $laststr); @@ -1013,7 +1046,7 @@ class Notice extends Memcached_DataObject $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, 0, 0, null))); - $windowstr = implode(',', $new_window); + $windowstr = implode(',', $window); $result = $cache->set($idkey, $windowstr); $result = $cache->set($idkey . ';last', $windowstr); -- cgit v1.2.3-54-g00ecf From 1e8ea1eb460b163176c4d7d1e7dffa500024ef91 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 16:09:03 -0400 Subject: Make the tag stream use ID mechanism --- classes/Notice.php | 5 +---- classes/Notice_tag.php | 61 +++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 14 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 6a7522bd3..2bb466155 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -328,10 +328,7 @@ class Notice extends Memcached_DataObject $tag->notice_id = $this->id; if ($tag->find()) { while ($tag->fetch()) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $tag->tag)); - if ($blowLast) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $tag->tag . ';last')); - } + $tag->blowCache($blowLast); } } $tag->free(); diff --git a/classes/Notice_tag.php b/classes/Notice_tag.php index f2247299a..e5b772243 100644 --- a/classes/Notice_tag.php +++ b/classes/Notice_tag.php @@ -37,21 +37,62 @@ class Notice_tag extends Memcached_DataObject ###END_AUTOCODE static function getStream($tag, $offset=0, $limit=20) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN notice_tag ON notice.id = notice_tag.notice_id ' . - "WHERE notice_tag.tag = '%s' "; - - return Notice::getStream(sprintf($qry, $tag), - 'notice_tag:notice_stream:' . common_keyize($tag), - $offset, $limit); + + $ids = Notice::stream(array('Notice_tag', '_streamDirect'), + array($tag), + 'notice_tag:notice_ids:' . common_keyize($tag), + $offset, $limit); + + return Notice::getStreamByIds($ids); + } + + function _streamDirect($tag, $offset, $limit, $since_id, $before_id, $since) + { + $nt = new Notice_tag(); + + $nt->tag = $tag; + + $nt->selectAdd(); + $nt->selectAdd('notice_id'); + + if ($since_id != 0) { + $nt->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $nt->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $nt->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $nt->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $nt->limit($offset, $limit); + } + + $ids = array(); + + if ($nt->find()) { + while ($nt->fetch()) { + $ids[] = $nt->notice_id; + } + } + + return $ids; } - function blowCache() + function blowCache($blowLast=false) { $cache = common_memcache(); if ($cache) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $this->tag)); + $idkey = common_cache_key('notice_tag:notice_ids:' . common_keyize($this->tag)); + $cache->delete($idkey); + if ($blowLast) { + $cache->delete($idkey.';last'); + } } } -- cgit v1.2.3-54-g00ecf From aee641ee1e311fb0af0f9f6d75ca7fae2c7d8477 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 20:45:33 -0400 Subject: make replies use new query format --- classes/Notice.php | 4 ++-- classes/Reply.php | 47 +++++++++++++++++++++++++++++++++++++++++++++-- classes/User.php | 10 +++------- 3 files changed, 50 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 2bb466155..808631f4d 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -380,9 +380,9 @@ class Notice extends Memcached_DataObject $reply->notice_id = $this->id; if ($reply->find()) { while ($reply->fetch()) { - $cache->delete(common_cache_key('user:replies:'.$reply->profile_id)); + $cache->delete(common_cache_key('reply:stream:'.$reply->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:replies:'.$reply->profile_id.';last')); + $cache->delete(common_cache_key('reply:stream:'.$reply->profile_id.';last')); } } } diff --git a/classes/Reply.php b/classes/Reply.php index af86aaf87..4439053b4 100644 --- a/classes/Reply.php +++ b/classes/Reply.php @@ -4,7 +4,7 @@ */ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; -class Reply extends Memcached_DataObject +class Reply extends Memcached_DataObject { ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -13,7 +13,7 @@ class Reply extends Memcached_DataObject public $notice_id; // int(4) primary_key not_null public $profile_id; // int(4) primary_key not_null public $modified; // timestamp() not_null default_CURRENT_TIMESTAMP - public $replied_id; // int(4) + public $replied_id; // int(4) /* Static get */ function staticGet($k,$v=null) @@ -21,4 +21,47 @@ class Reply extends Memcached_DataObject /* the code above is auto generated do not remove the tag below */ ###END_AUTOCODE + + function stream($user_id, $offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) + { + $ids = Notice::stream(array('Reply', '_streamDirect'), + array($user_id), + 'reply:stream:' . $user_id, + $offset, $limit, $since_id, $before_id, $since); + return $ids; + } + + function _streamDirect($user_id, $offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) + { + $reply = new Reply(); + $reply->profile_id = $user_id; + + if ($since_id != 0) { + $reply->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $reply->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $reply->whereAdd('modified > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $reply->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $reply->limit($offset, $limit); + } + + $ids = array(); + + if ($reply->find()) { + while ($reply->fetch()) { + $ids[] = $reply->notice_id; + } + } + + return $ids; + } } diff --git a/classes/User.php b/classes/User.php index ce7ea1464..b76e45c33 100644 --- a/classes/User.php +++ b/classes/User.php @@ -401,13 +401,9 @@ class User extends Memcached_DataObject function getReplies($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN reply ON notice.id = reply.notice_id ' . - 'WHERE reply.profile_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'user:replies:'.$this->id, - $offset, $limit, $since_id, $before_id, null, $since); + $ids = Reply::stream($this->id, $offset, $limit, $since_id, $before_id, $since); + common_debug("Ids = " . implode(',', $ids)); + return Notice::getStreamByIds($ids); } function getNotices($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) -- cgit v1.2.3-54-g00ecf From 3328ec545c36fc2408cdb9d048effe24feafe218 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 11:27:57 -0700 Subject: make profile notice getting use ids --- classes/Notice.php | 6 +++--- classes/Profile.php | 53 +++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 808631f4d..eceed325b 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -363,10 +363,10 @@ class Notice extends Memcached_DataObject { if ($this->is_local) { $cache = common_memcache(); - if ($cache) { - $cache->delete(common_cache_key('profile:notices:'.$this->profile_id)); + if (!empty($cache)) { + $cache->delete(common_cache_key('profile:notice_ids:'.$this->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('profile:notices:'.$this->profile_id.';last')); + $cache->delete(common_cache_key('profile:notice_ids:'.$this->profile_id.';last')); } } } diff --git a/classes/Profile.php b/classes/Profile.php index f3bfe299c..ae5641d79 100644 --- a/classes/Profile.php +++ b/classes/Profile.php @@ -155,14 +155,51 @@ class Profile extends Memcached_DataObject function getNotices($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0) { - $qry = - 'SELECT * ' . - 'FROM notice ' . - 'WHERE profile_id = %d '; - - return Notice::getStream(sprintf($qry, $this->id), - 'profile:notices:'.$this->id, - $offset, $limit, $since_id, $before_id); + // XXX: I'm not sure this is going to be any faster. It probably isn't. + $ids = Notice::stream(array($this, '_streamDirect'), + array(), + 'profile:notice_ids:' . $this->id, + $offset, $limit, $since_id, $before_id); + + return Notice::getStreamByIds($ids); + } + + function _streamDirect($offset, $limit, $since_id, $before_id, $since) + { + $notice = new Notice(); + + $notice->profile_id = $this->id; + + $notice->selectAdd(); + $notice->selectAdd('id'); + + if ($since_id != 0) { + $notice->whereAdd('id > ' . $since_id); + } + + if ($before_id != 0) { + $notice->whereAdd('id < ' . $before_id); + } + + if (!is_null($since)) { + $notice->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $notice->orderBy('id DESC'); + + if (!is_null($offset)) { + $notice->limit($offset, $limit); + } + + $ids = array(); + + if ($notice->find()) { + while ($notice->fetch()) { + $ids[] = $notice->id; + } + } + + return $ids; } function isMember($group) -- cgit v1.2.3-54-g00ecf From 021b520a11d3449a1476182e1ad117582999d364 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 11:38:50 -0700 Subject: Make user group stream use IDs --- classes/Notice.php | 4 ++-- classes/User_group.php | 53 ++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 10 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index eceed325b..c036e6e9e 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -299,9 +299,9 @@ class Notice extends Memcached_DataObject $group_inbox->notice_id = $this->id; if ($group_inbox->find()) { while ($group_inbox->fetch()) { - $cache->delete(common_cache_key('group:notices:'.$group_inbox->group_id)); + $cache->delete(common_cache_key('user_group:notice_ids:' . $group_inbox->group_id)); if ($blowLast) { - $cache->delete(common_cache_key('group:notices:'.$group_inbox->group_id.';last')); + $cache->delete(common_cache_key('user_group:notice_ids:' . $group_inbox->group_id.';last')); } $member = new Group_member(); $member->group_id = $group_inbox->group_id; diff --git a/classes/User_group.php b/classes/User_group.php index d152f9d56..7cc31e702 100755 --- a/classes/User_group.php +++ b/classes/User_group.php @@ -50,13 +50,50 @@ class User_group extends Memcached_DataObject function getNotices($offset, $limit) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN group_inbox ON notice.id = group_inbox.notice_id ' . - 'WHERE group_inbox.group_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'group:notices:'.$this->id, - $offset, $limit); + $ids = Notice::stream(array($this, '_streamDirect'), + array(), + 'user_group:notice_ids:' . $this->id, + $offset, $limit); + + return Notice::getStreamByIds($ids); + } + + function _streamDirect($offset, $limit, $since_id, $before_id, $since) + { + $inbox = new Group_inbox(); + + $inbox->group_id = $this->id; + + $inbox->selectAdd(); + $inbox->selectAdd('notice_id'); + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $inbox->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; } function allowedNickname($nickname) @@ -91,7 +128,7 @@ class User_group extends Memcached_DataObject function setOriginal($filename) { $imagefile = new ImageFile($this->id, Avatar::path($filename)); - + $orig = clone($this); $this->original_logo = Avatar::url($filename); $this->homepage_logo = Avatar::url($imagefile->resize(AVATAR_PROFILE_SIZE)); -- cgit v1.2.3-54-g00ecf From 5314d9b2cfaef3f2fd0ead262e18d1776fd99c8d Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 12:01:28 -0700 Subject: make faves work with ids --- classes/Fave.php | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- classes/Notice.php | 4 ++-- classes/User.php | 38 +++++++++++++++++--------------------- 3 files changed, 70 insertions(+), 25 deletions(-) (limited to 'classes') diff --git a/classes/Fave.php b/classes/Fave.php index 24df5938c..915b4572f 100644 --- a/classes/Fave.php +++ b/classes/Fave.php @@ -4,7 +4,7 @@ */ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; -class Fave extends Memcached_DataObject +class Fave extends Memcached_DataObject { ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -31,9 +31,58 @@ class Fave extends Memcached_DataObject } return $fave; } - + function &pkeyGet($kv) { return Memcached_DataObject::pkeyGet('Fave', $kv); } + + function stream($user_id, $offset=0, $limit=NOTICES_PER_PAGE) + { + $ids = Notice::stream(array('Fave', '_streamDirect'), + array($user_id), + 'fave:ids_by_user:'.$user_id, + $offset, $limit); + return $ids; + } + + function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) + { + $fav = new Fave(); + + $fav->user_id = $user_id; + + $fav->selectAdd(); + $fav->selectAdd('notice_id'); + + if ($since_id != 0) { + $fav->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $fav->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $fav->whereAdd('modified > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + // NOTE: we sort by fave time, not by notice time! + + $fav->orderBy('modified DESC'); + + if (!is_null($offset)) { + $fav->limit($offset, $limit); + } + + $ids = array(); + + if ($fav->find()) { + while ($fav->fetch()) { + $ids[] = $fav->notice_id; + } + } + + return $ids; + } } diff --git a/classes/Notice.php b/classes/Notice.php index c036e6e9e..771a4e715 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -412,9 +412,9 @@ class Notice extends Memcached_DataObject $fave->notice_id = $this->id; if ($fave->find()) { while ($fave->fetch()) { - $cache->delete(common_cache_key('user:faves:'.$fave->user_id)); + $cache->delete(common_cache_key('fave:ids_by_user:'.$fave->user_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:faves:'.$fave->user_id.';last')); + $cache->delete(common_cache_key('fave:ids_by_user:'.$fave->user_id.';last')); } } } diff --git a/classes/User.php b/classes/User.php index b76e45c33..b5ac7b220 100644 --- a/classes/User.php +++ b/classes/User.php @@ -349,30 +349,31 @@ class User extends Memcached_DataObject $cache = common_memcache(); // XXX: Kind of a hack. + if ($cache) { // This is the stream of favorite notices, in rev chron // order. This forces it into cache. - $faves = $this->favoriteNotices(0, NOTICE_CACHE_WINDOW); - $cnt = 0; - while ($faves->fetch()) { - if ($faves->id < $notice->id) { - // If we passed it, it's not a fave - return false; - } else if ($faves->id == $notice->id) { - // If it matches a cached notice, then it's a fave - return true; - } - $cnt++; + + $ids = Fave::stream($this->id, 0, NOTICE_CACHE_WINDOW); + + // If it's in the list, then it's a fave + + if (in_array($notice->id, $ids)) { + return true; } + // If we're not past the end of the cache window, // then the cache has all available faves, so this one // is not a fave. - if ($cnt < NOTICE_CACHE_WINDOW) { + + if (count($ids) < NOTICE_CACHE_WINDOW) { return false; } + // Otherwise, cache doesn't have all faves; // fall through to the default } + $fave = Fave::pkeyGet(array('user_id' => $this->id, 'notice_id' => $notice->id)); return ((is_null($fave)) ? false : true); @@ -418,13 +419,8 @@ class User extends Memcached_DataObject function favoriteNotices($offset=0, $limit=NOTICES_PER_PAGE) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN fave ON notice.id = fave.notice_id ' . - 'WHERE fave.user_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'user:faves:'.$this->id, - $offset, $limit); + $ids = Fave::stream($this->id, $offset, $limit); + return Notice::getStreamByIds($ids); } function noticesWithFriends($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) @@ -459,8 +455,8 @@ class User extends Memcached_DataObject if ($cache) { // Faves don't happen chronologically, so we need to blow // ;last cache, too - $cache->delete(common_cache_key('user:faves:'.$this->id)); - $cache->delete(common_cache_key('user:faves:'.$this->id).';last'); + $cache->delete(common_cache_key('fave:ids_by_user:'.$this->id)); + $cache->delete(common_cache_key('fave:ids_by_user:'.$this->id.';last')); } } -- cgit v1.2.3-54-g00ecf From 3e7b1e69e3e97ac007465376b62084f10bcf97ca Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 28 Apr 2009 17:08:20 -0700 Subject: Added dirty dates to Foreign_link --- classes/Foreign_link.php | 2 ++ classes/laconica.ini | 2 ++ db/laconica.sql | 2 ++ 3 files changed, 6 insertions(+) (limited to 'classes') diff --git a/classes/Foreign_link.php b/classes/Foreign_link.php index afc0e2180..af2b3f189 100644 --- a/classes/Foreign_link.php +++ b/classes/Foreign_link.php @@ -17,6 +17,8 @@ class Foreign_link extends Memcached_DataObject public $noticesync; // tinyint(1) not_null default_1 public $friendsync; // tinyint(1) not_null default_2 public $profilesync; // tinyint(1) not_null default_1 + public $last_noticesync; // datetime() + public $last_friendsync; // datetime() public $created; // datetime() not_null public $modified; // timestamp() not_null default_CURRENT_TIMESTAMP diff --git a/classes/laconica.ini b/classes/laconica.ini index 529454d99..c05419588 100755 --- a/classes/laconica.ini +++ b/classes/laconica.ini @@ -55,6 +55,8 @@ credentials = 2 noticesync = 145 friendsync = 145 profilesync = 145 +last_noticesync = 14 +last_friendsync = 14 created = 142 modified = 384 diff --git a/db/laconica.sql b/db/laconica.sql index 5b57494d9..c9730098e 100644 --- a/db/laconica.sql +++ b/db/laconica.sql @@ -289,6 +289,8 @@ create table foreign_link ( noticesync tinyint not null default 1 comment 'notice synchronization, bit 1 = sync outgoing, bit 2 = sync incoming, bit 3 = filter local replies', friendsync tinyint not null default 2 comment 'friend synchronization, bit 1 = sync outgoing, bit 2 = sync incoming', profilesync tinyint not null default 1 comment 'profile synchronization, bit 1 = sync outgoing, bit 2 = sync incoming', + last_noticesync datetime default null comment 'last time notices were imported', + last_friendsync datetime default null comment 'last time friends were imported', created datetime not null comment 'date this record was created', modified timestamp comment 'date this record was modified', -- cgit v1.2.3-54-g00ecf