From 3081b2e31708520e6ed83041ce3ba04329d3ec90 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 08:44:48 -0400 Subject: start of querybyid --- classes/User.php | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/User.php b/classes/User.php index 3b9b5cd83..098381f73 100644 --- a/classes/User.php +++ b/classes/User.php @@ -444,21 +444,42 @@ class User extends Memcached_DataObject 'SELECT notice.* ' . 'FROM notice JOIN subscription ON notice.profile_id = subscription.subscribed ' . 'WHERE subscription.subscriber = %d '; - $order = null; + return Notice::getStream(sprintf($qry, $this->id), + 'user:notices_with_friends:' . $this->id, + $offset, $limit, $since_id, $before_id, + $order, $since); } else if ($enabled === true || ($enabled == 'transitional' && $this->inboxed == 1)) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN notice_inbox ON notice.id = notice_inbox.notice_id ' . - 'WHERE notice_inbox.user_id = %d '; - // NOTE: we override ORDER - $order = null; + $cache = common_memcache(); + + if (!empty($cache)) { + + # Get the notices out of the cache + + $notices = $cache->get(common_cache_key($cachekey)); + + # On a cache hit, return a DB-object-like wrapper + + if ($notices !== false) { + $wrapper = new ArrayWrapper(array_slice($notices, $offset, $limit)); + return $wrapper; + } + } + + $inbox = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); + + $ids = array(); + + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + + $inbox->free(); + unset($inbox); + + return Notice::getStreamByIds($ids, 'user:notices_with_friends:' . $this->id); } - return Notice::getStream(sprintf($qry, $this->id), - 'user:notices_with_friends:' . $this->id, - $offset, $limit, $since_id, $before_id, - $order, $since); } function blowFavesCache() -- cgit v1.2.3-54-g00ecf From 84072aa5cf6124d59a06a7f0a7945c00ee2836da Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 12:13:49 -0400 Subject: run 'set names' after each connection to deal with UTF8 correctly --- classes/Memcached_DataObject.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'classes') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 5f71f716b..877bbf2e0 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,4 +227,20 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } + + // We overload so that 'SET NAMES "utf8"' is called for + // each connection + + function _connect() + { + global $_DB_DATAOBJECT; + $exists = !empty($this->_database_dsn_md5) && + isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); + $result = parent::_connect(); + if (!$exists) { + $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; + $DB->query('SET NAMES "utf8"'); + } + return $result; + } } -- cgit v1.2.3-54-g00ecf From 068f6801cc59488bfc50ef399a2a4d22b1b7e9c2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 12:27:32 -0400 Subject: Revert "run 'set names' after each connection to deal with UTF8 correctly" This reverts commit 84072aa5cf6124d59a06a7f0a7945c00ee2836da. This commit caused grievous harm to old notices on identi.ca. Reverting until we figure out how to convert the old notices. --- classes/Memcached_DataObject.php | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'classes') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 877bbf2e0..5f71f716b 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,20 +227,4 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } - - // We overload so that 'SET NAMES "utf8"' is called for - // each connection - - function _connect() - { - global $_DB_DATAOBJECT; - $exists = !empty($this->_database_dsn_md5) && - isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); - $result = parent::_connect(); - if (!$exists) { - $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; - $DB->query('SET NAMES "utf8"'); - } - return $result; - } } -- cgit v1.2.3-54-g00ecf From c7105c2af1ebe3cddd477265c6fea59a61d0c7e5 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:07:05 -0400 Subject: Change to avoid a join in notice inbox The join in notice_inbox is causing temp-table sorting on identi.ca, so I'm trying a finer-tuned approach. --- classes/Notice.php | 31 ++++++++++++--- classes/Notice_inbox.php | 101 ++++++++++++++++++++++++++++++++++++++++++++++- classes/User.php | 29 +------------- 3 files changed, 127 insertions(+), 34 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 27b98de1c..b087c94bc 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,6 +215,7 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { + common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -285,7 +286,7 @@ class Notice extends Memcached_DataObject // Clear the user's cache $cache = common_memcache(); if (!empty($cache)) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $this->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$this->profile_id)); } $this->blowNoticeCache($blowLast); $this->blowPublicCache($blowLast); @@ -307,9 +308,9 @@ class Notice extends Memcached_DataObject $member->group_id = $group_inbox->group_id; if ($member->find()) { while ($member->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $member->profile_id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:' . $member->profile_id . ';last')); } } } @@ -352,9 +353,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); } } $user->free(); @@ -613,6 +614,26 @@ class Notice extends Memcached_DataObject return $wrapper; } + function getStreamByIds($ids) + { + $cache = common_memcache(); + + if (!empty($cache)) { + $notices = array(); + foreach ($ids as $id) { + $notices[] = Notice::staticGet('id', $id); + } + return new ArrayWrapper($notices); + } else { + $notice = new Notice(); + $notice->whereAdd('id in (' . implode(', ', $ids) . ')'); + $notice->orderBy('id DESC'); + + $notice->find(); + return $notice; + } + } + function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 81ddb4538..162da74fe 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -1,7 +1,7 @@ INBOX_CACHE_WINDOW) { + common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); + return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); + } + + $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + common_debug('Cache hit for notice_inbox.'); + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + common_debug('Cache hit for notice_inbox on last item.'); + + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + $last_id, null, null); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, + null, null, null); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } + + function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) + { + $inbox = new Notice_inbox(); + + $inbox->user_id = $user_id; + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $inbox->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; + } } diff --git a/classes/User.php b/classes/User.php index 098381f73..ce7ea1464 100644 --- a/classes/User.php +++ b/classes/User.php @@ -451,34 +451,9 @@ class User extends Memcached_DataObject } else if ($enabled === true || ($enabled == 'transitional' && $this->inboxed == 1)) { - $cache = common_memcache(); + $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - if (!empty($cache)) { - - # Get the notices out of the cache - - $notices = $cache->get(common_cache_key($cachekey)); - - # On a cache hit, return a DB-object-like wrapper - - if ($notices !== false) { - $wrapper = new ArrayWrapper(array_slice($notices, $offset, $limit)); - return $wrapper; - } - } - - $inbox = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since); - - $ids = array(); - - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - - $inbox->free(); - unset($inbox); - - return Notice::getStreamByIds($ids, 'user:notices_with_friends:' . $this->id); + return Notice::getStreamByIds($ids); } } -- cgit v1.2.3-54-g00ecf From fe53e780be5db4ceb2831a1d69faec6130a10deb Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 28 Apr 2009 13:31:56 -0400 Subject: Remove some debug comments in query-by-id --- classes/Notice.php | 5 ++--- classes/Notice_inbox.php | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index b087c94bc..49d0939c1 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -215,7 +215,6 @@ class Notice extends Memcached_DataObject if (common_config('queue', 'enabled')) { $notice->blowAuthorCaches(); } else { - common_debug("Blowing caches for new notice."); $notice->blowCaches(); } } @@ -353,9 +352,9 @@ class Notice extends Memcached_DataObject 'WHERE subscription.subscribed = ' . $this->profile_id); while ($user->fetch()) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id)); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id)); if ($blowLast) { - $cache->delete(common_cache_key('notice_inbox:by_user:'.$user_id.';last')); + $cache->delete(common_cache_key('notice_inbox:by_user:'.$user->id.';last')); } } $user->free(); diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 162da74fe..f32137038 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -50,7 +50,6 @@ class Notice_inbox extends Memcached_DataObject if (empty($cache) || $since_id != 0 || $before_id != 0 || !is_null($since) || ($offset + $limit) > INBOX_CACHE_WINDOW) { - common_debug('Doing direct DB hit for notice_inbox since the params are screwy.'); return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); } @@ -60,7 +59,6 @@ class Notice_inbox extends Memcached_DataObject if (!empty($idstr)) { // Cache hit! Woohoo! - common_debug('Cache hit for notice_inbox.'); $window = explode(',', $idstr); $ids = array_slice($window, $offset, $limit); return $ids; @@ -69,8 +67,6 @@ class Notice_inbox extends Memcached_DataObject $laststr = common_cache_key($idkey.';last'); if (!empty($laststr)) { - common_debug('Cache hit for notice_inbox on last item.'); - $window = explode(',', $laststr); $last_id = $window[0]; $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, -- cgit v1.2.3-54-g00ecf From 10ef8a2f7112a83188e9702d480abd3c6062c26c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 11:27:45 -0400 Subject: Move algorithm for caching to Notice class Moved the algorithm for notice stream caching to the Notice class. --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++++++ classes/Notice_inbox.php | 56 +++++------------------------------------------- 2 files changed, 60 insertions(+), 51 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index 49d0939c1..faabb1e14 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -967,4 +967,59 @@ class Notice extends Memcached_DataObject array('notice' => $this->id)); } } + + function stream($fn, $args, $cachekey, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $cache = common_memcache(); + + if (empty($cache) || + $since_id != 0 || $before_id != 0 || !is_null($since) || + ($offset + $limit) > NOTICE_CACHE_WINDOW) { + return call_user_func_array($fn, array_merge($args, array($offset, $limit, $since_id, + $before_id, $since))); + } + + $idkey = common_cache_key($cachekey); + + $idstr = $cache->get($idkey); + + if (!empty($idstr)) { + // Cache hit! Woohoo! + $window = explode(',', $idstr); + $ids = array_slice($window, $offset, $limit); + return $ids; + } + + $laststr = common_cache_key($idkey.';last'); + + if (!empty($laststr)) { + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + $last_id, 0, null))); + + $new_window = array_merge($new_ids, $window); + + $new_windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); + + $ids = array_slice($new_window, $offset, $limit); + + return $ids; + } + + $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, + 0, 0, null))); + + $windowstr = implode(',', $new_window); + + $result = $cache->set($idkey, $windowstr); + $result = $cache->set($idkey . ';last', $windowstr); + + $ids = array_slice($window, $offset, $limit); + + return $ids; + } } diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index f32137038..dec14b0d1 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -43,58 +43,12 @@ class Notice_inbox extends Memcached_DataObject /* the code above is auto generated do not remove the tag below */ ###END_AUTOCODE - function stream($user_id, $offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + function stream($user_id, $offset, $limit, $since_id, $before_id, $since) { - $cache = common_memcache(); - - if (empty($cache) || - $since_id != 0 || $before_id != 0 || !is_null($since) || - ($offset + $limit) > INBOX_CACHE_WINDOW) { - return Notice_inbox::_streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since); - } - - $idkey = common_cache_key('notice_inbox:by_user:'.$user_id); - - $idstr = $cache->get($idkey); - - if (!empty($idstr)) { - // Cache hit! Woohoo! - $window = explode(',', $idstr); - $ids = array_slice($window, $offset, $limit); - return $ids; - } - - $laststr = common_cache_key($idkey.';last'); - - if (!empty($laststr)) { - $window = explode(',', $laststr); - $last_id = $window[0]; - $new_ids = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - $last_id, null, null); - - $new_window = array_merge($new_ids, $window); - - $new_windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $new_windowstr); - $result = $cache->set($idkey . ';last', $new_windowstr); - - $ids = array_slice($new_window, $offset, $limit); - - return $ids; - } - - $window = Notice_inbox::_streamDirect($user_id, 0, INBOX_CACHE_WINDOW, - null, null, null); - - $windowstr = implode(',', $new_window); - - $result = $cache->set($idkey, $windowstr); - $result = $cache->set($idkey . ';last', $windowstr); - - $ids = array_slice($window, $offset, $limit); - - return $ids; + return Notice::stream(array('Notice_inbox', '_streamDirect'), + array($user_id), + 'notice_inbox:by_user:'.$user_id, + $offset, $limit, $since_id, $before_id, $since); } function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) -- cgit v1.2.3-54-g00ecf From a4d959b8a2254d173bdf45f418f6add9f6f62cda Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 12:05:31 -0400 Subject: Public stream uses IDs method Public stream now uses IDs method --- classes/Notice.php | 55 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) (limited to 'classes') diff --git a/classes/Notice.php b/classes/Notice.php index faabb1e14..6a7522bd3 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -635,25 +635,58 @@ class Notice extends Memcached_DataObject function publicStream($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) { + $ids = Notice::stream(array('Notice', '_publicStreamDirect'), + array(), + 'public', + $offset, $limit, $since_id, $before_id, $since); - $parts = array(); + return Notice::getStreamByIds($ids); + } + + function _publicStreamDirect($offset=0, $limit=20, $since_id=0, $before_id=0, $since=null) + { + $notice = new Notice(); + + $notice->selectAdd(); // clears it + $notice->selectAdd('id'); - $qry = 'SELECT * FROM notice '; + $notice->orderBy('id DESC'); + + if (!is_null($offset)) { + $notice->limit($offset, $limit); + } if (common_config('public', 'localonly')) { - $parts[] = 'is_local = 1'; + $notice->whereAdd('is_local = 1'); } else { # -1 == blacklisted - $parts[] = 'is_local != -1'; + $notice->whereAdd('is_local != -1'); } - if ($parts) { - $qry .= ' WHERE ' . implode(' AND ', $parts); + if ($since_id != 0) { + $notice->whereAdd('id > ' . $since_id); } - return Notice::getStream($qry, - 'public', - $offset, $limit, $since_id, $before_id, null, $since); + if ($before_id != 0) { + $notice->whereAdd('id < ' . $before_id); + } + + if (!is_null($since)) { + $notice->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $ids = array(); + + if ($notice->find()) { + while ($notice->fetch()) { + $ids[] = $notice->id; + } + } + + $notice->free(); + $notice = NULL; + + return $ids; } function addToInboxes() @@ -990,7 +1023,7 @@ class Notice extends Memcached_DataObject return $ids; } - $laststr = common_cache_key($idkey.';last'); + $laststr = $cache->get($idkey.';last'); if (!empty($laststr)) { $window = explode(',', $laststr); @@ -1013,7 +1046,7 @@ class Notice extends Memcached_DataObject $window = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, 0, 0, null))); - $windowstr = implode(',', $new_window); + $windowstr = implode(',', $window); $result = $cache->set($idkey, $windowstr); $result = $cache->set($idkey . ';last', $windowstr); -- cgit v1.2.3-54-g00ecf