diff options
-rw-r--r-- | classes/Inbox.php | 151 | ||||
-rw-r--r-- | classes/Notice.php | 29 | ||||
-rw-r--r-- | classes/Notice_inbox.php | 126 | ||||
-rw-r--r-- | classes/User.php | 82 | ||||
-rw-r--r-- | classes/statusnet.ini | 7 | ||||
-rw-r--r-- | db/statusnet.sql | 8 | ||||
-rw-r--r-- | plugins/PubSubHubBub/PubSubHubBubPlugin.php | 16 | ||||
-rw-r--r-- | plugins/Realtime/RealtimePlugin.php | 11 | ||||
-rwxr-xr-x | plugins/TwitterBridge/daemons/twitterstatusfetcher.php | 14 | ||||
-rw-r--r-- | scripts/triminboxes.php | 76 |
10 files changed, 218 insertions, 302 deletions
diff --git a/classes/Inbox.php b/classes/Inbox.php new file mode 100644 index 000000000..e14d4f4e7 --- /dev/null +++ b/classes/Inbox.php @@ -0,0 +1,151 @@ +<?php +/** + * StatusNet, the distributed open-source microblogging tool + * + * Data class for user location preferences + * + * PHP version 5 + * + * LICENCE: This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @category Data + * @package StatusNet + * @author Evan Prodromou <evan@status.net> + * @copyright 2009 StatusNet Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; + +class Inbox extends Memcached_DataObject +{ + const BOXCAR = 128; + + ###START_AUTOCODE + /* the code below is auto generated do not remove the above tag */ + + public $__table = 'inbox'; // table name + public $user_id; // int(4) primary_key not_null + public $notice_ids; // blob + + /* Static get */ + function staticGet($k,$v=NULL) { return Memcached_DataObject::staticGet('Inbox',$k,$v); } + + /* the code above is auto generated do not remove the tag below */ + ###END_AUTOCODE + + function sequenceKey() + { + return array(false, false, false); + } + + /** + * Create a new inbox from existing Notice_inbox stuff + */ + + static function initialize($user_id) + { + $ids = array(); + + $ni = new Notice_inbox(); + + $ni->user_id = $user_id; + $ni->selectAdd(); + $ni->selectAdd('notice_id'); + $ni->orderBy('notice_id DESC'); + $ni->limit(0, 1024); + + if ($ni->find()) { + while($ni->fetch()) { + $ids[] = $ni->notice_id; + } + } + + $ni->free(); + unset($ni); + + $inbox = new Inbox(); + + $inbox->user_id = $user_id; + $inbox->notice_ids = call_user_func_array('pack', array_merge(array('N*'), $ids)); + + $result = $inbox->insert(); + + if (!$result) { + common_log_db_error($inbox, 'INSERT', __FILE__); + return null; + } + + return $inbox; + } + + static function insertNotice($user_id, $notice_id) + { + $inbox = Inbox::staticGet('user_id', $user_id); + + if (empty($inbox)) { + $inbox = Inbox::initialize($user_id); + } + + if (empty($inbox)) { + return false; + } + + $result = $inbox->query(sprintf('UPDATE inbox '. + 'set notice_ids = concat(cast(0x%08x as binary(4)), '. + 'substr(notice_ids, 1, 4092)) '. + 'WHERE user_id = %d', + $notice_id, $user_id)); + + if ($result) { + $c = self::memcache(); + + if (!empty($c)) { + $c->delete(self::cacheKey('inbox', 'user_id', $user_id)); + } + } + + return $result; + } + + static function bulkInsert($notice_id, $user_ids) + { + foreach ($user_ids as $user_id) + { + Inbox::insertNotice($user_id, $notice_id); + } + } + + function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) + { + $inbox = Inbox::staticGet('user_id', $user_id); + + if (empty($inbox)) { + $inbox = Inbox::initialize($user_id); + if (empty($inbox)) { + return array(); + } + } + + $ids = unpack('N*', $inbox->notice_ids); + + // XXX: handle since_id + // XXX: handle max_id + + $ids = array_slice($ids, $offset, $limit); + + return $ids; + } +} diff --git a/classes/Notice.php b/classes/Notice.php index 9bda47827..6284b8ca5 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -125,8 +125,7 @@ class Notice extends Memcached_DataObject 'Fave', 'Notice_tag', 'Group_inbox', - 'Queue_item', - 'Notice_inbox'); + 'Queue_item'); foreach ($related as $cls) { $inst = new $cls(); @@ -504,17 +503,6 @@ class Notice extends Memcached_DataObject unset($original); } - $ni = new Notice_inbox(); - - $ni->notice_id = $this->id; - - if ($ni->find()) { - while ($ni->fetch()) { - $tmk = common_cache_key('user:repeated_to_me:'.$ni->user_id); - $cache->delete($tmk); - } - } - $ni->free(); unset($ni); } @@ -842,12 +830,8 @@ class Notice extends Memcached_DataObject return $ids; } - function addToInboxes() + function whoGets() { - // XXX: loads constants - - $inbox = new Notice_inbox(); - $users = $this->getSubscribedUsers(); // FIXME: kind of ignoring 'transitional'... @@ -887,7 +871,14 @@ class Notice extends Memcached_DataObject } } - Notice_inbox::bulkInsert($this->id, $this->created, $ni); + return $ni; + } + + function addToInboxes() + { + $ni = $this->whoGets(); + + Inbox::bulkInsert($this->id, array_keys($ni)); return; } diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index e350e6e2f..6c328e685 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -1,7 +1,7 @@ <?php /* * StatusNet - the distributed open-source microblogging tool - * Copyright (C) 2008, 2009, StatusNet, Inc. + * Copyright (C) 2008-2010, StatusNet, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -17,7 +17,9 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } +if (!defined('STATUSNET')) { + exit(1); +} require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; @@ -55,139 +57,31 @@ class Notice_inbox extends Memcached_DataObject function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) { - return Notice::stream(array('Notice_inbox', '_streamDirect'), - array($user_id, $own), - ($own) ? 'notice_inbox:by_user:'.$user_id : - 'notice_inbox:by_user_own:'.$user_id, - $offset, $limit, $since_id, $max_id, $since); + throw new Exception('Notice_inbox no longer used; use Inbox'); } function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id, $since) { - $inbox = new Notice_inbox(); - - $inbox->user_id = $user_id; - - if (!$own) { - $inbox->whereAdd('source != ' . NOTICE_INBOX_SOURCE_GATEWAY); - } - - if ($since_id != 0) { - $inbox->whereAdd('notice_id > ' . $since_id); - } - - if ($max_id != 0) { - $inbox->whereAdd('notice_id <= ' . $max_id); - } - - if (!is_null($since)) { - $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); - } - - $inbox->orderBy('created DESC'); - - if (!is_null($offset)) { - $inbox->limit($offset, $limit); - } - - $ids = array(); - - if ($inbox->find()) { - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - } - - return $ids; + throw new Exception('Notice_inbox no longer used; use Inbox'); } - function pkeyGet($kv) + function &pkeyGet($kv) { return Memcached_DataObject::pkeyGet('Notice_inbox', $kv); } - /** - * Trim inbox for a given user to latest NOTICE_INBOX_LIMIT items - * (up to NOTICE_INBOX_GC_MAX will be deleted). - * - * @param int $user_id - * @return int count of notices dropped from the inbox, if any - */ static function gc($user_id) { - $entry = new Notice_inbox(); - $entry->user_id = $user_id; - $entry->orderBy('created DESC'); - $entry->limit(NOTICE_INBOX_LIMIT - 1, NOTICE_INBOX_GC_MAX); - - $total = $entry->find(); - - if ($total > 0) { - $notices = array(); - $cnt = 0; - while ($entry->fetch()) { - $notices[] = $entry->notice_id; - $cnt++; - if ($cnt >= NOTICE_INBOX_GC_BOXCAR) { - self::deleteMatching($user_id, $notices); - $notices = array(); - $cnt = 0; - } - } - - if ($cnt > 0) { - self::deleteMatching($user_id, $notices); - $notices = array(); - } - } - - return $total; + throw new Exception('Notice_inbox no longer used; use Inbox'); } static function deleteMatching($user_id, $notices) { - $entry = new Notice_inbox(); - return $entry->query('DELETE FROM notice_inbox '. - 'WHERE user_id = ' . $user_id . ' ' . - 'AND notice_id in ('.implode(',', $notices).')'); + throw new Exception('Notice_inbox no longer used; use Inbox'); } static function bulkInsert($notice_id, $created, $ni) { - $cnt = 0; - - $qryhdr = 'INSERT INTO notice_inbox (user_id, notice_id, source, created) VALUES '; - $qry = $qryhdr; - - foreach ($ni as $id => $source) { - if ($cnt > 0) { - $qry .= ', '; - } - $qry .= '('.$id.', '.$notice_id.', '.$source.", '".$created. "') "; - $cnt++; - if (rand() % NOTICE_INBOX_SOFT_LIMIT == 0) { - // FIXME: Causes lag in replicated servers - // Notice_inbox::gc($id); - } - if ($cnt >= MAX_BOXCARS) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - $qry = $qryhdr; - $cnt = 0; - } - } - - if ($cnt > 0) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - } - - return; + throw new Exception('Notice_inbox no longer used; use Inbox'); } } diff --git a/classes/User.php b/classes/User.php index 34151778c..bde3f71b9 100644 --- a/classes/User.php +++ b/classes/User.php @@ -291,6 +291,20 @@ class User extends Memcached_DataObject return false; } + // Everyone gets an inbox + + $inbox = new Inbox(); + + $inbox->user_id = $user->id; + $inbox->notice_ids = ''; + + $result = $inbox->insert(); + + if (!$result) { + common_log_db_error($inbox, 'INSERT', __FILE__); + return false; + } + // Everyone is subscribed to themself $subscription = new Subscription(); @@ -482,89 +496,30 @@ class User extends Memcached_DataObject function noticesWithFriends($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); - + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); return Notice::getStreamByIds($ids); } function noticeInbox($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); - + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); return Notice::getStreamByIds($ids); } function friendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice::stream(array($this, '_friendsTimelineDirect'), - array(false), - 'user:friends_timeline:'.$this->id, - $offset, $limit, $since_id, $before_id, $since); + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); return Notice::getStreamByIds($ids); } function ownFriendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice::stream(array($this, '_friendsTimelineDirect'), - array(true), - 'user:friends_timeline_own:'.$this->id, - $offset, $limit, $since_id, $before_id, $since); + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); return Notice::getStreamByIds($ids); } - function _friendsTimelineDirect($own, $offset, $limit, $since_id, $max_id, $since) - { - $qry = - 'SELECT notice.id AS id ' . - 'FROM notice JOIN notice_inbox ON notice.id = notice_inbox.notice_id ' . - 'WHERE notice_inbox.user_id = ' . $this->id . ' ' . - 'AND notice.repeat_of IS NULL '; - - if (!$own) { - // XXX: autoload notice inbox for constant - $inbox = new Notice_inbox(); - - $qry .= 'AND notice_inbox.source != ' . NOTICE_INBOX_SOURCE_GATEWAY . ' '; - } - - if ($since_id != 0) { - $qry .= 'AND notice.id > ' . $since_id . ' '; - } - - if ($max_id != 0) { - $qry .= 'AND notice.id <= ' . $max_id . ' '; - } - - if (!is_null($since)) { - $qry .= 'AND notice.modified > \'' . date('Y-m-d H:i:s', $since) . '\' '; - } - - // NOTE: we sort by fave time, not by notice time! - - $qry .= 'ORDER BY notice_id DESC '; - - if (!is_null($offset)) { - $qry .= "LIMIT $limit OFFSET $offset"; - } - - $ids = array(); - - $notice = new Notice(); - - $notice->query($qry); - - while ($notice->fetch()) { - $ids[] = $notice->id; - } - - $notice->free(); - $notice = NULL; - - return $ids; - } - function blowFavesCache() { $cache = common_memcache(); @@ -777,7 +732,6 @@ class User extends Memcached_DataObject 'Remember_me', 'Foreign_link', 'Invitation', - 'Notice_inbox', ); Event::handle('UserDeleteRelated', array($this, &$related)); diff --git a/classes/statusnet.ini b/classes/statusnet.ini index 0db2c5d6e..73727a6d6 100644 --- a/classes/statusnet.ini +++ b/classes/statusnet.ini @@ -241,6 +241,13 @@ address = 130 address_type = 130 created = 142 +[inbox] +user_id = 129 +notice_ids = 66 + +[inbox__keys] +user_id = K + [invitation__keys] code = K diff --git a/db/statusnet.sql b/db/statusnet.sql index 94b03df63..cb33ccf33 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -596,3 +596,11 @@ create table user_location_prefs ( constraint primary key (user_id) ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; +create table inbox ( + + user_id integer not null comment 'user receiving the notice' references user (id), + notice_ids blob comment 'packed list of notice ids', + + constraint primary key (user_id) + +) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; diff --git a/plugins/PubSubHubBub/PubSubHubBubPlugin.php b/plugins/PubSubHubBub/PubSubHubBubPlugin.php index d15a869cb..d9c047de8 100644 --- a/plugins/PubSubHubBub/PubSubHubBubPlugin.php +++ b/plugins/PubSubHubBub/PubSubHubBubPlugin.php @@ -95,14 +95,16 @@ class PubSubHubBubPlugin extends Plugin } //feed of each user that subscribes to the notice's author - $notice_inbox = new Notice_inbox(); - $notice_inbox->notice_id = $notice->id; - if ($notice_inbox->find()) { - while ($notice_inbox->fetch()) { - $user = User::staticGet('id',$notice_inbox->user_id); - $feeds[]=common_local_url('ApiTimelineUser',array('id' => $user->nickname, 'format'=>'rss')); - $feeds[]=common_local_url('ApiTimelineUser',array('id' => $user->nickname, 'format'=>'atom')); + + $ni = $notice->whoGets(); + + foreach (array_keys($ni) as $user_id) { + $user = User::staticGet('id', $user_id); + if (empty($user)) { + continue; } + $feeds[]=common_local_url('ApiTimelineUser',array('id' => $user->nickname, 'format'=>'rss')); + $feeds[]=common_local_url('ApiTimelineUser',array('id' => $user->nickname, 'format'=>'atom')); } //feed of user replied to diff --git a/plugins/Realtime/RealtimePlugin.php b/plugins/Realtime/RealtimePlugin.php index 21e465b53..89640f5be 100644 --- a/plugins/Realtime/RealtimePlugin.php +++ b/plugins/Realtime/RealtimePlugin.php @@ -154,14 +154,11 @@ class RealtimePlugin extends Plugin // Add to inbox timelines // XXX: do a join - $inbox = new Notice_inbox(); - $inbox->notice_id = $notice->id; + $ni = $notice->whoGets(); - if ($inbox->find()) { - while ($inbox->fetch()) { - $user = User::staticGet('id', $inbox->user_id); - $paths[] = array('all', $user->nickname); - } + foreach (array_keys($ni) as $user_id) { + $user = User::staticGet('id', $user_id); + $paths[] = array('all', $user->nickname); } // Add to the replies timeline diff --git a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php index b4ca12be2..36732ce46 100755 --- a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php +++ b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php @@ -268,19 +268,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon } - if (!Notice_inbox::pkeyGet(array('notice_id' => $notice->id, - 'user_id' => $flink->user_id))) { - // Add to inbox - $inbox = new Notice_inbox(); - - $inbox->user_id = $flink->user_id; - $inbox->notice_id = $notice->id; - $inbox->created = $notice->created; - $inbox->source = NOTICE_INBOX_SOURCE_GATEWAY; // From a private source - - $inbox->insert(); - - } + Inbox::insertNotice($flink->user_id, $notice->id); $notice->blowCaches(); diff --git a/scripts/triminboxes.php b/scripts/triminboxes.php deleted file mode 100644 index ea4751305..000000000 --- a/scripts/triminboxes.php +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env php -<?php -/* - * StatusNet - the distributed open-source microblogging tool - * Copyright (C) 2009, StatusNet, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); - -$shortoptions = 'u::'; -$longoptions = array('start-user-id=', 'sleep-time='); - -$helptext = <<<END_OF_TRIM_HELP -Batch script for trimming notice inboxes to a reasonable size. - - -u <id> - --start-user-id=<id> User ID to start after. Default is all. - --sleep-time=<integer> Amount of time to wait (in seconds) between trims. Default is zero. - -END_OF_TRIM_HELP; - -require_once INSTALLDIR.'/scripts/commandline.inc'; - -$id = null; -$sleep_time = 0; - -if (have_option('u')) { - $id = get_option_value('u'); -} else if (have_option('--start-user-id')) { - $id = get_option_value('--start-user-id'); -} else { - $id = null; -} - -if (have_option('--sleep-time')) { - $sleep_time = intval(get_option_value('--sleep-time')); -} - -$quiet = have_option('q') || have_option('--quiet'); - -$user = new User(); - -if (!empty($id)) { - $user->whereAdd('id > ' . $id); -} - -$cnt = $user->find(); - -while ($user->fetch()) { - if (!$quiet) { - print "Trimming inbox for user $user->id"; - } - $count = Notice_inbox::gc($user->id); - if ($count) { - if (!$quiet) { - print ": $count trimmed..."; - } - sleep($sleep_time); - } - if (!$quiet) { - print "\n"; - } -} |