From 7ec27b657e6b7ad9416e531cf70fffcf48614708 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 30 Dec 2009 09:03:06 -1000 Subject: Replace Notice_inbox with Inbox --- classes/Notice_inbox.php | 193 ----------------------------------------------- 1 file changed, 193 deletions(-) delete mode 100644 classes/Notice_inbox.php (limited to 'classes/Notice_inbox.php') diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php deleted file mode 100644 index e350e6e2f..000000000 --- a/classes/Notice_inbox.php +++ /dev/null @@ -1,193 +0,0 @@ -. - */ - -if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } - -require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; - -// We keep 5 pages of inbox notices in memcache, +1 for pagination check - -define('INBOX_CACHE_WINDOW', 101); -define('NOTICE_INBOX_GC_BOXCAR', 128); -define('NOTICE_INBOX_GC_MAX', 12800); -define('NOTICE_INBOX_LIMIT', 1000); -define('NOTICE_INBOX_SOFT_LIMIT', 1000); - -define('NOTICE_INBOX_SOURCE_SUB', 1); -define('NOTICE_INBOX_SOURCE_GROUP', 2); -define('NOTICE_INBOX_SOURCE_REPLY', 3); -define('NOTICE_INBOX_SOURCE_FORWARD', 4); -define('NOTICE_INBOX_SOURCE_GATEWAY', -1); - -class Notice_inbox extends Memcached_DataObject -{ - ###START_AUTOCODE - /* the code below is auto generated do not remove the above tag */ - - public $__table = 'notice_inbox'; // table name - public $user_id; // int(4) primary_key not_null - public $notice_id; // int(4) primary_key not_null - public $created; // datetime() not_null - public $source; // tinyint(1) default_1 - - /* Static get */ - function staticGet($k,$v=null) - { return Memcached_DataObject::staticGet('Notice_inbox',$k,$v); } - - /* the code above is auto generated do not remove the tag below */ - ###END_AUTOCODE - - function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) - { - return Notice::stream(array('Notice_inbox', '_streamDirect'), - array($user_id, $own), - ($own) ? 'notice_inbox:by_user:'.$user_id : - 'notice_inbox:by_user_own:'.$user_id, - $offset, $limit, $since_id, $max_id, $since); - } - - function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id, $since) - { - $inbox = new Notice_inbox(); - - $inbox->user_id = $user_id; - - if (!$own) { - $inbox->whereAdd('source != ' . NOTICE_INBOX_SOURCE_GATEWAY); - } - - if ($since_id != 0) { - $inbox->whereAdd('notice_id > ' . $since_id); - } - - if ($max_id != 0) { - $inbox->whereAdd('notice_id <= ' . $max_id); - } - - if (!is_null($since)) { - $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); - } - - $inbox->orderBy('created DESC'); - - if (!is_null($offset)) { - $inbox->limit($offset, $limit); - } - - $ids = array(); - - if ($inbox->find()) { - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - } - - return $ids; - } - - function pkeyGet($kv) - { - return Memcached_DataObject::pkeyGet('Notice_inbox', $kv); - } - - /** - * Trim inbox for a given user to latest NOTICE_INBOX_LIMIT items - * (up to NOTICE_INBOX_GC_MAX will be deleted). - * - * @param int $user_id - * @return int count of notices dropped from the inbox, if any - */ - static function gc($user_id) - { - $entry = new Notice_inbox(); - $entry->user_id = $user_id; - $entry->orderBy('created DESC'); - $entry->limit(NOTICE_INBOX_LIMIT - 1, NOTICE_INBOX_GC_MAX); - - $total = $entry->find(); - - if ($total > 0) { - $notices = array(); - $cnt = 0; - while ($entry->fetch()) { - $notices[] = $entry->notice_id; - $cnt++; - if ($cnt >= NOTICE_INBOX_GC_BOXCAR) { - self::deleteMatching($user_id, $notices); - $notices = array(); - $cnt = 0; - } - } - - if ($cnt > 0) { - self::deleteMatching($user_id, $notices); - $notices = array(); - } - } - - return $total; - } - - static function deleteMatching($user_id, $notices) - { - $entry = new Notice_inbox(); - return $entry->query('DELETE FROM notice_inbox '. - 'WHERE user_id = ' . $user_id . ' ' . - 'AND notice_id in ('.implode(',', $notices).')'); - } - - static function bulkInsert($notice_id, $created, $ni) - { - $cnt = 0; - - $qryhdr = 'INSERT INTO notice_inbox (user_id, notice_id, source, created) VALUES '; - $qry = $qryhdr; - - foreach ($ni as $id => $source) { - if ($cnt > 0) { - $qry .= ', '; - } - $qry .= '('.$id.', '.$notice_id.', '.$source.", '".$created. "') "; - $cnt++; - if (rand() % NOTICE_INBOX_SOFT_LIMIT == 0) { - // FIXME: Causes lag in replicated servers - // Notice_inbox::gc($id); - } - if ($cnt >= MAX_BOXCARS) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - $qry = $qryhdr; - $cnt = 0; - } - } - - if ($cnt > 0) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - } - - return; - } -} -- cgit v1.2.3-54-g00ecf From 72934e9f50ee8970372b39b3ff27dc4ded30e0c5 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sat, 9 Jan 2010 09:31:57 -0800 Subject: Revert "Replace Notice_inbox with Inbox" We use Notice_inbox to transition to Inbox. This reverts commit 7640d3f07bad0710d69575efc7ceda115f24a60a. --- classes/Notice_inbox.php | 184 +++++++++++++++++++++++++++++++++++++++++++++++ scripts/inbox_users.php | 107 +++++++++++++++++++++++++++ scripts/triminboxes.php | 56 +++++++++++++++ 3 files changed, 347 insertions(+) create mode 100644 classes/Notice_inbox.php create mode 100755 scripts/inbox_users.php create mode 100644 scripts/triminboxes.php (limited to 'classes/Notice_inbox.php') diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php new file mode 100644 index 000000000..b39006542 --- /dev/null +++ b/classes/Notice_inbox.php @@ -0,0 +1,184 @@ +. + */ + +if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } + +require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; + +// We keep 5 pages of inbox notices in memcache, +1 for pagination check + +define('INBOX_CACHE_WINDOW', 101); +define('NOTICE_INBOX_GC_BOXCAR', 128); +define('NOTICE_INBOX_GC_MAX', 12800); +define('NOTICE_INBOX_LIMIT', 1000); +define('NOTICE_INBOX_SOFT_LIMIT', 1000); + +define('NOTICE_INBOX_SOURCE_SUB', 1); +define('NOTICE_INBOX_SOURCE_GROUP', 2); +define('NOTICE_INBOX_SOURCE_REPLY', 3); +define('NOTICE_INBOX_SOURCE_FORWARD', 4); +define('NOTICE_INBOX_SOURCE_GATEWAY', -1); + +class Notice_inbox extends Memcached_DataObject +{ + ###START_AUTOCODE + /* the code below is auto generated do not remove the above tag */ + + public $__table = 'notice_inbox'; // table name + public $user_id; // int(4) primary_key not_null + public $notice_id; // int(4) primary_key not_null + public $created; // datetime() not_null + public $source; // tinyint(1) default_1 + + /* Static get */ + function staticGet($k,$v=null) + { return Memcached_DataObject::staticGet('Notice_inbox',$k,$v); } + + /* the code above is auto generated do not remove the tag below */ + ###END_AUTOCODE + + function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) + { + return Notice::stream(array('Notice_inbox', '_streamDirect'), + array($user_id, $own), + ($own) ? 'notice_inbox:by_user:'.$user_id : + 'notice_inbox:by_user_own:'.$user_id, + $offset, $limit, $since_id, $max_id, $since); + } + + function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id, $since) + { + $inbox = new Notice_inbox(); + + $inbox->user_id = $user_id; + + if (!$own) { + $inbox->whereAdd('source != ' . NOTICE_INBOX_SOURCE_GATEWAY); + } + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($max_id != 0) { + $inbox->whereAdd('notice_id <= ' . $max_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('created DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; + } + + function &pkeyGet($kv) + { + return Memcached_DataObject::pkeyGet('Notice_inbox', $kv); + } + + static function gc($user_id) + { + $entry = new Notice_inbox(); + $entry->user_id = $user_id; + $entry->orderBy('created DESC'); + $entry->limit(NOTICE_INBOX_LIMIT - 1, NOTICE_INBOX_GC_MAX); + + $total = $entry->find(); + + if ($total > 0) { + $notices = array(); + $cnt = 0; + while ($entry->fetch()) { + $notices[] = $entry->notice_id; + $cnt++; + if ($cnt >= NOTICE_INBOX_GC_BOXCAR) { + self::deleteMatching($user_id, $notices); + $notices = array(); + $cnt = 0; + } + } + + if ($cnt > 0) { + self::deleteMatching($user_id, $notices); + $notices = array(); + } + } + } + + static function deleteMatching($user_id, $notices) + { + $entry = new Notice_inbox(); + return $entry->query('DELETE FROM notice_inbox '. + 'WHERE user_id = ' . $user_id . ' ' . + 'AND notice_id in ('.implode(',', $notices).')'); + } + + static function bulkInsert($notice_id, $created, $ni) + { + $cnt = 0; + + $qryhdr = 'INSERT INTO notice_inbox (user_id, notice_id, source, created) VALUES '; + $qry = $qryhdr; + + foreach ($ni as $id => $source) { + if ($cnt > 0) { + $qry .= ', '; + } + $qry .= '('.$id.', '.$notice_id.', '.$source.", '".$created. "') "; + $cnt++; + if (rand() % NOTICE_INBOX_SOFT_LIMIT == 0) { + // FIXME: Causes lag in replicated servers + // Notice_inbox::gc($id); + } + if ($cnt >= MAX_BOXCARS) { + $inbox = new Notice_inbox(); + $result = $inbox->query($qry); + if (PEAR::isError($result)) { + common_log_db_error($inbox, $qry); + } + $qry = $qryhdr; + $cnt = 0; + } + } + + if ($cnt > 0) { + $inbox = new Notice_inbox(); + $result = $inbox->query($qry); + if (PEAR::isError($result)) { + common_log_db_error($inbox, $qry); + } + } + + return; + } +} diff --git a/scripts/inbox_users.php b/scripts/inbox_users.php new file mode 100755 index 000000000..32adcea21 --- /dev/null +++ b/scripts/inbox_users.php @@ -0,0 +1,107 @@ +#!/usr/bin/env php +. + */ + +# Abort if called from a web server + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$helptext = << + +Update users to use inbox table. Listed in an ID file, default 'ids.txt'. + +ENDOFHELP; + +require_once INSTALLDIR.'/scripts/commandline.inc'; + +$id_file = (count($args) > 1) ? $args[0] : 'ids.txt'; + +common_log(LOG_INFO, 'Updating user inboxes.'); + +$ids = file($id_file); + +foreach ($ids as $id) { + + $user = User::staticGet('id', $id); + + if (!$user) { + common_log(LOG_WARNING, 'No such user: ' . $id); + continue; + } + + if ($user->inboxed) { + common_log(LOG_WARNING, 'Already inboxed: ' . $id); + continue; + } + + common_log(LOG_INFO, 'Updating inbox for user ' . $user->id); + + $user->query('BEGIN'); + + $old_inbox = new Notice_inbox(); + $old_inbox->user_id = $user->id; + + $result = $old_inbox->delete(); + + if (is_null($result) || $result === false) { + common_log_db_error($old_inbox, 'DELETE', __FILE__); + continue; + } + + $old_inbox->free(); + + $inbox = new Notice_inbox(); + + $result = $inbox->query('INSERT INTO notice_inbox (user_id, notice_id, created) ' . + 'SELECT ' . $user->id . ', notice.id, notice.created ' . + 'FROM subscription JOIN notice ON subscription.subscribed = notice.profile_id ' . + 'WHERE subscription.subscriber = ' . $user->id . ' ' . + 'AND notice.created >= subscription.created ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + 'WHERE user_id = ' . $user->id . ' ' . + 'AND notice_id = notice.id) ' . + 'ORDER BY notice.created DESC ' . + 'LIMIT 0, 1000'); + + if (is_null($result) || $result === false) { + common_log_db_error($inbox, 'INSERT', __FILE__); + continue; + } + + $orig = clone($user); + $user->inboxed = 1; + $result = $user->update($orig); + + if (!$result) { + common_log_db_error($user, 'UPDATE', __FILE__); + continue; + } + + $user->query('COMMIT'); + + $inbox->free(); + unset($inbox); + + if ($cache) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + } +} diff --git a/scripts/triminboxes.php b/scripts/triminboxes.php new file mode 100644 index 000000000..da09817e5 --- /dev/null +++ b/scripts/triminboxes.php @@ -0,0 +1,56 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$shortoptions = 'u::'; +$longoptions = array('start-user-id::'); + +$helptext = << + --start-user-id= User ID to start after. Default is all. + +END_OF_TRIM_HELP; + +require_once INSTALLDIR.'/scripts/commandline.inc'; + +$id = null; + +if (have_option('u')) { + $id = get_option_value('u'); +} else if (have_option('--start-user-id')) { + $id = get_option_value('--start-user-id'); +} else { + $id = null; +} + +$user = new User(); + +if (!empty($id)) { + $user->whereAdd('id > ' . $id); +} + +$cnt = $user->find(); + +while ($user->fetch()) { + Notice_inbox::gc($user->id); +} -- cgit v1.2.3-54-g00ecf From 1de0f37f0d6e4706fe9f1ce086df730ad4a05b78 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sat, 9 Jan 2010 09:59:58 -0800 Subject: disable all Notice_inbox functions --- classes/Notice_inbox.php | 115 ++++------------------------------------------- 1 file changed, 9 insertions(+), 106 deletions(-) (limited to 'classes/Notice_inbox.php') diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index b39006542..6c328e685 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -1,7 +1,7 @@ . */ -if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } +if (!defined('STATUSNET')) { + exit(1); +} require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; @@ -55,50 +57,12 @@ class Notice_inbox extends Memcached_DataObject function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) { - return Notice::stream(array('Notice_inbox', '_streamDirect'), - array($user_id, $own), - ($own) ? 'notice_inbox:by_user:'.$user_id : - 'notice_inbox:by_user_own:'.$user_id, - $offset, $limit, $since_id, $max_id, $since); + throw new Exception('Notice_inbox no longer used; use Inbox'); } function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id, $since) { - $inbox = new Notice_inbox(); - - $inbox->user_id = $user_id; - - if (!$own) { - $inbox->whereAdd('source != ' . NOTICE_INBOX_SOURCE_GATEWAY); - } - - if ($since_id != 0) { - $inbox->whereAdd('notice_id > ' . $since_id); - } - - if ($max_id != 0) { - $inbox->whereAdd('notice_id <= ' . $max_id); - } - - if (!is_null($since)) { - $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); - } - - $inbox->orderBy('created DESC'); - - if (!is_null($offset)) { - $inbox->limit($offset, $limit); - } - - $ids = array(); - - if ($inbox->find()) { - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - } - - return $ids; + throw new Exception('Notice_inbox no longer used; use Inbox'); } function &pkeyGet($kv) @@ -108,77 +72,16 @@ class Notice_inbox extends Memcached_DataObject static function gc($user_id) { - $entry = new Notice_inbox(); - $entry->user_id = $user_id; - $entry->orderBy('created DESC'); - $entry->limit(NOTICE_INBOX_LIMIT - 1, NOTICE_INBOX_GC_MAX); - - $total = $entry->find(); - - if ($total > 0) { - $notices = array(); - $cnt = 0; - while ($entry->fetch()) { - $notices[] = $entry->notice_id; - $cnt++; - if ($cnt >= NOTICE_INBOX_GC_BOXCAR) { - self::deleteMatching($user_id, $notices); - $notices = array(); - $cnt = 0; - } - } - - if ($cnt > 0) { - self::deleteMatching($user_id, $notices); - $notices = array(); - } - } + throw new Exception('Notice_inbox no longer used; use Inbox'); } static function deleteMatching($user_id, $notices) { - $entry = new Notice_inbox(); - return $entry->query('DELETE FROM notice_inbox '. - 'WHERE user_id = ' . $user_id . ' ' . - 'AND notice_id in ('.implode(',', $notices).')'); + throw new Exception('Notice_inbox no longer used; use Inbox'); } static function bulkInsert($notice_id, $created, $ni) { - $cnt = 0; - - $qryhdr = 'INSERT INTO notice_inbox (user_id, notice_id, source, created) VALUES '; - $qry = $qryhdr; - - foreach ($ni as $id => $source) { - if ($cnt > 0) { - $qry .= ', '; - } - $qry .= '('.$id.', '.$notice_id.', '.$source.", '".$created. "') "; - $cnt++; - if (rand() % NOTICE_INBOX_SOFT_LIMIT == 0) { - // FIXME: Causes lag in replicated servers - // Notice_inbox::gc($id); - } - if ($cnt >= MAX_BOXCARS) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - $qry = $qryhdr; - $cnt = 0; - } - } - - if ($cnt > 0) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - } - - return; + throw new Exception('Notice_inbox no longer used; use Inbox'); } } -- cgit v1.2.3-54-g00ecf From 0e1f2d4b47e5e340679c4245b62e1d64c6b9c9b9 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 13 Jan 2010 18:17:36 +0000 Subject: Fix regressions in XMPP output during inbox refactoring - NOTICE_INBOX_SOURCE_* constants moved to common.php since Notice_inbox.php not always loaded - fixed typo in User::staticGet() call which caused user #1 to receive messages once for each subscriber instead of for him/herself - 'continue' -> 'continue 2' inside switch() statement to fix loop escape (PHP considers switch() a looping construct for break & continue) --- classes/Notice_inbox.php | 6 ------ lib/common.php | 6 ++++++ lib/jabber.php | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'classes/Notice_inbox.php') diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 6c328e685..c27dcdfd6 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -31,12 +31,6 @@ define('NOTICE_INBOX_GC_MAX', 12800); define('NOTICE_INBOX_LIMIT', 1000); define('NOTICE_INBOX_SOFT_LIMIT', 1000); -define('NOTICE_INBOX_SOURCE_SUB', 1); -define('NOTICE_INBOX_SOURCE_GROUP', 2); -define('NOTICE_INBOX_SOURCE_REPLY', 3); -define('NOTICE_INBOX_SOURCE_FORWARD', 4); -define('NOTICE_INBOX_SOURCE_GATEWAY', -1); - class Notice_inbox extends Memcached_DataObject { ###START_AUTOCODE diff --git a/lib/common.php b/lib/common.php index 61decebb7..00e80373e 100644 --- a/lib/common.php +++ b/lib/common.php @@ -41,6 +41,12 @@ define('FOREIGN_NOTICE_SEND_REPLY', 4); define('FOREIGN_FRIEND_SEND', 1); define('FOREIGN_FRIEND_RECV', 2); +define('NOTICE_INBOX_SOURCE_SUB', 1); +define('NOTICE_INBOX_SOURCE_GROUP', 2); +define('NOTICE_INBOX_SOURCE_REPLY', 3); +define('NOTICE_INBOX_SOURCE_FORWARD', 4); +define('NOTICE_INBOX_SOURCE_GATEWAY', -1); + # append our extlib dir as the last-resort place to find libs set_include_path(get_include_path() . PATH_SEPARATOR . INSTALLDIR . '/extlib/'); diff --git a/lib/jabber.php b/lib/jabber.php index 69f0d3570..4cdfa6746 100644 --- a/lib/jabber.php +++ b/lib/jabber.php @@ -359,7 +359,7 @@ function jabber_broadcast_notice($notice) $ni = $notice->whoGets(); foreach ($ni as $user_id => $reason) { - $user = User::staticGet('user_id', $user_id); + $user = User::staticGet($user_id); if (empty($user) || empty($user->jabber) || !$user->jabbernotify) { @@ -369,20 +369,20 @@ function jabber_broadcast_notice($notice) switch ($reason) { case NOTICE_INBOX_SOURCE_REPLY: if (!$user->jabberreplies) { - continue; + continue 2; } break; case NOTICE_INBOX_SOURCE_SUB: $sub = Subscription::pkeyGet(array('subscriber' => $user->id, 'subscribed' => $notice->profile_id)); if (empty($sub) || !$sub->jabber) { - continue; + continue 2; } break; case NOTICE_INBOX_SOURCE_GROUP: break; default: - throw new Exception(_("Unknown inbox source.")); + throw new Exception(sprintf(_("Unknown inbox source %d."), $reason)); } common_log(LOG_INFO, -- cgit v1.2.3-54-g00ecf