diff options
| -rw-r--r-- | classes/Notice_inbox.php | 184 | ||||
| -rwxr-xr-x | scripts/inbox_users.php | 107 | ||||
| -rw-r--r-- | scripts/triminboxes.php | 56 |
3 files changed, 347 insertions, 0 deletions
diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php new file mode 100644 index 000000000..b39006542 --- /dev/null +++ b/classes/Notice_inbox.php @@ -0,0 +1,184 @@ +<?php +/* + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2008, 2009, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } + +require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; + +// We keep 5 pages of inbox notices in memcache, +1 for pagination check + +define('INBOX_CACHE_WINDOW', 101); +define('NOTICE_INBOX_GC_BOXCAR', 128); +define('NOTICE_INBOX_GC_MAX', 12800); +define('NOTICE_INBOX_LIMIT', 1000); +define('NOTICE_INBOX_SOFT_LIMIT', 1000); + +define('NOTICE_INBOX_SOURCE_SUB', 1); +define('NOTICE_INBOX_SOURCE_GROUP', 2); +define('NOTICE_INBOX_SOURCE_REPLY', 3); +define('NOTICE_INBOX_SOURCE_FORWARD', 4); +define('NOTICE_INBOX_SOURCE_GATEWAY', -1); + +class Notice_inbox extends Memcached_DataObject +{ + ###START_AUTOCODE + /* the code below is auto generated do not remove the above tag */ + + public $__table = 'notice_inbox'; // table name + public $user_id; // int(4) primary_key not_null + public $notice_id; // int(4) primary_key not_null + public $created; // datetime() not_null + public $source; // tinyint(1) default_1 + + /* Static get */ + function staticGet($k,$v=null) + { return Memcached_DataObject::staticGet('Notice_inbox',$k,$v); } + + /* the code above is auto generated do not remove the tag below */ + ###END_AUTOCODE + + function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) + { + return Notice::stream(array('Notice_inbox', '_streamDirect'), + array($user_id, $own), + ($own) ? 'notice_inbox:by_user:'.$user_id : + 'notice_inbox:by_user_own:'.$user_id, + $offset, $limit, $since_id, $max_id, $since); + } + + function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id, $since) + { + $inbox = new Notice_inbox(); + + $inbox->user_id = $user_id; + + if (!$own) { + $inbox->whereAdd('source != ' . NOTICE_INBOX_SOURCE_GATEWAY); + } + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($max_id != 0) { + $inbox->whereAdd('notice_id <= ' . $max_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('created DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; + } + + function &pkeyGet($kv) + { + return Memcached_DataObject::pkeyGet('Notice_inbox', $kv); + } + + static function gc($user_id) + { + $entry = new Notice_inbox(); + $entry->user_id = $user_id; + $entry->orderBy('created DESC'); + $entry->limit(NOTICE_INBOX_LIMIT - 1, NOTICE_INBOX_GC_MAX); + + $total = $entry->find(); + + if ($total > 0) { + $notices = array(); + $cnt = 0; + while ($entry->fetch()) { + $notices[] = $entry->notice_id; + $cnt++; + if ($cnt >= NOTICE_INBOX_GC_BOXCAR) { + self::deleteMatching($user_id, $notices); + $notices = array(); + $cnt = 0; + } + } + + if ($cnt > 0) { + self::deleteMatching($user_id, $notices); + $notices = array(); + } + } + } + + static function deleteMatching($user_id, $notices) + { + $entry = new Notice_inbox(); + return $entry->query('DELETE FROM notice_inbox '. + 'WHERE user_id = ' . $user_id . ' ' . + 'AND notice_id in ('.implode(',', $notices).')'); + } + + static function bulkInsert($notice_id, $created, $ni) + { + $cnt = 0; + + $qryhdr = 'INSERT INTO notice_inbox (user_id, notice_id, source, created) VALUES '; + $qry = $qryhdr; + + foreach ($ni as $id => $source) { + if ($cnt > 0) { + $qry .= ', '; + } + $qry .= '('.$id.', '.$notice_id.', '.$source.", '".$created. "') "; + $cnt++; + if (rand() % NOTICE_INBOX_SOFT_LIMIT == 0) { + // FIXME: Causes lag in replicated servers + // Notice_inbox::gc($id); + } + if ($cnt >= MAX_BOXCARS) { + $inbox = new Notice_inbox(); + $result = $inbox->query($qry); + if (PEAR::isError($result)) { + common_log_db_error($inbox, $qry); + } + $qry = $qryhdr; + $cnt = 0; + } + } + + if ($cnt > 0) { + $inbox = new Notice_inbox(); + $result = $inbox->query($qry); + if (PEAR::isError($result)) { + common_log_db_error($inbox, $qry); + } + } + + return; + } +} diff --git a/scripts/inbox_users.php b/scripts/inbox_users.php new file mode 100755 index 000000000..32adcea21 --- /dev/null +++ b/scripts/inbox_users.php @@ -0,0 +1,107 @@ +#!/usr/bin/env php +<?php +/* + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2008, 2009, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +# Abort if called from a web server + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$helptext = <<<ENDOFHELP +inbox_users.php <idfile> + +Update users to use inbox table. Listed in an ID file, default 'ids.txt'. + +ENDOFHELP; + +require_once INSTALLDIR.'/scripts/commandline.inc'; + +$id_file = (count($args) > 1) ? $args[0] : 'ids.txt'; + +common_log(LOG_INFO, 'Updating user inboxes.'); + +$ids = file($id_file); + +foreach ($ids as $id) { + + $user = User::staticGet('id', $id); + + if (!$user) { + common_log(LOG_WARNING, 'No such user: ' . $id); + continue; + } + + if ($user->inboxed) { + common_log(LOG_WARNING, 'Already inboxed: ' . $id); + continue; + } + + common_log(LOG_INFO, 'Updating inbox for user ' . $user->id); + + $user->query('BEGIN'); + + $old_inbox = new Notice_inbox(); + $old_inbox->user_id = $user->id; + + $result = $old_inbox->delete(); + + if (is_null($result) || $result === false) { + common_log_db_error($old_inbox, 'DELETE', __FILE__); + continue; + } + + $old_inbox->free(); + + $inbox = new Notice_inbox(); + + $result = $inbox->query('INSERT INTO notice_inbox (user_id, notice_id, created) ' . + 'SELECT ' . $user->id . ', notice.id, notice.created ' . + 'FROM subscription JOIN notice ON subscription.subscribed = notice.profile_id ' . + 'WHERE subscription.subscriber = ' . $user->id . ' ' . + 'AND notice.created >= subscription.created ' . + 'AND NOT EXISTS (SELECT user_id, notice_id ' . + 'FROM notice_inbox ' . + 'WHERE user_id = ' . $user->id . ' ' . + 'AND notice_id = notice.id) ' . + 'ORDER BY notice.created DESC ' . + 'LIMIT 0, 1000'); + + if (is_null($result) || $result === false) { + common_log_db_error($inbox, 'INSERT', __FILE__); + continue; + } + + $orig = clone($user); + $user->inboxed = 1; + $result = $user->update($orig); + + if (!$result) { + common_log_db_error($user, 'UPDATE', __FILE__); + continue; + } + + $user->query('COMMIT'); + + $inbox->free(); + unset($inbox); + + if ($cache) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + } +} diff --git a/scripts/triminboxes.php b/scripts/triminboxes.php new file mode 100644 index 000000000..da09817e5 --- /dev/null +++ b/scripts/triminboxes.php @@ -0,0 +1,56 @@ +#!/usr/bin/env php +<?php +/* + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2009, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$shortoptions = 'u::'; +$longoptions = array('start-user-id::'); + +$helptext = <<<END_OF_TRIM_HELP +Batch script for trimming notice inboxes to a reasonable size. + + -u <id> + --start-user-id=<id> User ID to start after. Default is all. + +END_OF_TRIM_HELP; + +require_once INSTALLDIR.'/scripts/commandline.inc'; + +$id = null; + +if (have_option('u')) { + $id = get_option_value('u'); +} else if (have_option('--start-user-id')) { + $id = get_option_value('--start-user-id'); +} else { + $id = null; +} + +$user = new User(); + +if (!empty($id)) { + $user->whereAdd('id > ' . $id); +} + +$cnt = $user->find(); + +while ($user->fetch()) { + Notice_inbox::gc($user->id); +} |