From 7746611f20554c92ad0213de196e880e41c3cc4e Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 5 Jul 2010 14:12:14 -0400 Subject: create an index on user.created so sitemap generation is faster --- plugins/Sitemap/SitemapPlugin.php | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'plugins') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index d4d295237..b6d3b1ad3 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -202,6 +202,12 @@ class SitemapPlugin extends Plugin null, false), new ColumnDef('modified', 'timestamp'))); + $userCreated = $schema->getColumnDef('user', 'created'); + + if (empty($userCreated) || $userCreated->key != 'MUL') { + $schema->createIndex('user', 'created'); + } + return true; } -- cgit v1.2.3-54-g00ecf From 4c5098cd32599a2c376beaadb43cd9d471477c90 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 8 Jul 2010 21:17:10 +0000 Subject: Handle the case where a screen name has shifted from one Twitter ID to another --- classes/Foreign_user.php | 16 ++++++++++++++++ plugins/TwitterBridge/twitter.php | 21 +++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/classes/Foreign_user.php b/classes/Foreign_user.php index 0dd94ffb9..e98a16064 100644 --- a/classes/Foreign_user.php +++ b/classes/Foreign_user.php @@ -39,6 +39,22 @@ class Foreign_user extends Memcached_DataObject return null; } + static function getByNickname($nickname, $service) + { + if (empty($nickname) || empty($service)) { + return null; + } else { + $fuser = new Foreign_user(); + $fuser->service = $service; + $fuser->nickname = $nickname; + $fuser->limit(1); + + $result = $fuser->find(true); + + return empty($result) ? null : $fuser; + } + } + function updateKeys(&$orig) { $this->_connect(); diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index 896eee2da..306ba2442 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -75,8 +75,6 @@ function save_twitter_user($twitter_id, $screen_name) if (!empty($fuser)) { - $result = true; - // Delete old record if Twitter user changed screen name if ($fuser->nickname != $screen_name) { @@ -88,6 +86,25 @@ function save_twitter_user($twitter_id, $screen_name) $screen_name, $oldname)); } + + } else { + + // Kill any old, invalid records for this screen name + + $fuser = Foreign_user::getByNickname($screen_name, TWITTER_SERVICE); + + if (!empty($fuser)) { + $fuser->delete(); + common_log( + LOG_INFO, + sprintf( + 'Twitter bridge - deteted old record for Twitter ' . + 'screen name "%s" belonging to Twitter ID %d.', + $screen_name, + $fuser->id + ) + ); + } } return add_twitter_user($twitter_id, $screen_name); -- cgit v1.2.3-54-g00ecf From 869ebb7dc7a5f4241b94512646cdd7ae89e35962 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 9 Jul 2010 17:50:14 -0400 Subject: script to fill counts table from command line --- plugins/Sitemap/scripts/updatecounts.php | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 plugins/Sitemap/scripts/updatecounts.php (limited to 'plugins') diff --git a/plugins/Sitemap/scripts/updatecounts.php b/plugins/Sitemap/scripts/updatecounts.php new file mode 100644 index 000000000..15c986418 --- /dev/null +++ b/plugins/Sitemap/scripts/updatecounts.php @@ -0,0 +1,36 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..')); + +$helptext = << Date: Sat, 10 Jul 2010 09:58:27 -0400 Subject: remove debugging statement and add a bit of output to updatecounts --- plugins/Sitemap/scripts/updatecounts.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/Sitemap/scripts/updatecounts.php b/plugins/Sitemap/scripts/updatecounts.php index 15c986418..91bc0ac4e 100644 --- a/plugins/Sitemap/scripts/updatecounts.php +++ b/plugins/Sitemap/scripts/updatecounts.php @@ -26,11 +26,11 @@ Update the notice and user counts cached in the database. END_OF_UPDATECOUNTS_HELP; -echo "INSTALLDIR=".INSTALLDIR."\n"; - require_once INSTALLDIR.'/scripts/commandline.inc'; // Will fill the cache $userCounts = Sitemap_user_count::getAll(); $noticeCounts = Sitemap_notice_count::getAll(); + +echo "Done.\n"; -- cgit v1.2.3-54-g00ecf From 1339f1f908acf2ad69928494f7dfc5c09c5fb211 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 12 Jul 2010 14:22:36 -0700 Subject: Commit hubprepqueuehandler.php -- fix for OStatus bulk output. --- plugins/OStatus/lib/hubprepqueuehandler.php | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 plugins/OStatus/lib/hubprepqueuehandler.php (limited to 'plugins') diff --git a/plugins/OStatus/lib/hubprepqueuehandler.php b/plugins/OStatus/lib/hubprepqueuehandler.php new file mode 100644 index 000000000..0d585938f --- /dev/null +++ b/plugins/OStatus/lib/hubprepqueuehandler.php @@ -0,0 +1,87 @@ +. + */ + +/** + * When we have a large batch of PuSH consumers, we break the data set + * into smaller chunks. Enqueue final destinations... + * + * @package Hub + * @author Brion Vibber + */ +class HubPrepQueueHandler extends QueueHandler +{ + // Enqueue this many low-level distributions before re-queueing the rest + // of the batch to be processed later. Helps to keep latency down for other + // things happening during a particularly long OStatus delivery session. + // + // [Could probably ditch this if we had working message delivery priorities + // for queueing, but this isn't supported in ActiveMQ 5.3.] + const ROLLING_BATCH = 20; + + function transport() + { + return 'hubprep'; + } + + function handle($data) + { + $topic = $data['topic']; + $atom = $data['atom']; + $pushCallbacks = $data['pushCallbacks']; + + assert(is_string($atom)); + assert(is_string($topic)); + assert(is_array($pushCallbacks)); + + // Set up distribution for the first n subscribing sites... + // If we encounter an uncatchable error, queue handling should + // automatically re-run the batch, which could lead to some dupe + // distributions. + // + // Worst case is if one of these hubprep entries dies too many + // times and gets dropped; the rest of the batch won't get processed. + try { + $n = 0; + while (count($pushCallbacks) && $n < self::ROLLING_BATCH) { + $n++; + $callback = array_shift($pushCallbacks); + $sub = HubSub::staticGet($topic, $callback); + if (!$sub) { + common_log(LOG_ERR, "Skipping PuSH delivery for deleted(?) consumer $callback on $topic"); + continue; + } + + $sub->distribute($atom); + } + } catch (Exception $e) { + common_log(LOG_ERR, "Exception during PuSH batch out: " . + $e->getMessage() . + " prepping $topic to $callback"); + } + + // And re-queue the rest of the batch! + if (count($pushCallbacks) > 0) { + $sub = new HubSub(); + $sub->topic = $topic; + $sub->bulkDistribute($atom, $pushCallbacks); + } + + return true; + } +} -- cgit v1.2.3-54-g00ecf From d73feb82d89d66593fd81f8bb5d10b1873fc9458 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 14 Jul 2010 10:38:34 -0400 Subject: cache sitemap notice and user counts for 4h --- classes/Memcached_DataObject.php | 4 ++-- plugins/Sitemap/Sitemap_notice_count.php | 4 +++- plugins/Sitemap/Sitemap_user_count.php | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'plugins') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 4579f64df..a7fec365e 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -593,7 +593,7 @@ class Memcached_DataObject extends Safe_DataObject return $c->get($cacheKey); } - static function cacheSet($keyPart, $value) + static function cacheSet($keyPart, $value, $flag=null, $expiry=null) { $c = self::memcache(); @@ -603,7 +603,7 @@ class Memcached_DataObject extends Safe_DataObject $cacheKey = common_cache_key($keyPart); - return $c->set($cacheKey, $value); + return $c->set($cacheKey, $value, $flag, $expiry); } static function valueString($v) diff --git a/plugins/Sitemap/Sitemap_notice_count.php b/plugins/Sitemap/Sitemap_notice_count.php index 2a375b3e4..6e0061e97 100644 --- a/plugins/Sitemap/Sitemap_notice_count.php +++ b/plugins/Sitemap/Sitemap_notice_count.php @@ -153,7 +153,9 @@ class Sitemap_notice_count extends Memcached_DataObject $noticeCounts[$snc->notice_date] = $snc->notice_count; } - self::cacheSet('sitemap:notice:counts', $noticeCounts); + // Cache notice counts for 4 hours. + + self::cacheSet('sitemap:notice:counts', $noticeCounts, null, time() + 4 * 60 * 60); } return $noticeCounts; diff --git a/plugins/Sitemap/Sitemap_user_count.php b/plugins/Sitemap/Sitemap_user_count.php index 64b4c3442..98dd05bfe 100644 --- a/plugins/Sitemap/Sitemap_user_count.php +++ b/plugins/Sitemap/Sitemap_user_count.php @@ -154,7 +154,9 @@ class Sitemap_user_count extends Memcached_DataObject $userCounts[$suc->registration_date] = $suc->user_count; } - self::cacheSet('sitemap:user:counts', $userCounts); + // Cache user counts for 4 hours. + + self::cacheSet('sitemap:user:counts', $userCounts, null, time() + 4 * 60 * 60); } return $userCounts; -- cgit v1.2.3-54-g00ecf