From f3a82e787c70e8cf749c79f22fe37ce6c9c9d4d3 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 12 Feb 2010 19:00:35 -0800 Subject: Add OStatus PuSH hub and Salmon links back into user and group feeds --- plugins/OStatus/OStatusPlugin.php | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index 8444c3d73..bf7dde296 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -63,9 +63,9 @@ class OStatusPlugin extends Plugin $m->connect('main/ostatus?nickname=:nickname', array('action' => 'ostatusinit'), array('nickname' => '[A-Za-z0-9_-]+')); $m->connect('main/ostatussub', - array('action' => 'ostatussub')); + array('action' => 'ostatussub')); $m->connect('main/ostatussub', - array('action' => 'ostatussub'), array('feed' => '[A-Za-z0-9\.\/\:]+')); + array('action' => 'ostatussub'), array('feed' => '[A-Za-z0-9\.\/\:]+')); // PuSH actions $m->connect('main/push/hub', array('action' => 'pushhub')); @@ -112,35 +112,34 @@ class OStatusPlugin extends Plugin * Set up a PuSH hub link to our internal link for canonical timeline * Atom feeds for users and groups. */ - function onStartApiAtom(Action $action) + function onStartApiAtom(AtomNoticeFeed $feed) { - if ($action instanceof ApiTimelineUserAction) { + $id = null; + + if ($feed instanceof AtomUserNoticeFeed) { $salmonAction = 'salmon'; - } else if ($action instanceof ApiTimelineGroupAction) { + $id = $feed->getUser()->id; + } else if ($feed instanceof AtomGroupNoticeFeed) { $salmonAction = 'salmongroup'; + $id = $feed->getGroup()->id; } else { return; } - $id = $action->arg('id'); - if (strval(intval($id)) === strval($id)) { - // Canonical form of id in URL? These are used for OStatus syndication. - + if (!empty($id)) { $hub = common_config('ostatus', 'hub'); if (empty($hub)) { // Updates will be handled through our internal PuSH hub. $hub = common_local_url('pushhub'); } - $action->element('link', array('rel' => 'hub', - 'href' => $hub)); + $feed->addLink($hub, array('rel' => 'hub')); // Also, we'll add in the salmon link $salmon = common_local_url($salmonAction, array('id' => $id)); - $action->element('link', array('rel' => 'salmon', - 'href' => $salmon)); + $feed->addLink($salmon, array('rel' => 'salmon')); } } - + /** * Add the feed settings page to the Connect Settings menu * @@ -201,7 +200,7 @@ class OStatusPlugin extends Plugin $output->element('a', array('href' => $url, 'class' => 'entity_remote_subscribe'), _m('OStatus')); - + $output->elementEnd('li'); } } @@ -221,25 +220,25 @@ class OStatusPlugin extends Plugin $w = new Webfinger; $endpoint_uri = ''; - + $result = $w->lookup($webfinger); if (empty($result)) { continue; } - + foreach ($result->links as $link) { if ($link['rel'] == 'salmon') { $endpoint_uri = $link['href']; } } - + if (empty($endpoint_uri)) { continue; } $xml = ''; $xml .= $notice->asAtomEntry(); - + $salmon = new Salmon(); $salmon->post($endpoint_uri, $xml); } -- cgit v1.2.3-54-g00ecf From f6e766d5acd4d9ca8e9bea30f1c00762ac193bd4 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Sat, 13 Feb 2010 18:42:00 +0100 Subject: Using the new remote subscription event and updated subscribe label --- plugins/OStatus/OStatusPlugin.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index bf7dde296..0e05a75cd 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -188,7 +188,7 @@ class OStatusPlugin extends Plugin /** * Add in an OStatus subscribe button */ - function onStartProfilePageActionsElements($output, $profile) + function onStartProfileRemoteSubscribe($output, $profile) { $cur = common_current_user(); @@ -199,10 +199,12 @@ class OStatusPlugin extends Plugin array('nickname' => $profile->nickname)); $output->element('a', array('href' => $url, 'class' => 'entity_remote_subscribe'), - _m('OStatus')); + _m('Subscribe')); $output->elementEnd('li'); } + + return false; } /** -- cgit v1.2.3-54-g00ecf From 24394269fa7d89c556835ebb55afd5b4c8400059 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Sat, 13 Feb 2010 18:44:41 +0100 Subject: Updated OStatus form markup --- plugins/OStatus/actions/ostatusinit.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/actions/ostatusinit.php b/plugins/OStatus/actions/ostatusinit.php index bac2c4d43..11f81ecb9 100644 --- a/plugins/OStatus/actions/ostatusinit.php +++ b/plugins/OStatus/actions/ostatusinit.php @@ -79,15 +79,15 @@ class OStatusInitAction extends Action 'class' => 'form_settings', 'action' => common_local_url('ostatusinit'))); $this->elementStart('fieldset'); - $this->element('legend', _('Subscribe to a remote user')); + $this->element('legend', null, sprintf(_('Subscribe to %s'), $this->nickname)); $this->hidden('token', common_session_token()); $this->elementStart('ul', 'form_data'); - $this->elementStart('li'); + $this->elementStart('li', array('id' => 'ostatus_nickname')); $this->input('nickname', _('User nickname'), $this->nickname, _('Nickname of the user you want to follow')); $this->elementEnd('li'); - $this->elementStart('li'); + $this->elementStart('li', array('id' => 'ostatus_profile')); $this->input('acct', _('Profile Account'), $this->acct, _('Your account id (i.e. user@identi.ca)')); $this->elementEnd('li'); @@ -95,7 +95,7 @@ class OStatusInitAction extends Action $this->submit('submit', _('Subscribe')); $this->elementEnd('fieldset'); $this->elementEnd('form'); - } + } function ostatusConnect() { @@ -125,4 +125,4 @@ class OStatusInitAction extends Action return _('OStatus Connect'); } -} \ No newline at end of file +} -- cgit v1.2.3-54-g00ecf From f30af7047b0131f4a7e6ed9c77f49fe5c113ea59 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Sat, 13 Feb 2010 18:46:10 +0100 Subject: Updated feed subscription form markup --- plugins/OStatus/actions/ostatussub.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/actions/ostatussub.php b/plugins/OStatus/actions/ostatussub.php index 9774286fd..239122501 100644 --- a/plugins/OStatus/actions/ostatussub.php +++ b/plugins/OStatus/actions/ostatussub.php @@ -76,7 +76,7 @@ class OStatusSubAction extends Action $this->elementStart('fieldset', array('id' => 'settings_feeds')); $this->elementStart('ul', 'form_data'); - $this->elementStart('li', array('id' => 'settings_twitter_login_button')); + $this->elementStart('li'); $this->input('feedurl', _('Feed URL'), $this->feedurl, _('Enter the URL of a PubSubHubbub-enabled feed')); $this->elementEnd('li'); $this->elementEnd('ul'); @@ -223,4 +223,4 @@ class OStatusSubAction extends Action } -} \ No newline at end of file +} -- cgit v1.2.3-54-g00ecf From 4d050f96f3b6c43ed74521d7a7215676cda6835e Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Sat, 13 Feb 2010 18:49:14 +0100 Subject: Added XHR channel for OStatus Subscribe button --- plugins/OStatus/actions/ostatusinit.php | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/actions/ostatusinit.php b/plugins/OStatus/actions/ostatusinit.php index 11f81ecb9..d21774420 100644 --- a/plugins/OStatus/actions/ostatusinit.php +++ b/plugins/OStatus/actions/ostatusinit.php @@ -67,9 +67,21 @@ class OStatusInitAction extends Action function showForm($err = null) { - $this->err = $err; - $this->showPage(); - + $this->err = $err; + if ($this->boolean('ajax')) { + header('Content-Type: text/xml;charset=utf-8'); + $this->xw->startDocument('1.0', 'UTF-8'); + $this->elementStart('html'); + $this->elementStart('head'); + $this->element('title', null, _('Subscribe to user')); + $this->elementEnd('head'); + $this->elementStart('body'); + $this->showContent(); + $this->elementEnd('body'); + $this->elementEnd('html'); + } else { + $this->showPage(); + } } function showContent() -- cgit v1.2.3-54-g00ecf From 1cb94e0be63f16d85bb5e9c4a8f6791caf91c8c5 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Sat, 13 Feb 2010 19:07:21 +0100 Subject: Added dialogbox styles for OStatus subscribe form --- plugins/OStatus/OStatusPlugin.php | 5 +++++ plugins/OStatus/theme/base/css/ostatus.css | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 plugins/OStatus/theme/base/css/ostatus.css (limited to 'plugins') diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index 0e05a75cd..e84d68504 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -274,4 +274,9 @@ class OStatusPlugin extends Plugin $schema->ensureTable('hubsub', HubSub::schemaDef()); return true; } + + function onEndShowStatusNetStyles($action) { + $action->cssLink(common_path('plugins/OStatus/theme/base/css/ostatus.css')); + return true; + } } diff --git a/plugins/OStatus/theme/base/css/ostatus.css b/plugins/OStatus/theme/base/css/ostatus.css new file mode 100644 index 000000000..9bc90a731 --- /dev/null +++ b/plugins/OStatus/theme/base/css/ostatus.css @@ -0,0 +1,30 @@ +/** theme: base for OStatus + * + * @package StatusNet + * @author Sarven Capadisli + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +#form_ostatus_connect.dialogbox { +width:70%; +background-image:none; +} +#form_ostatus_connect.dialogbox .form_data label { +width:34%; +} +#form_ostatus_connect.dialogbox .form_data input { +width:57%; +} +#form_ostatus_connect.dialogbox .form_data .form_guide { +margin-left:36%; +} + +#form_ostatus_connect.dialogbox #ostatus_nickname { +display:none; +} + +#form_ostatus_connect.dialogbox .submit_dialogbox { +min-width:96px; +} -- cgit v1.2.3-54-g00ecf From 171bf3093a4cf3f3ff0b4d255392ffd5375e4c36 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Sat, 13 Feb 2010 20:28:05 +0100 Subject: Dialogbox for OStatus remote subscription --- plugins/OStatus/OStatusPlugin.php | 5 ++++ plugins/OStatus/js/ostatus.js | 60 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 plugins/OStatus/js/ostatus.js (limited to 'plugins') diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index e84d68504..276ca1b3d 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -279,4 +279,9 @@ class OStatusPlugin extends Plugin $action->cssLink(common_path('plugins/OStatus/theme/base/css/ostatus.css')); return true; } + + function onEndShowStatusNetScripts($action) { + $action->script(common_path('plugins/OStatus/js/ostatus.js')); + return true; + } } diff --git a/plugins/OStatus/js/ostatus.js b/plugins/OStatus/js/ostatus.js new file mode 100644 index 000000000..671795558 --- /dev/null +++ b/plugins/OStatus/js/ostatus.js @@ -0,0 +1,60 @@ +SN.U.DialogBox = { + Subscribe: function(a) { + var f = a.parent().find('#form_ostatus_connect'); + if (f.length > 0) { + f.show(); + } + else { + $.ajax({ + type: 'GET', + dataType: 'xml', + url: a[0].href+'&ajax=1', + beforeSend: function(formData) { + a.addClass('processing'); + }, + error: function (xhr, textStatus, errorThrown) { + alert(errorThrown || textStatus); + }, + success: function(data, textStatus, xhr) { + if (typeof($('form', data)[0]) != 'undefined') { + a.after(document._importNode($('form', data)[0], true)); + + var form = a.parent().find('#form_ostatus_connect'); + + form + .addClass('dialogbox') + .append(''); + + form + .find('.submit') + .addClass('submit_dialogbox') + .removeClass('submit') + .bind('click', function() { + form.addClass('processing'); + }); + + form.find('button.close').click(function(){ + form.hide(); + + return false; + }); + + form.find('#acct').focus(); + } + + a.removeClass('processing'); + } + }); + } + } +}; + +SN.Init.Subscribe = function() { + $('.entity_subscribe a').live('click', function() { SN.U.DialogBox.Subscribe($(this)); return false; }); +}; + +$(document).ready(function() { + if ($('.entity_subscribe .entity_remote_subscribe').length > 0) { + SN.Init.Subscribe(); + } +}); -- cgit v1.2.3-54-g00ecf From 62f5c04ad234be6d1a26097300944eacee895738 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 14 Feb 2010 12:08:09 -0500 Subject: More complete activity parsing Began the process of actually digging up activity information from an Atom entry. Added a test script to make sure parsing is working right. --- plugins/OStatus/tests/ActivityParseTests.php | 80 ++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 plugins/OStatus/tests/ActivityParseTests.php (limited to 'plugins') diff --git a/plugins/OStatus/tests/ActivityParseTests.php b/plugins/OStatus/tests/ActivityParseTests.php new file mode 100644 index 000000000..889fa892f --- /dev/null +++ b/plugins/OStatus/tests/ActivityParseTests.php @@ -0,0 +1,80 @@ +documentElement); + + $this->assertFalse(empty($act)); + $this->assertEquals($act->time, 1243860840); + $this->assertEquals($act->verb, ActivityVerb::POST); + } +} + +$_example1 = << + + tag:versioncentral.example.org,2009:/commit/1643245 + 2009-06-01T12:54:00Z + Geraldine committed a change to yate + Geraldine just committed a change to yate on VersionCentral + + http://activitystrea.ms/schema/1.0/post + http://versioncentral.example.org/activity/commit + + http://versioncentral.example.org/activity/changeset + tag:versioncentral.example.org,2009:/change/1643245 + Punctuation Changeset + Fixing punctuation because it makes it more readable. + + + +EXAMPLE1; + +$_example2 = << + + tag:photopanic.example.com,2008:activity01 + Geraldine posted a Photo on PhotoPanic + 2008-11-02T15:29:00Z + + + http://activitystrea.ms/schema/1.0/post + + + tag:photopanic.example.com,2008:photo01 + My Cat + 2008-11-02T15:29:00Z + + + tag:atomactivity.example.com,2008:photo + + + Geraldine's Photos + + + + + + <p>Geraldine posted a Photo on PhotoPanic</p> + <img src="/geraldine/photo1.jpg"> + + +EXAMPLE2; -- cgit v1.2.3-54-g00ecf From f78cf31303f9973447b85ecfe7557d452e97a21c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 14 Feb 2010 12:12:47 -0500 Subject: update activity and salmon for previous commit --- plugins/OStatus/actions/salmon.php | 2 +- plugins/OStatus/lib/activity.php | 303 +++++++++++++++++++++++++++++++++++-- 2 files changed, 293 insertions(+), 12 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/actions/salmon.php b/plugins/OStatus/actions/salmon.php index b616027a9..c79d09c95 100644 --- a/plugins/OStatus/actions/salmon.php +++ b/plugins/OStatus/actions/salmon.php @@ -61,7 +61,7 @@ class SalmonAction extends Action // XXX: check that document element is Atom entry // XXX: check the signature - $this->act = Activity::fromAtomEntry($dom->documentElement); + $this->act = new Activity($dom->documentElement); } function handle($args) diff --git a/plugins/OStatus/lib/activity.php b/plugins/OStatus/lib/activity.php index 36e227913..11aab2848 100644 --- a/plugins/OStatus/lib/activity.php +++ b/plugins/OStatus/lib/activity.php @@ -31,7 +31,72 @@ if (!defined('STATUSNET')) { exit(1); } -class ActivityNoun +/** + * Utilities for turning DOMish things into Activityish things + * + * Some common functions that I didn't have the bandwidth to try to factor + * into some kind of reasonable superclass, so just dumped here. Might + * be useful to have an ActivityObject parent class or something. + * + * @category OStatus + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPLv3 + * @link http://status.net/ + */ + +class ActivityUtils +{ + const ATOM = 'http://www.w3.org/2005/Atom'; + + const LINK = 'link'; + const REL = 'rel'; + const TYPE = 'type'; + const HREF = 'href'; + + /** + * Get the permalink for an Activity object + * + * @param DOMElement $element A DOM element + * + * @return string related link, if any + */ + + static function getLink($element) + { + $links = $element->getElementsByTagnameNS(self::ATOM, self::LINK); + + foreach ($links as $link) { + if ($link->getAttributeNS(self::ATOM, self::REL) == 'alternate' && + $link->getAttributeNS(self::ATOM, self::TYPE) == 'text/html') { + return $link->getAttributeNS(self::ATOM, self::HREF); + } + } + + return null; + } +} + +/** + * A noun-ish thing in the activity universe + * + * The activity streams spec talks about activity objects, while also having + * a tag activity:object, which is in fact an activity object. Aaaaaah! + * + * This is just a thing in the activity universe. Can be the subject, object, + * or indirect object (target!) of an activity verb. Rotten name, and I'm + * propagating it. *sigh* + * + * @category OStatus + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPLv3 + * @link http://status.net/ + */ + +class ActivityObject { const ARTICLE = 'http://activitystrea.ms/schema/1.0/article'; const BLOGENTRY = 'http://activitystrea.ms/schema/1.0/blog-entry'; @@ -47,19 +112,101 @@ class ActivityNoun const PERSON = 'http://activitystrea.ms/schema/1.0/person'; const GROUP = 'http://activitystrea.ms/schema/1.0/group'; const PLACE = 'http://activitystrea.ms/schema/1.0/place'; - const COMMENT = 'http://activitystrea.ms/schema/1.0/comment'; // tea + const COMMENT = 'http://activitystrea.ms/schema/1.0/comment'; + // ^^^^^^^^^^ tea! + + // Atom elements we snarf + + const TITLE = 'title'; + const SUMMARY = 'summary'; + const CONTENT = 'content'; + const ID = 'id'; + const SOURCE = 'source'; + + const NAME = 'name'; + const URI = 'uri'; public $type; public $id; public $title; public $summary; public $content; + public $link; + public $source; + + /** + * Constructor + * + * This probably needs to be refactored + * to generate a local class (ActivityPerson, ActivityFile, ...) + * based on the object type. + * + * @param DOMElement $element DOM thing to turn into an Activity thing + */ + + function __construct($element) + { + $this->source = $element; + + if ($element->tagName == 'author') { + + $this->type = self::PERSON; // XXX: is this fair? + $this->title = $this->_childContent($element, self::NAME); + $this->id = $this->_childContent($element, self::URI); + + } else { + + $this->type = $this->_childContent($element, Activity::OBJECTTYPE, + Activity::SPEC); + + $this->id = $this->_childContent($element, self::ID); + $this->title = $this->_childContent($element, self::TITLE); + $this->summary = $this->_childContent($element, self::SUMMARY); + $this->content = $this->_childContent($element, self::CONTENT); + $this->source = $this->_childContent($element, self::SOURCE); + + $this->link = ActivityUtils::getLink($element); + + // XXX: grab PoCo stuff + } + } + + /** + * Grab the text content of a DOM element child of the current element + * + * @param DOMElement $element Element whose children we examine + * @param string $tag Tag to look up + * @param string $namespace Namespace to use, defaults to Atom + * + * @return string content of the child + */ + + private function _childContent($element, $tag, $namespace=Activity::ATOM) + { + $els = $element->getElementsByTagnameNS($namespace, $tag); + + if (empty($els) || $els->length == 0) { + return null; + } else { + $el = $els->item(0); + return $el->textContent; + } + } } -class Activity -{ - const NAMESPACE = 'http://activitystrea.ms/schema/1.0/'; +/** + * Utility class to hold a bunch of constant defining default verb types + * + * @category OStatus + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPLv3 + * @link http://status.net/ + */ +class ActivityVerb +{ const POST = 'http://activitystrea.ms/schema/1.0/post'; const SHARE = 'http://activitystrea.ms/schema/1.0/share'; const SAVE = 'http://activitystrea.ms/schema/1.0/save'; @@ -69,17 +216,151 @@ class Activity const FRIEND = 'http://activitystrea.ms/schema/1.0/make-friend'; const JOIN = 'http://activitystrea.ms/schema/1.0/join'; const TAG = 'http://activitystrea.ms/schema/1.0/tag'; +} + +/** + * An activity in the ActivityStrea.ms world + * + * An activity is kind of like a sentence: someone did something + * to something else. + * + * 'someone' is the 'actor'; 'did something' is the verb; + * 'something else' is the object. + * + * @category OStatus + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPLv3 + * @link http://status.net/ + */ + +class Activity +{ + const SPEC = 'http://activitystrea.ms/spec/1.0/'; + const SCHEMA = 'http://activitystrea.ms/schema/1.0/'; + + const VERB = 'verb'; + const OBJECT = 'object'; + const ACTOR = 'actor'; + const SUBJECT = 'subject'; + const OBJECTTYPE = 'object-type'; + const CONTEXT = 'context'; + const TARGET = 'target'; + + const ATOM = 'http://www.w3.org/2005/Atom'; - public $actor; // an ActivityNoun - public $verb; // a string (the URL) - public $object; // an ActivityNoun - public $target; // an ActivityNoun + const AUTHOR = 'author'; + const PUBLISHED = 'published'; - static function fromAtomEntry($domEntry) + public $actor; // an ActivityObject + public $verb; // a string (the URL) + public $object; // an ActivityObject + public $target; // an ActivityObject + public $context; // an ActivityObject + public $time; // Time of the activity + public $link; // an ActivityObject + public $entry; // the source entry + public $feed; // the source feed + + /** + * Turns a regular old Atom into a magical activity + * + * @param DOMElement $entry Atom entry to poke at + * @param DOMElement $feed Atom feed, for context + */ + + function __construct($entry, $feed = null) { + $this->entry = $entry; + $this->feed = $feed; + + $pubEl = $this->_child($entry, self::PUBLISHED, self::ATOM); + + if (!empty($pubEl)) { + $this->time = strtotime($pubEl->textContent); + } else { + // XXX technically an error; being liberal. Good idea...? + $this->time = null; + } + + $this->link = ActivityUtils::getLink($entry); + + $verbEl = $this->_child($entry, self::VERB); + + if (!empty($verbEl)) { + $this->verb = trim($verbEl->textContent); + } else { + $this->verb = ActivityVerb::POST; + // XXX: do other implied stuff here + } + + $objectEl = $this->_child($entry, self::OBJECT); + + if (!empty($objectEl)) { + $this->object = new ActivityObject($objectEl); + } else { + $this->object = new ActivityObject($entry); + } + + $actorEl = $this->_child($entry, self::ACTOR); + + if (!empty($actorEl)) { + + $this->actor = new ActivityObject($actorEl); + + } else if (!empty($feed) && + $subjectEl = $this->_child($feed, self::SUBJECT)) { + + $this->actor = new ActivityObject($subjectEl); + + } else if ($authorEl = $this->_child($entry, self::AUTHOR, self::ATOM)) { + + $this->actor = new ActivityObject($authorEl); + } + + $contextEl = $this->_child($entry, self::CONTEXT); + + if (!empty($contextEl)) { + $this->context = new ActivityObject($contextEl); + } + + $targetEl = $this->_child($entry, self::TARGET); + + if (!empty($targetEl)) { + $this->target = new ActivityObject($targetEl); + } } + /** + * Returns an Atom based on this activity + * + * @return DOMElement Atom entry + */ + function toAtomEntry() { + return null; } -} + + /** + * Gets the first child element with the given tag + * + * @param DOMElement $element element to pick at + * @param string $tag tag to look for + * @param string $namespace Namespace to look under + * + * @return DOMElement found element or null + */ + + private function _child($element, $tag, $namespace=self::SPEC) + { + $els = $element->getElementsByTagnameNS($namespace, $tag); + + if (empty($els) || $els->length == 0) { + return null; + } else { + return $els->item(0); + } + } +} \ No newline at end of file -- cgit v1.2.3-54-g00ecf From 73e2264c6aa7c1fa3a6e4b63f41b210af3c50597 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 14 Feb 2010 13:19:32 -0500 Subject: test parsing a default atom feed for activities --- plugins/OStatus/lib/activity.php | 39 +++++++++++++--- plugins/OStatus/tests/ActivityParseTests.php | 67 ++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/lib/activity.php b/plugins/OStatus/lib/activity.php index 11aab2848..048efda2c 100644 --- a/plugins/OStatus/lib/activity.php +++ b/plugins/OStatus/lib/activity.php @@ -68,9 +68,12 @@ class ActivityUtils $links = $element->getElementsByTagnameNS(self::ATOM, self::LINK); foreach ($links as $link) { - if ($link->getAttributeNS(self::ATOM, self::REL) == 'alternate' && - $link->getAttributeNS(self::ATOM, self::TYPE) == 'text/html') { - return $link->getAttributeNS(self::ATOM, self::HREF); + + $rel = $link->getAttribute(self::REL); + $type = $link->getAttribute(self::TYPE); + + if ($rel == 'alternate' && $type == 'text/html') { + return $link->getAttribute(self::HREF); } } @@ -123,8 +126,9 @@ class ActivityObject const ID = 'id'; const SOURCE = 'source'; - const NAME = 'name'; - const URI = 'uri'; + const NAME = 'name'; + const URI = 'uri'; + const EMAIL = 'email'; public $type; public $id; @@ -154,11 +158,23 @@ class ActivityObject $this->title = $this->_childContent($element, self::NAME); $this->id = $this->_childContent($element, self::URI); + if (empty($this->id)) { + $email = $this->_childContent($element, self::EMAIL); + if (!empty($email)) { + // XXX: acct: ? + $this->id = 'mailto:'.$email; + } + } + } else { $this->type = $this->_childContent($element, Activity::OBJECTTYPE, Activity::SPEC); + if (empty($this->type)) { + $this->type = ActivityObject::NOTE; + } + $this->id = $this->_childContent($element, self::ID); $this->title = $this->_childContent($element, self::TITLE); $this->summary = $this->_childContent($element, self::SUMMARY); @@ -252,6 +268,7 @@ class Activity const AUTHOR = 'author'; const PUBLISHED = 'published'; + const UPDATED = 'updated'; public $actor; // an ActivityObject public $verb; // a string (the URL) @@ -281,7 +298,12 @@ class Activity $this->time = strtotime($pubEl->textContent); } else { // XXX technically an error; being liberal. Good idea...? - $this->time = null; + $updateEl = $this->_child($entry, self::UPDATED, self::ATOM); + if (!empty($updateEl)) { + $this->time = strtotime($updateEl->textContent); + } else { + $this->time = null; + } } $this->link = ActivityUtils::getLink($entry); @@ -317,6 +339,11 @@ class Activity } else if ($authorEl = $this->_child($entry, self::AUTHOR, self::ATOM)) { $this->actor = new ActivityObject($authorEl); + + } else if (!empty($feed) && $authorEl = $this->_child($feed, self::AUTHOR, + self::ATOM)) { + + $this->actor = new ActivityObject($authorEl); } $contextEl = $this->_child($entry, self::CONTEXT); diff --git a/plugins/OStatus/tests/ActivityParseTests.php b/plugins/OStatus/tests/ActivityParseTests.php index 889fa892f..fa8bcdda2 100644 --- a/plugins/OStatus/tests/ActivityParseTests.php +++ b/plugins/OStatus/tests/ActivityParseTests.php @@ -25,6 +25,44 @@ class ActivityParseTests extends PHPUnit_Framework_TestCase $this->assertEquals($act->time, 1243860840); $this->assertEquals($act->verb, ActivityVerb::POST); } + + public function testExample3() + { + global $_example3; + $dom = DOMDocument::loadXML($_example3); + + $feed = $dom->documentElement; + + $entries = $feed->getElementsByTagName('entry'); + + $entry = $entries->item(0); + + $act = new Activity($entry, $feed); + + $this->assertFalse(empty($act)); + $this->assertEquals($act->time, 1071340202); + $this->assertEquals($act->link, 'http://example.org/2003/12/13/atom03.html'); + + $this->assertEquals($act->verb, ActivityVerb::POST); + + $this->assertFalse(empty($act->actor)); + $this->assertEquals($act->actor->type, ActivityObject::PERSON); + $this->assertEquals($act->actor->title, 'John Doe'); + $this->assertEquals($act->actor->id, 'mailto:johndoe@example.com'); + + $this->assertFalse(empty($act->object)); + $this->assertEquals($act->object->type, ActivityObject::NOTE); + $this->assertEquals($act->object->id, 'urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a'); + $this->assertEquals($act->object->title, 'Atom-Powered Robots Run Amok'); + $this->assertEquals($act->object->summary, 'Some text.'); + $this->assertEquals($act->object->link, 'http://example.org/2003/12/13/atom03.html'); + + $this->assertTrue(empty($act->context)); + $this->assertTrue(empty($act->target)); + + $this->assertEquals($act->entry, $entry); + $this->assertEquals($act->feed, $feed); + } } $_example1 = << EXAMPLE2; + +$_example3 = << + + + + Example Feed + A subtitle. + + + urn:uuid:60a76c80-d399-11d9-b91C-0003939e0af6 + 2003-12-13T18:30:02Z + + John Doe + johndoe@example.com + + + + Atom-Powered Robots Run Amok + + + + urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a + 2003-12-13T18:30:02Z + Some text. + + + +EXAMPLE3; -- cgit v1.2.3-54-g00ecf From 03edbfe24e99308b8c27363a0b81b96f9fda18a0 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Mon, 15 Feb 2010 20:41:46 +0100 Subject: Added single whitespace to separate inline text words. --- lib/action.php | 3 +++ lib/grouplist.php | 4 ++++ lib/groupsection.php | 3 ++- lib/noticelist.php | 11 ++++++++++- lib/profilelist.php | 4 ++++ lib/profilesection.php | 2 ++ plugins/PoweredByStatusNet/PoweredByStatusNetPlugin.php | 1 + 7 files changed, 26 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/lib/action.php b/lib/action.php index cc4f4aad0..b85f353a3 100644 --- a/lib/action.php +++ b/lib/action.php @@ -405,6 +405,7 @@ class Action extends HTMLOutputter // lawsuit 'src' => (common_config('site', 'logo')) ? common_config('site', 'logo') : Theme::path('logo.png'), 'alt' => common_config('site', 'name'))); } + $this->text(' '); $this->element('span', array('class' => 'fn org'), common_config('site', 'name')); $this->elementEnd('a'); Event::handle('EndAddressData', array($this)); @@ -822,12 +823,14 @@ class Action extends HTMLOutputter // lawsuit 'alt' => common_config('license', 'title'), 'width' => '80', 'height' => '15')); + $this->text(' '); //TODO: This is dirty: i18n $this->text(_('All '.common_config('site', 'name').' content and data are available under the ')); $this->element('a', array('class' => 'license', 'rel' => 'external license', 'href' => common_config('license', 'url')), common_config('license', 'title')); + $this->text(' '); $this->text(_('license.')); $this->elementEnd('p'); break; diff --git a/lib/grouplist.php b/lib/grouplist.php index 99bff9cdc..854bc34e2 100644 --- a/lib/grouplist.php +++ b/lib/grouplist.php @@ -105,6 +105,7 @@ class GroupList extends Widget 'alt' => ($this->group->fullname) ? $this->group->fullname : $this->group->nickname)); + $this->out->text(' '); $hasFN = ($this->group->fullname) ? 'nickname' : 'fn org nickname'; $this->out->elementStart('span', $hasFN); $this->out->raw($this->highlight($this->group->nickname)); @@ -112,16 +113,19 @@ class GroupList extends Widget $this->out->elementEnd('a'); if ($this->group->fullname) { + $this->out->text(' '); $this->out->elementStart('span', 'fn org'); $this->out->raw($this->highlight($this->group->fullname)); $this->out->elementEnd('span'); } if ($this->group->location) { + $this->out->text(' '); $this->out->elementStart('span', 'label'); $this->out->raw($this->highlight($this->group->location)); $this->out->elementEnd('span'); } if ($this->group->homepage) { + $this->out->text(' '); $this->out->elementStart('a', array('href' => $this->group->homepage, 'class' => 'url')); $this->out->raw($this->highlight($this->group->homepage)); diff --git a/lib/groupsection.php b/lib/groupsection.php index 7327f9e1a..3b0b3029d 100644 --- a/lib/groupsection.php +++ b/lib/groupsection.php @@ -85,9 +85,9 @@ class GroupSection extends Section 'href' => $group->homeUrl(), 'rel' => 'contact group', 'class' => 'url')); + $this->out->text(' '); $logo = ($group->stream_logo) ? $group->stream_logo : User_group::defaultLogo(AVATAR_STREAM_SIZE); - $this->out->element('img', array('src' => $logo, 'width' => AVATAR_MINI_SIZE, 'height' => AVATAR_MINI_SIZE, @@ -95,6 +95,7 @@ class GroupSection extends Section 'alt' => ($group->fullname) ? $group->fullname : $group->nickname)); + $this->out->text(' '); $this->out->element('span', 'fn org nickname', $group->nickname); $this->out->elementEnd('a'); $this->out->elementEnd('span'); diff --git a/lib/noticelist.php b/lib/noticelist.php index a4a0f2651..c05b99024 100644 --- a/lib/noticelist.php +++ b/lib/noticelist.php @@ -294,6 +294,7 @@ class NoticeListItem extends Widget } $this->out->elementStart('a', $attrs); $this->showAvatar(); + $this->out->text(' '); $this->showNickname(); $this->out->elementEnd('a'); $this->out->elementEnd('span'); @@ -432,8 +433,10 @@ class NoticeListItem extends Widget $url = $location->getUrl(); + $this->out->text(' '); $this->out->elementStart('span', array('class' => 'location')); $this->out->text(_('at')); + $this->out->text(' '); if (empty($url)) { $this->out->element('span', array('class' => 'geo', 'title' => $latlon), @@ -473,9 +476,11 @@ class NoticeListItem extends Widget function showNoticeSource() { if ($this->notice->source) { + $this->out->text(' '); $this->out->elementStart('span', 'source'); $this->out->text(_('from')); $source_name = _($this->notice->source); + $this->out->text(' '); switch ($this->notice->source) { case 'web': case 'xmpp': @@ -540,6 +545,7 @@ class NoticeListItem extends Widget } } if ($hasConversation){ + $this->out->text(' '); $convurl = common_local_url('conversation', array('id' => $this->notice->conversation)); $this->out->element('a', array('href' => $convurl.'#notice-'.$this->notice->id, @@ -591,12 +597,14 @@ class NoticeListItem extends Widget function showReplyLink() { if (common_logged_in()) { + $this->out->text(' '); $reply_url = common_local_url('newnotice', array('replyto' => $this->profile->nickname, 'inreplyto' => $this->notice->id)); $this->out->elementStart('a', array('href' => $reply_url, 'class' => 'notice_reply', 'title' => _('Reply to this notice'))); $this->out->text(_('Reply')); + $this->out->text(' '); $this->out->element('span', 'notice_id', $this->notice->id); $this->out->elementEnd('a'); } @@ -616,7 +624,7 @@ class NoticeListItem extends Widget if (!empty($user) && ($todel->profile_id == $user->id || $user->hasRight(Right::DELETEOTHERSNOTICE))) { - + $this->out->text(' '); $deleteurl = common_local_url('deletenotice', array('notice' => $todel->id)); $this->out->element('a', array('href' => $deleteurl, @@ -635,6 +643,7 @@ class NoticeListItem extends Widget { $user = common_current_user(); if ($user && $user->id != $this->notice->profile_id) { + $this->out->text(' '); $profile = $user->getProfile(); if ($profile->hasRepeated($this->notice->id)) { $this->out->element('span', array('class' => 'repeated', diff --git a/lib/profilelist.php b/lib/profilelist.php index 3412d41d1..693cd6449 100644 --- a/lib/profilelist.php +++ b/lib/profilelist.php @@ -191,6 +191,7 @@ class ProfileListItem extends Widget 'alt' => ($this->profile->fullname) ? $this->profile->fullname : $this->profile->nickname)); + $this->out->text(' '); $hasFN = (!empty($this->profile->fullname)) ? 'nickname' : 'fn nickname'; $this->out->elementStart('span', $hasFN); $this->out->raw($this->highlight($this->profile->nickname)); @@ -201,6 +202,7 @@ class ProfileListItem extends Widget function showFullName() { if (!empty($this->profile->fullname)) { + $this->out->text(' '); $this->out->elementStart('span', 'fn'); $this->out->raw($this->highlight($this->profile->fullname)); $this->out->elementEnd('span'); @@ -210,6 +212,7 @@ class ProfileListItem extends Widget function showLocation() { if (!empty($this->profile->location)) { + $this->out->text(' '); $this->out->elementStart('span', 'location'); $this->out->raw($this->highlight($this->profile->location)); $this->out->elementEnd('span'); @@ -219,6 +222,7 @@ class ProfileListItem extends Widget function showHomepage() { if (!empty($this->profile->homepage)) { + $this->out->text(' '); $this->out->elementStart('a', array('href' => $this->profile->homepage, 'class' => 'url')); $this->out->raw($this->highlight($this->profile->homepage)); diff --git a/lib/profilesection.php b/lib/profilesection.php index 504b1b7f7..a9482cd63 100644 --- a/lib/profilesection.php +++ b/lib/profilesection.php @@ -85,6 +85,7 @@ class ProfileSection extends Section 'href' => $profile->profileurl, 'rel' => 'contact member', 'class' => 'url')); + $this->out->text(' '); $avatar = $profile->getAvatar(AVATAR_MINI_SIZE); $this->out->element('img', array('src' => (($avatar) ? $avatar->displayUrl() : Avatar::defaultImage(AVATAR_MINI_SIZE)), 'width' => AVATAR_MINI_SIZE, @@ -93,6 +94,7 @@ class ProfileSection extends Section 'alt' => ($profile->fullname) ? $profile->fullname : $profile->nickname)); + $this->out->text(' '); $this->out->element('span', 'fn nickname', $profile->nickname); $this->out->elementEnd('a'); $this->out->elementEnd('span'); diff --git a/plugins/PoweredByStatusNet/PoweredByStatusNetPlugin.php b/plugins/PoweredByStatusNet/PoweredByStatusNetPlugin.php index 14d1608d3..fb4eff738 100644 --- a/plugins/PoweredByStatusNet/PoweredByStatusNetPlugin.php +++ b/plugins/PoweredByStatusNet/PoweredByStatusNetPlugin.php @@ -45,6 +45,7 @@ class PoweredByStatusNetPlugin extends Plugin { function onEndAddressData($action) { + $action->text(' '); $action->elementStart('span', 'poweredby'); $action->raw(sprintf(_m('powered by %s'), sprintf('%s', -- cgit v1.2.3-54-g00ecf From 2b6a39f70f1c1917ff53b7908ac2a1819c876166 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 21:10:45 +0000 Subject: Fix for regression introduced with my last update to the TwitterStatusFetcher: the Twitter bridge was not saving a foreign user record when making a foreign link. --- plugins/TwitterBridge/twitter.php | 7 ++++--- plugins/TwitterBridge/twitterauthorization.php | 14 +++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index de30d9ebf..bd7f19e71 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -28,6 +28,7 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; function add_twitter_user($twitter_id, $screen_name) { + common_debug("Add Twitter user"); $new_uri = 'http://twitter.com/' . $screen_name; @@ -40,7 +41,7 @@ function add_twitter_user($twitter_id, $screen_name) $luser->service = TWITTER_SERVICE; $result = $luser->delete(); - if (empty($result)) { + if ($result != false) { common_log(LOG_WARNING, "Twitter bridge - removed invalid Twitter user squatting on uri: $new_uri"); } @@ -93,9 +94,9 @@ function save_twitter_user($twitter_id, $screen_name) $screen_name, $oldname)); } - - return add_twitter_user($twitter_id, $screen_name); } + + return add_twitter_user($twitter_id, $screen_name); } function is_twitter_bound($notice, $flink) { diff --git a/plugins/TwitterBridge/twitterauthorization.php b/plugins/TwitterBridge/twitterauthorization.php index dbef438a4..6822d33dd 100644 --- a/plugins/TwitterBridge/twitterauthorization.php +++ b/plugins/TwitterBridge/twitterauthorization.php @@ -89,11 +89,15 @@ class TwitterauthorizationAction extends Action $user = common_current_user(); $flink = Foreign_link::getByUserID($user->id, TWITTER_SERVICE); - // If there's already a foreign link record, it means we already - // have an access token, and this is unecessary. So go back. + // If there's already a foreign link record and a foreign user + // it means the accounts are already linked, and this is unecessary. + // So go back. if (isset($flink)) { - common_redirect(common_local_url('twittersettings')); + $fuser = $flink->getForeignUser(); + if (!empty($fuser)) { + common_redirect(common_local_url('twittersettings')); + } } } @@ -254,6 +258,10 @@ class TwitterauthorizationAction extends Action { $flink = new Foreign_link(); + $flink->user_id = $user_id; + $flink->service = TWITTER_SERVICE; + $flink->delete(); // delete stale flink, if any + $flink->user_id = $user_id; $flink->foreign_id = $twuid; $flink->service = TWITTER_SERVICE; -- cgit v1.2.3-54-g00ecf From 5cc1f8b001057e9c4301b173391a7f0a5415f153 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 21:10:45 +0000 Subject: Fix for regression introduced with my last update to the TwitterStatusFetcher: the Twitter bridge was not saving a foreign user record when making a foreign link. --- plugins/TwitterBridge/twitter.php | 7 ++++--- plugins/TwitterBridge/twitterauthorization.php | 14 +++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index de30d9ebf..bd7f19e71 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -28,6 +28,7 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; function add_twitter_user($twitter_id, $screen_name) { + common_debug("Add Twitter user"); $new_uri = 'http://twitter.com/' . $screen_name; @@ -40,7 +41,7 @@ function add_twitter_user($twitter_id, $screen_name) $luser->service = TWITTER_SERVICE; $result = $luser->delete(); - if (empty($result)) { + if ($result != false) { common_log(LOG_WARNING, "Twitter bridge - removed invalid Twitter user squatting on uri: $new_uri"); } @@ -93,9 +94,9 @@ function save_twitter_user($twitter_id, $screen_name) $screen_name, $oldname)); } - - return add_twitter_user($twitter_id, $screen_name); } + + return add_twitter_user($twitter_id, $screen_name); } function is_twitter_bound($notice, $flink) { diff --git a/plugins/TwitterBridge/twitterauthorization.php b/plugins/TwitterBridge/twitterauthorization.php index dbef438a4..6822d33dd 100644 --- a/plugins/TwitterBridge/twitterauthorization.php +++ b/plugins/TwitterBridge/twitterauthorization.php @@ -89,11 +89,15 @@ class TwitterauthorizationAction extends Action $user = common_current_user(); $flink = Foreign_link::getByUserID($user->id, TWITTER_SERVICE); - // If there's already a foreign link record, it means we already - // have an access token, and this is unecessary. So go back. + // If there's already a foreign link record and a foreign user + // it means the accounts are already linked, and this is unecessary. + // So go back. if (isset($flink)) { - common_redirect(common_local_url('twittersettings')); + $fuser = $flink->getForeignUser(); + if (!empty($fuser)) { + common_redirect(common_local_url('twittersettings')); + } } } @@ -254,6 +258,10 @@ class TwitterauthorizationAction extends Action { $flink = new Foreign_link(); + $flink->user_id = $user_id; + $flink->service = TWITTER_SERVICE; + $flink->delete(); // delete stale flink, if any + $flink->user_id = $user_id; $flink->foreign_id = $twuid; $flink->service = TWITTER_SERVICE; -- cgit v1.2.3-54-g00ecf From fdf6ed7b1ade7dc4cf2f1f71063025719f3e722a Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 21:23:26 +0000 Subject: Better log msgs. Removed debugging statement. --- plugins/TwitterBridge/twitter.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index bd7f19e71..2a075300f 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -28,8 +28,6 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; function add_twitter_user($twitter_id, $screen_name) { - common_debug("Add Twitter user"); - $new_uri = 'http://twitter.com/' . $screen_name; // Clear out any bad old foreign_users with the new user's legit URL @@ -42,8 +40,8 @@ function add_twitter_user($twitter_id, $screen_name) $result = $luser->delete(); if ($result != false) { - common_log(LOG_WARNING, - "Twitter bridge - removed invalid Twitter user squatting on uri: $new_uri"); + common_log(LOG_INFO, + "Twitter bridge - removed old Twitter user: $screen_name ($twitter_id)."); } $luser->free(); @@ -65,7 +63,8 @@ function add_twitter_user($twitter_id, $screen_name) "Twitter bridge - failed to add new Twitter user: $twitter_id - $screen_name."); common_log_db_error($fuser, 'INSERT', __FILE__); } else { - common_debug("Twitter bridge - Added new Twitter user: $screen_name ($twitter_id)."); + common_log(LOG_INFO, + "Twitter bridge - Added new Twitter user: $screen_name ($twitter_id)."); } return $result; -- cgit v1.2.3-54-g00ecf From a69863eae6f82440b50c3fb5e500e6feec052ae4 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 21:23:26 +0000 Subject: Better log msgs. Removed debugging statement. --- plugins/TwitterBridge/twitter.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index bd7f19e71..2a075300f 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -28,8 +28,6 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; function add_twitter_user($twitter_id, $screen_name) { - common_debug("Add Twitter user"); - $new_uri = 'http://twitter.com/' . $screen_name; // Clear out any bad old foreign_users with the new user's legit URL @@ -42,8 +40,8 @@ function add_twitter_user($twitter_id, $screen_name) $result = $luser->delete(); if ($result != false) { - common_log(LOG_WARNING, - "Twitter bridge - removed invalid Twitter user squatting on uri: $new_uri"); + common_log(LOG_INFO, + "Twitter bridge - removed old Twitter user: $screen_name ($twitter_id)."); } $luser->free(); @@ -65,7 +63,8 @@ function add_twitter_user($twitter_id, $screen_name) "Twitter bridge - failed to add new Twitter user: $twitter_id - $screen_name."); common_log_db_error($fuser, 'INSERT', __FILE__); } else { - common_debug("Twitter bridge - Added new Twitter user: $screen_name ($twitter_id)."); + common_log(LOG_INFO, + "Twitter bridge - Added new Twitter user: $screen_name ($twitter_id)."); } return $result; -- cgit v1.2.3-54-g00ecf From 66f427c373b6559aa81f4b3b18b0aa58ec689c0d Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 21:53:49 +0000 Subject: Twitter-bridge: lookup old foreign_user by primary key not url --- plugins/TwitterBridge/twitter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index 2a075300f..ce989768e 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -35,7 +35,7 @@ function add_twitter_user($twitter_id, $screen_name) // repoed, and things like that. $luser = new Foreign_user(); - $luser->uri = $new_uri; + $luser->id = $twitter_id; $luser->service = TWITTER_SERVICE; $result = $luser->delete(); -- cgit v1.2.3-54-g00ecf From 01c428796f7986fddc0e602656a57ea63055bffd Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 21:53:49 +0000 Subject: Twitter-bridge: lookup old foreign_user by primary key not url --- plugins/TwitterBridge/twitter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index 2a075300f..ce989768e 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -35,7 +35,7 @@ function add_twitter_user($twitter_id, $screen_name) // repoed, and things like that. $luser = new Foreign_user(); - $luser->uri = $new_uri; + $luser->id = $twitter_id; $luser->service = TWITTER_SERVICE; $result = $luser->delete(); -- cgit v1.2.3-54-g00ecf From 9f8e25bfe70134b7e19bb0067062e55bbb5b6a23 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 22:13:10 +0000 Subject: Use static class method for looking up Twitter user --- plugins/TwitterBridge/twitter.php | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index ce989768e..e5afde62c 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -28,15 +28,11 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; function add_twitter_user($twitter_id, $screen_name) { - $new_uri = 'http://twitter.com/' . $screen_name; - // Clear out any bad old foreign_users with the new user's legit URL // This can happen when users move around or fakester accounts get // repoed, and things like that. - $luser = new Foreign_user(); - $luser->id = $twitter_id; - $luser->service = TWITTER_SERVICE; + $luser = Foreign_user::getForeignUser($twitter_id, TWITTER_SERVICE); $result = $luser->delete(); if ($result != false) { @@ -44,11 +40,6 @@ function add_twitter_user($twitter_id, $screen_name) "Twitter bridge - removed old Twitter user: $screen_name ($twitter_id)."); } - $luser->free(); - unset($luser); - - // Otherwise, create a new Twitter user - $fuser = new Foreign_user(); $fuser->nickname = $screen_name; -- cgit v1.2.3-54-g00ecf From 0ba375917129eaee2608203ed532efb3b9db879c Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 15 Feb 2010 22:13:10 +0000 Subject: Use static class method for looking up Twitter user --- plugins/TwitterBridge/twitter.php | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'plugins') diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index ce989768e..e5afde62c 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -28,15 +28,11 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; function add_twitter_user($twitter_id, $screen_name) { - $new_uri = 'http://twitter.com/' . $screen_name; - // Clear out any bad old foreign_users with the new user's legit URL // This can happen when users move around or fakester accounts get // repoed, and things like that. - $luser = new Foreign_user(); - $luser->id = $twitter_id; - $luser->service = TWITTER_SERVICE; + $luser = Foreign_user::getForeignUser($twitter_id, TWITTER_SERVICE); $result = $luser->delete(); if ($result != false) { @@ -44,11 +40,6 @@ function add_twitter_user($twitter_id, $screen_name) "Twitter bridge - removed old Twitter user: $screen_name ($twitter_id)."); } - $luser->free(); - unset($luser); - - // Otherwise, create a new Twitter user - $fuser = new Foreign_user(); $fuser->nickname = $screen_name; -- cgit v1.2.3-54-g00ecf From f414544d0d289df2c103d9b16400e1ed91d35e91 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 16 Feb 2010 06:12:08 +0000 Subject: Upgrade Twitter bridge to use OAuth 1.0a. It's more secure, and allows us to automatically send in a callback url instead of having to manually configure one for each StatusNet instance. --- lib/oauthclient.php | 88 +++++++++++++++++++------- plugins/TwitterBridge/twitterauthorization.php | 10 +-- plugins/TwitterBridge/twitteroauthclient.php | 28 ++++++++ 3 files changed, 98 insertions(+), 28 deletions(-) (limited to 'plugins') diff --git a/lib/oauthclient.php b/lib/oauthclient.php index b22fd7897..bc7587183 100644 --- a/lib/oauthclient.php +++ b/lib/oauthclient.php @@ -90,20 +90,47 @@ class OAuthClient /** * Gets a request token from the given url * - * @param string $url OAuth endpoint for grabbing request tokens + * @param string $url OAuth endpoint for grabbing request tokens + * @param string $callback authorized request token callback * * @return OAuthToken $token the request token */ - function getRequestToken($url) + function getRequestToken($url, $callback = null) { - $response = $this->oAuthGet($url); + $params = null; + + if (!is_null($callback)) { + $params['oauth_callback'] = $callback; + } + + $response = $this->oAuthGet($url, $params); + $arr = array(); parse_str($response, $arr); - if (isset($arr['oauth_token']) && isset($arr['oauth_token_secret'])) { - $token = new OAuthToken($arr['oauth_token'], @$arr['oauth_token_secret']); + + $token = $arr['oauth_token']; + $secret = $arr['oauth_token_secret']; + $confirm = $arr['oauth_callback_confirmed']; + + if (isset($token) && isset($secret)) { + + $token = new OAuthToken($token, $secret); + + if (isset($confirm)) { + if ($confirm == 'true') { + common_debug('Twitter bridge - callback confirmed.'); + return $token; + } else { + throw new OAuthClientException( + 'Callback was not confirmed by Twitter.' + ); + } + } return $token; } else { - throw new OAuthClientException(); + throw new OAuthClientException( + 'Could not get a request token from Twitter.' + ); } } @@ -113,49 +140,64 @@ class OAuthClient * * @param string $url endpoint for authorizing request tokens * @param OAuthToken $request_token the request token to be authorized - * @param string $oauth_callback optional callback url * * @return string $authorize_url the url to redirect to */ - function getAuthorizeLink($url, $request_token, $oauth_callback = null) + function getAuthorizeLink($url, $request_token) { $authorize_url = $url . '?oauth_token=' . $request_token->key; - if (isset($oauth_callback)) { - $authorize_url .= '&oauth_callback=' . urlencode($oauth_callback); - } - return $authorize_url; } /** * Fetches an access token * - * @param string $url OAuth endpoint for exchanging authorized request tokens - * for access tokens + * @param string $url OAuth endpoint for exchanging authorized request tokens + * for access tokens + * @param string $verifier 1.0a verifier * * @return OAuthToken $token the access token */ - function getAccessToken($url) + function getAccessToken($url, $verifier = null) { - $response = $this->oAuthPost($url); - parse_str($response); - $token = new OAuthToken($oauth_token, $oauth_token_secret); - return $token; + $params = array(); + + if (!is_null($verifier)) { + $params['oauth_verifier'] = $verifier; + } + + $response = $this->oAuthPost($url, $params); + + $arr = array(); + parse_str($response, $arr); + + $token = $arr['oauth_token']; + $secret = $arr['oauth_token_secret']; + + if (isset($token) && isset($secret)) { + $token = new OAuthToken($token, $secret); + return $token; + } else { + throw new OAuthClientException( + 'Could not get a access token from Twitter.' + ); + } } /** - * Use HTTP GET to make a signed OAuth request + * Use HTTP GET to make a signed OAuth requesta * - * @param string $url OAuth endpoint + * @param string $url OAuth request token endpoint + * @param array $params additional parameters * * @return mixed the request */ - function oAuthGet($url) + function oAuthGet($url, $params = null) { $request = OAuthRequest::from_consumer_and_token($this->consumer, - $this->token, 'GET', $url, null); + $this->token, 'GET', $url, $params); $request->sign_request($this->sha1_method, $this->consumer, $this->token); diff --git a/plugins/TwitterBridge/twitterauthorization.php b/plugins/TwitterBridge/twitterauthorization.php index 6822d33dd..c154932bb 100644 --- a/plugins/TwitterBridge/twitterauthorization.php +++ b/plugins/TwitterBridge/twitterauthorization.php @@ -56,6 +56,7 @@ class TwitterauthorizationAction extends Action var $tw_fields = null; var $access_token = null; var $signin = null; + var $verifier = null; /** * Initialize class members. Looks for 'oauth_token' parameter. @@ -70,6 +71,7 @@ class TwitterauthorizationAction extends Action $this->signin = $this->boolean('signin'); $this->oauth_token = $this->arg('oauth_token'); + $this->verifier = $this->arg('oauth_verifier'); return true; } @@ -160,8 +162,7 @@ class TwitterauthorizationAction extends Action // Get a new request token and authorize it $client = new TwitterOAuthClient(); - $req_tok = - $client->getRequestToken(TwitterOAuthClient::$requestTokenURL); + $req_tok = $client->getRequestToken(); // Sock the request token away in the session temporarily @@ -171,7 +172,7 @@ class TwitterauthorizationAction extends Action $auth_link = $client->getAuthorizeLink($req_tok, $this->signin); } catch (OAuthClientException $e) { - $msg = sprintf('OAuth client cURL error - code: %1s, msg: %2s', + $msg = sprintf('OAuth client error - code: %1s, msg: %2s', $e->getCode(), $e->getMessage()); $this->serverError(_m('Couldn\'t link your Twitter account.')); } @@ -187,7 +188,6 @@ class TwitterauthorizationAction extends Action */ function saveAccessToken() { - // Check to make sure Twitter returned the same request // token we sent them @@ -204,7 +204,7 @@ class TwitterauthorizationAction extends Action // Exchange the request token for an access token - $atok = $client->getAccessToken(TwitterOAuthClient::$accessTokenURL); + $atok = $client->getAccessToken($this->verifier); // Test the access token and get the user's Twitter info diff --git a/plugins/TwitterBridge/twitteroauthclient.php b/plugins/TwitterBridge/twitteroauthclient.php index 277e7ab40..ba45b533d 100644 --- a/plugins/TwitterBridge/twitteroauthclient.php +++ b/plugins/TwitterBridge/twitteroauthclient.php @@ -91,6 +91,19 @@ class TwitterOAuthClient extends OAuthClient } } + /** + * Gets a request token from Twitter + * + * @return OAuthToken $token the request token + */ + function getRequestToken() + { + return parent::getRequestToken( + self::$requestTokenURL, + common_local_url('twitterauthorization') + ); + } + /** * Builds a link to Twitter's endpoint for authorizing a request token * @@ -107,6 +120,21 @@ class TwitterOAuthClient extends OAuthClient common_local_url('twitterauthorization')); } + /** + * Fetches an access token from Twitter + * + * @param string $verifier 1.0a verifier + * + * @return OAuthToken $token the access token + */ + function getAccessToken($verifier = null) + { + return parent::getAccessToken( + self::$accessTokenURL, + $verifier + ); + } + /** * Calls Twitter's /account/verify_credentials API method * -- cgit v1.2.3-54-g00ecf From 5a357d539902089030ae2e8aa645813c0e8173ec Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 16 Feb 2010 09:58:33 -0500 Subject: change find() to staticGet() to use cache --- plugins/OStatus/classes/Ostatus_profile.php | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'plugins') diff --git a/plugins/OStatus/classes/Ostatus_profile.php b/plugins/OStatus/classes/Ostatus_profile.php index 733d8843b..b750e1883 100644 --- a/plugins/OStatus/classes/Ostatus_profile.php +++ b/plugins/OStatus/classes/Ostatus_profile.php @@ -29,15 +29,15 @@ PuSH subscription flow: generate random verification token save to verify_token sends a sub request to the hub... - + main/push/callback hub sends confirmation back to us via GET We verify the request, then echo back the challenge. On our end, we save the time we subscribed and the lease expiration - + main/push/callback hub sends us updates via POST - + */ class FeedDBException extends FeedSubException @@ -75,7 +75,6 @@ class Ostatus_profile extends Memcached_DataObject public $created; public $lastupdate; - public /*static*/ function staticGet($k, $v=null) { return parent::staticGet(__CLASS__, $k, $v); @@ -107,7 +106,7 @@ class Ostatus_profile extends Memcached_DataObject 'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL, 'lastupdate' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL); } - + static function schemaDef() { return array(new ColumnDef('id', 'integer', @@ -486,7 +485,7 @@ class Ostatus_profile extends Memcached_DataObject } if ($this->salmonuri) { $text = 'update'; // @fixme - $id = 'tag:' . common_config('site', 'server') . + $id = 'tag:' . common_config('site', 'server') . ':' . $verb . ':' . $actor->id . ':' . time(); // @fixme @@ -589,7 +588,7 @@ class Ostatus_profile extends Memcached_DataObject require_once "XML/Feed/Parser.php"; $feed = new XML_Feed_Parser($xml, false, false, true); $munger = new FeedMunger($feed); - + $hits = 0; foreach ($feed as $index => $entry) { // @fixme this might sort in wrong order if we get multiple updates @@ -598,9 +597,10 @@ class Ostatus_profile extends Memcached_DataObject // Double-check for oldies // @fixme this could explode horribly for multiple feeds on a blog. sigh - $dupe = new Notice(); - $dupe->uri = $notice->uri; - if ($dupe->find(true)) { + + $dupe = Notice::staticGet('uri', $notice->uri); + + if (!empty($dupe)) { common_log(LOG_WARNING, __METHOD__ . ": tried to save dupe notice for entry {$notice->uri} of feed {$this->feeduri}"); continue; } -- cgit v1.2.3-54-g00ecf From 813451c9f9aac0690019d936dea2bf9d37ca6c13 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 16 Feb 2010 10:18:23 -0500 Subject: add a couple of FIXME comments --- plugins/OStatus/OStatusPlugin.php | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'plugins') diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index 276ca1b3d..3b1329d6c 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -218,6 +218,9 @@ class OStatusPlugin extends Plugin $count = preg_match_all('/(\w+\.)*\w+@(\w+\.)*\w+(\w+\-\w+)*\.\w+/', $notice->content, $matches); if ($count) { foreach ($matches[0] as $webfinger) { + + // FIXME: look up locally first + // Check to see if we've got an actual webfinger $w = new Webfinger; @@ -238,6 +241,8 @@ class OStatusPlugin extends Plugin continue; } + // FIXME: this needs to go out in a queue handler + $xml = ''; $xml .= $notice->asAtomEntry(); -- cgit v1.2.3-54-g00ecf From a8c2a8261e79dd2047caaa158e7c5609d9835449 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 16 Feb 2010 11:06:10 -0500 Subject: move some nickname-guessing code to lib/util.php from OpenID --- lib/util.php | 54 +++++++++++++++++++++++++++++++++++- plugins/OpenID/finishopenidlogin.php | 47 ++----------------------------- 2 files changed, 55 insertions(+), 46 deletions(-) (limited to 'plugins') diff --git a/lib/util.php b/lib/util.php index 209dc2254..ae812e8cf 100644 --- a/lib/util.php +++ b/lib/util.php @@ -1007,7 +1007,6 @@ function common_enqueue_notice($notice) if (Event::hasHandler('HandleQueuedNotice')) { $transports[] = 'plugin'; } - $xmpp = common_config('xmpp', 'enabled'); @@ -1574,3 +1573,56 @@ function common_client_ip() return array($proxy, $ip); } + +function common_url_to_nickname($url) +{ + static $bad = array('query', 'user', 'password', 'port', 'fragment'); + + $parts = parse_url($url); + + # If any of these parts exist, this won't work + + foreach ($bad as $badpart) { + if (array_key_exists($badpart, $parts)) { + return null; + } + } + + # We just have host and/or path + + # If it's just a host... + if (array_key_exists('host', $parts) && + (!array_key_exists('path', $parts) || strcmp($parts['path'], '/') == 0)) + { + $hostparts = explode('.', $parts['host']); + + # Try to catch common idiom of nickname.service.tld + + if ((count($hostparts) > 2) && + (strlen($hostparts[count($hostparts) - 2]) > 3) && # try to skip .co.uk, .com.au + (strcmp($hostparts[0], 'www') != 0)) + { + return common_nicknamize($hostparts[0]); + } else { + # Do the whole hostname + return common_nicknamize($parts['host']); + } + } else { + if (array_key_exists('path', $parts)) { + # Strip starting, ending slashes + $path = preg_replace('@/$@', '', $parts['path']); + $path = preg_replace('@^/@', '', $path); + if (strpos($path, '/') === false) { + return common_nicknamize($path); + } + } + } + + return null; +} + +function common_nicknamize($str) +{ + $str = preg_replace('/\W/', '', $str); + return strtolower($str); +} diff --git a/plugins/OpenID/finishopenidlogin.php b/plugins/OpenID/finishopenidlogin.php index d25ce696c..438a728d8 100644 --- a/plugins/OpenID/finishopenidlogin.php +++ b/plugins/OpenID/finishopenidlogin.php @@ -438,49 +438,7 @@ class FinishopenidloginAction extends Action function urlToNickname($openid) { - static $bad = array('query', 'user', 'password', 'port', 'fragment'); - - $parts = parse_url($openid); - - # If any of these parts exist, this won't work - - foreach ($bad as $badpart) { - if (array_key_exists($badpart, $parts)) { - return null; - } - } - - # We just have host and/or path - - # If it's just a host... - if (array_key_exists('host', $parts) && - (!array_key_exists('path', $parts) || strcmp($parts['path'], '/') == 0)) - { - $hostparts = explode('.', $parts['host']); - - # Try to catch common idiom of nickname.service.tld - - if ((count($hostparts) > 2) && - (strlen($hostparts[count($hostparts) - 2]) > 3) && # try to skip .co.uk, .com.au - (strcmp($hostparts[0], 'www') != 0)) - { - return $this->nicknamize($hostparts[0]); - } else { - # Do the whole hostname - return $this->nicknamize($parts['host']); - } - } else { - if (array_key_exists('path', $parts)) { - # Strip starting, ending slashes - $path = preg_replace('@/$@', '', $parts['path']); - $path = preg_replace('@^/@', '', $path); - if (strpos($path, '/') === false) { - return $this->nicknamize($path); - } - } - } - - return null; + return common_url_to_nickname($openid); } function xriToNickname($xri) @@ -510,7 +468,6 @@ class FinishopenidloginAction extends Action function nicknamize($str) { - $str = preg_replace('/\W/', '', $str); - return strtolower($str); + return common_nicknamize($str); } } -- cgit v1.2.3-54-g00ecf From c74aea589d5a79d7048470d44e457dffc8919ad3 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 16 Feb 2010 09:01:59 -0800 Subject: Stomp queue restructuring for mass scalability: - Multiplexing queues into groups and for multiple sites. - Sharing vs breakout configurable per site and per queue via $config['queue']['breakout'] - Detect how many times a message is redelivered, discard if it's killed too many daemons - count configurable with $config['queue']['max_retries'] - can dump the items to files in $config['queue']['dead_letter_dir'] Queue daemon memory & resource leak fixes: - avoid unnecessary reconnections to memcached server (switch persistent connections back in on second initialization, assuming it's child process) - monkey-patch for leaky .ini loads in DB_DataObject::databaseStructure() - was leaking 200k per active switch - applied leak fixes to Status_network as well, using intermediate base Safe_DataObject for both it and Memcache_DataObject Misc queue fixes: - correct handling of child processes exiting due to signal termination instead of regular exit - shutdown instead of infinite respawn loop if we're already past the soft memory limit at startup - Added --all option for xmppdaemon... still opens one xmpp connection per site that has xmpp active Cache updates: - add Cache::increment() method with native support for memcached atomic increment --- classes/Memcached_DataObject.php | 52 +----- classes/Safe_DataObject.php | 250 ++++++++++++++++++++++++++ classes/Status_network.php | 16 +- lib/cache.php | 26 +++ lib/dbqueuemanager.php | 7 +- lib/default.php | 8 +- lib/iomanager.php | 5 +- lib/iomaster.php | 74 +++----- lib/queued_xmpp.php | 2 +- lib/queuemanager.php | 97 ++++++---- lib/spawningdaemon.php | 16 +- lib/statusnet.php | 54 ++++++ lib/stompqueuemanager.php | 369 ++++++++++++++++++--------------------- lib/xmppmanager.php | 8 +- plugins/MemcachePlugin.php | 33 +++- scripts/queuedaemon.php | 20 +-- scripts/xmppdaemon.php | 28 ++- 17 files changed, 697 insertions(+), 368 deletions(-) create mode 100644 classes/Safe_DataObject.php (limited to 'plugins') diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index ab65c30ce..16c3d906c 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -19,57 +19,8 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } -class Memcached_DataObject extends DB_DataObject +class Memcached_DataObject extends Safe_DataObject { - /** - * Destructor to free global memory resources associated with - * this data object when it's unset or goes out of scope. - * DB_DataObject doesn't do this yet by itself. - */ - - function __destruct() - { - $this->free(); - if (method_exists('DB_DataObject', '__destruct')) { - parent::__destruct(); - } - } - - /** - * Magic function called at serialize() time. - * - * We use this to drop a couple process-specific references - * from DB_DataObject which can cause trouble in future - * processes. - * - * @return array of variable names to include in serialization. - */ - function __sleep() - { - $vars = array_keys(get_object_vars($this)); - $skip = array('_DB_resultid', '_link_loaded'); - return array_diff($vars, $skip); - } - - /** - * Magic function called at unserialize() time. - * - * Clean out some process-specific variables which might - * be floating around from a previous process's cached - * objects. - * - * Old cached objects may still have them. - */ - function __wakeup() - { - // Refers to global state info from a previous process. - // Clear this out so we don't accidentally break global - // state in *this* process. - $this->_DB_resultid = null; - // We don't have any local DBO refs, so clear these out. - $this->_link_loaded = false; - } - /** * Wrapper for DB_DataObject's static lookup using memcached * as backing instead of an in-process cache array. @@ -579,3 +530,4 @@ class Memcached_DataObject extends DB_DataObject return $c->set($cacheKey, $value); } } + diff --git a/classes/Safe_DataObject.php b/classes/Safe_DataObject.php new file mode 100644 index 000000000..021f7b506 --- /dev/null +++ b/classes/Safe_DataObject.php @@ -0,0 +1,250 @@ +. + */ + +if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } + +/** + * Extended DB_DataObject to improve a few things: + * - free global resources from destructor + * - remove bogus global references from serialized objects + * - don't leak memory when loading already-used .ini files + * (eg when using the same schema on thousands of databases) + */ +class Safe_DataObject extends DB_DataObject +{ + /** + * Destructor to free global memory resources associated with + * this data object when it's unset or goes out of scope. + * DB_DataObject doesn't do this yet by itself. + */ + + function __destruct() + { + $this->free(); + if (method_exists('DB_DataObject', '__destruct')) { + parent::__destruct(); + } + } + + /** + * Magic function called at serialize() time. + * + * We use this to drop a couple process-specific references + * from DB_DataObject which can cause trouble in future + * processes. + * + * @return array of variable names to include in serialization. + */ + function __sleep() + { + $vars = array_keys(get_object_vars($this)); + $skip = array('_DB_resultid', '_link_loaded'); + return array_diff($vars, $skip); + } + + /** + * Magic function called at unserialize() time. + * + * Clean out some process-specific variables which might + * be floating around from a previous process's cached + * objects. + * + * Old cached objects may still have them. + */ + function __wakeup() + { + // Refers to global state info from a previous process. + // Clear this out so we don't accidentally break global + // state in *this* process. + $this->_DB_resultid = null; + // We don't have any local DBO refs, so clear these out. + $this->_link_loaded = false; + } + + + /** + * Work around memory-leak bugs... + * Had to copy-paste the whole function in order to patch a couple lines of it. + * Would be nice if this code was better factored. + * + * @param optional string name of database to assign / read + * @param optional array structure of database, and keys + * @param optional array table links + * + * @access public + * @return true or PEAR:error on wrong paramenters.. or false if no file exists.. + * or the array(tablename => array(column_name=>type)) if called with 1 argument.. (databasename) + */ + function databaseStructure() + { + + global $_DB_DATAOBJECT; + + // Assignment code + + if ($args = func_get_args()) { + + if (count($args) == 1) { + + // this returns all the tables and their structure.. + if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { + $this->debug("Loading Generator as databaseStructure called with args",1); + } + + $x = new DB_DataObject; + $x->_database = $args[0]; + $this->_connect(); + $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; + + $tables = $DB->getListOf('tables'); + class_exists('DB_DataObject_Generator') ? '' : + require_once 'DB/DataObject/Generator.php'; + + foreach($tables as $table) { + $y = new DB_DataObject_Generator; + $y->fillTableSchema($x->_database,$table); + } + return $_DB_DATAOBJECT['INI'][$x->_database]; + } else { + + $_DB_DATAOBJECT['INI'][$args[0]] = isset($_DB_DATAOBJECT['INI'][$args[0]]) ? + $_DB_DATAOBJECT['INI'][$args[0]] + $args[1] : $args[1]; + + if (isset($args[1])) { + $_DB_DATAOBJECT['LINKS'][$args[0]] = isset($_DB_DATAOBJECT['LINKS'][$args[0]]) ? + $_DB_DATAOBJECT['LINKS'][$args[0]] + $args[2] : $args[2]; + } + return true; + } + + } + + + + if (!$this->_database) { + $this->_connect(); + } + + // loaded already? + if (!empty($_DB_DATAOBJECT['INI'][$this->_database])) { + + // database loaded - but this is table is not available.. + if ( + empty($_DB_DATAOBJECT['INI'][$this->_database][$this->__table]) + && !empty($_DB_DATAOBJECT['CONFIG']['proxy']) + ) { + if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { + $this->debug("Loading Generator to fetch Schema",1); + } + class_exists('DB_DataObject_Generator') ? '' : + require_once 'DB/DataObject/Generator.php'; + + + $x = new DB_DataObject_Generator; + $x->fillTableSchema($this->_database,$this->__table); + } + return true; + } + + + if (empty($_DB_DATAOBJECT['CONFIG'])) { + DB_DataObject::_loadConfig(); + } + + // if you supply this with arguments, then it will take those + // as the database and links array... + + $schemas = isset($_DB_DATAOBJECT['CONFIG']['schema_location']) ? + array("{$_DB_DATAOBJECT['CONFIG']['schema_location']}/{$this->_database}.ini") : + array() ; + + if (isset($_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"])) { + $schemas = is_array($_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"]) ? + $_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"] : + explode(PATH_SEPARATOR,$_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"]); + } + + + /* BEGIN CHANGED FROM UPSTREAM */ + $_DB_DATAOBJECT['INI'][$this->_database] = $this->parseIniFiles($schemas); + /* END CHANGED FROM UPSTREAM */ + + // now have we loaded the structure.. + + if (!empty($_DB_DATAOBJECT['INI'][$this->_database][$this->__table])) { + return true; + } + // - if not try building it.. + if (!empty($_DB_DATAOBJECT['CONFIG']['proxy'])) { + class_exists('DB_DataObject_Generator') ? '' : + require_once 'DB/DataObject/Generator.php'; + + $x = new DB_DataObject_Generator; + $x->fillTableSchema($this->_database,$this->__table); + // should this fail!!!??? + return true; + } + $this->debug("Cant find database schema: {$this->_database}/{$this->__table} \n". + "in links file data: " . print_r($_DB_DATAOBJECT['INI'],true),"databaseStructure",5); + // we have to die here!! - it causes chaos if we dont (including looping forever!) + $this->raiseError( "Unable to load schema for database and table (turn debugging up to 5 for full error message)", DB_DATAOBJECT_ERROR_INVALIDARGS, PEAR_ERROR_DIE); + return false; + } + + /** For parseIniFiles */ + protected static $iniCache = array(); + + /** + * When switching site configurations, DB_DataObject was loading its + * .ini files over and over, leaking gobs of memory. + * This refactored helper function uses a local cache of .ini files + * to minimize the leaks. + * + * @param array of .ini file names $schemas + * @return array + */ + protected function parseIniFiles($schemas) + { + $key = implode("|", $schemas); + if (!isset(Safe_DataObject::$iniCache[$key])) { + $data = array(); + foreach ($schemas as $ini) { + if (file_exists($ini) && is_file($ini)) { + $data = array_merge($data, parse_ini_file($ini, true)); + + if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { + if (!is_readable ($ini)) { + $this->debug("ini file is not readable: $ini","databaseStructure",1); + } else { + $this->debug("Loaded ini file: $ini","databaseStructure",1); + } + } + } else { + if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { + $this->debug("Missing ini file: $ini","databaseStructure",1); + } + } + } + Safe_DataObject::$iniCache[$key] = $data; + } + + return Safe_DataObject::$iniCache[$key]; + } +} + diff --git a/classes/Status_network.php b/classes/Status_network.php index 4bda24b6a..a452c32ce 100644 --- a/classes/Status_network.php +++ b/classes/Status_network.php @@ -21,7 +21,7 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } -class Status_network extends DB_DataObject +class Status_network extends Safe_DataObject { ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -57,6 +57,7 @@ class Status_network extends DB_DataObject ###END_AUTOCODE static $cache = null; + static $cacheInitialized = false; static $base = null; static $wildcard = null; @@ -78,11 +79,15 @@ class Status_network extends DB_DataObject if (class_exists('Memcache')) { self::$cache = new Memcache(); - // Can't close persistent connections, making forking painful. + // If we're a parent command-line process we need + // to be able to close out the connection after + // forking, so disable persistence. // - // @fixme only do this in *parent* CLI processes. - // single-process and child-processes *should* use persistent. - $persist = php_sapi_name() != 'cli'; + // We'll turn it back on again the second time + // through which will either be in a child process, + // or a single-process script which is switching + // configurations. + $persist = php_sapi_name() != 'cli' || self::$cacheInitialized; if (is_array($servers)) { foreach($servers as $server) { self::$cache->addServer($server, 11211, $persist); @@ -90,6 +95,7 @@ class Status_network extends DB_DataObject } else { self::$cache->addServer($servers, 11211, $persist); } + self::$cacheInitialized = true; } self::$base = $dbname; diff --git a/lib/cache.php b/lib/cache.php index 635c96ad4..b3ec7534f 100644 --- a/lib/cache.php +++ b/lib/cache.php @@ -157,6 +157,32 @@ class Cache return $success; } + /** + * Atomically increment an existing numeric value. + * Existing expiration time should remain unchanged, if any. + * + * @param string $key The key to use for lookups + * @param int $step Amount to increment (default 1) + * + * @return mixed incremented value, or false if not set. + */ + function increment($key, $step=1) + { + $value = false; + if (Event::handle('StartCacheIncrement', array(&$key, &$step, &$value))) { + // Fallback is not guaranteed to be atomic, + // and may original expiry value. + $value = $this->get($key); + if ($value !== false) { + $value += $step; + $ok = $this->set($key, $value); + $got = $this->get($key); + } + Event::handle('EndCacheIncrement', array($key, $step, $value)); + } + return $value; + } + /** * Delete the value associated with a key * diff --git a/lib/dbqueuemanager.php b/lib/dbqueuemanager.php index c6350fc66..3032e4ec7 100644 --- a/lib/dbqueuemanager.php +++ b/lib/dbqueuemanager.php @@ -72,7 +72,7 @@ class DBQueueManager extends QueueManager public function poll() { $this->_log(LOG_DEBUG, 'Checking for notices...'); - $qi = Queue_item::top($this->getQueues()); + $qi = Queue_item::top($this->activeQueues()); if (empty($qi)) { $this->_log(LOG_DEBUG, 'No notices waiting; idling.'); return false; @@ -142,9 +142,4 @@ class DBQueueManager extends QueueManager $this->stats('error', $queue); } - - protected function _log($level, $msg) - { - common_log($level, 'DBQueueManager: '.$msg); - } } diff --git a/lib/default.php b/lib/default.php index bf4b83718..a74cccae1 100644 --- a/lib/default.php +++ b/lib/default.php @@ -81,7 +81,7 @@ $default = 'subsystem' => 'db', # default to database, or 'stomp' 'stomp_server' => null, 'queue_basename' => '/queue/statusnet/', - 'control_channel' => '/topic/statusnet-control', // broadcasts to all queue daemons + 'control_channel' => '/topic/statusnet/control', // broadcasts to all queue daemons 'stomp_username' => null, 'stomp_password' => null, 'stomp_persistent' => true, // keep items across queue server restart, if persistence is enabled @@ -91,6 +91,12 @@ $default = 'spawndelay' => 1, // Wait at least N seconds between (re)spawns of child processes to avoid slamming the queue server with subscription startup 'debug_memory' => false, // true to spit memory usage to log 'inboxes' => true, // true to do inbox distribution & output queueing from in background via 'distrib' queue + 'breakout' => array('*' => 'shared'), // set global or per-handler queue breakout + // 'shared': use a shared queue for all sites + // 'handler': share each/this handler over multiple sites + // 'site': break out for each/this handler on this site + 'max_retries' => 10, // drop messages after N failed attempts to process (Stomp) + 'dead_letter_dir' => false, // set to directory to save dropped messages into (Stomp) ), 'license' => array('type' => 'cc', # can be 'cc', 'allrightsreserved', 'private' diff --git a/lib/iomanager.php b/lib/iomanager.php index ee2ff958b..217599a6d 100644 --- a/lib/iomanager.php +++ b/lib/iomanager.php @@ -59,9 +59,10 @@ abstract class IoManager * your manager about each site you'll have to handle so you * can do any necessary per-site setup. * - * @param string $site target site server name + * The new site will be the currently live configuration during + * this call. */ - public function addSite($site) + public function addSite() { /* no-op */ } diff --git a/lib/iomaster.php b/lib/iomaster.php index bcab3542b..54e2dfe84 100644 --- a/lib/iomaster.php +++ b/lib/iomaster.php @@ -56,9 +56,9 @@ abstract class IoMaster $this->multiSite = $multiSite; } if ($this->multiSite) { - $this->sites = $this->findAllSites(); + $this->sites = StatusNet::findAllSites(); } else { - $this->sites = array(common_config('site', 'server')); + $this->sites = array(StatusNet::currentSite()); } if (empty($this->sites)) { @@ -66,9 +66,7 @@ abstract class IoMaster } foreach ($this->sites as $site) { - if ($site != common_config('site', 'server')) { - StatusNet::init($site); - } + StatusNet::switchSite($site); $this->initManagers(); } } @@ -81,58 +79,32 @@ abstract class IoMaster */ abstract function initManagers(); - /** - * Pull all local sites from status_network table. - * @return array of hostnames - */ - protected function findAllSites() - { - $hosts = array(); - $sn = new Status_network(); - $sn->find(); - while ($sn->fetch()) { - $hosts[] = $sn->getServerName(); - } - return $hosts; - } - /** * Instantiate an i/o manager class for the current site. * If a multi-site capable handler is already present, * we don't need to build a new one. * - * @param string $class + * @param mixed $manager class name (to run $class::get()) or object */ - protected function instantiate($class) + protected function instantiate($manager) { - if (isset($this->singletons[$class])) { - // Already instantiated a multi-site-capable handler. - // Just let it know it should listen to this site too! - $this->singletons[$class]->addSite(common_config('site', 'server')); - return; + if (is_string($manager)) { + $manager = call_user_func(array($class, 'get')); } - $manager = $this->getManager($class); - - if ($this->multiSite) { - $caps = $manager->multiSite(); - if ($caps == IoManager::SINGLE_ONLY) { + $caps = $manager->multiSite(); + if ($caps == IoManager::SINGLE_ONLY) { + if ($this->multiSite) { throw new Exception("$class can't run with --all; aborting."); } - if ($caps == IoManager::INSTANCE_PER_PROCESS) { - // Save this guy for later! - // We'll only need the one to cover multiple sites. - $this->singletons[$class] = $manager; - $manager->addSite(common_config('site', 'server')); - } + } else if ($caps == IoManager::INSTANCE_PER_PROCESS) { + $manager->addSite(); } - $this->managers[] = $manager; - } - - protected function getManager($class) - { - return call_user_func(array($class, 'get')); + if (!in_array($manager, $this->managers, true)) { + // Only need to save singletons once + $this->managers[] = $manager; + } } /** @@ -146,6 +118,7 @@ abstract class IoMaster { $this->logState('init'); $this->start(); + $this->checkMemory(false); while (!$this->shutdown) { $timeouts = array_values($this->pollTimeouts); @@ -209,17 +182,24 @@ abstract class IoMaster /** * Check runtime memory usage, possibly triggering a graceful shutdown * and thread respawn if we've crossed the soft limit. + * + * @param boolean $respawn if false we'll shut down instead of respawning */ - protected function checkMemory() + protected function checkMemory($respawn=true) { $memoryLimit = $this->softMemoryLimit(); if ($memoryLimit > 0) { $usage = memory_get_usage(); if ($usage > $memoryLimit) { common_log(LOG_INFO, "Queue thread hit soft memory limit ($usage > $memoryLimit); gracefully restarting."); - $this->requestRestart(); + if ($respawn) { + $this->requestRestart(); + } else { + $this->requestShutdown(); + } } else if (common_config('queue', 'debug_memory')) { - common_log(LOG_DEBUG, "Memory usage $usage"); + $fmt = number_format($usage); + common_log(LOG_DEBUG, "Memory usage $fmt"); } } } diff --git a/lib/queued_xmpp.php b/lib/queued_xmpp.php index 4b890c4ca..fdd074db2 100644 --- a/lib/queued_xmpp.php +++ b/lib/queued_xmpp.php @@ -63,7 +63,7 @@ class Queued_XMPP extends XMPPHP_XMPP */ public function send($msg, $timeout=NULL) { - $qm = QueueManager::get(); + $qm = QueueManager::get('xmppout'); $qm->enqueue(strval($msg), 'xmppout'); } diff --git a/lib/queuemanager.php b/lib/queuemanager.php index 149617eb5..8f8c8f133 100644 --- a/lib/queuemanager.php +++ b/lib/queuemanager.php @@ -39,9 +39,10 @@ abstract class QueueManager extends IoManager { static $qm = null; - public $master = null; - public $handlers = array(); - public $groups = array(); + protected $master = null; + protected $handlers = array(); + protected $groups = array(); + protected $activeGroups = array(); /** * Factory function to pull the appropriate QueueManager object @@ -215,55 +216,64 @@ abstract class QueueManager extends IoManager if (class_exists($class)) { return new $class(); } else { - common_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'"); + $this->_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'"); } } else { - common_log(LOG_ERR, "Requested handler for unkown queue '$queue'"); + $this->_log(LOG_ERR, "Requested handler for unkown queue '$queue'"); } return null; } /** * Get a list of registered queue transport names to be used - * for this daemon. + * for listening in this daemon. * * @return array of strings */ - function getQueues() + function activeQueues() { - $group = $this->activeGroup(); - return array_keys($this->groups[$group]); + $queues = array(); + foreach ($this->activeGroups as $group) { + if (isset($this->groups[$group])) { + $queues = array_merge($queues, $this->groups[$group]); + } + } + + return array_keys($queues); } /** - * Initialize the list of queue handlers + * Initialize the list of queue handlers for the current site. * * @event StartInitializeQueueManager * @event EndInitializeQueueManager */ function initialize() { - // @fixme we'll want to be able to listen to particular queues... + $this->handlers = array(); + $this->groups = array(); + $this->groupsByTransport = array(); + if (Event::handle('StartInitializeQueueManager', array($this))) { - $this->connect('plugin', 'PluginQueueHandler'); + $this->connect('distrib', 'DistribQueueHandler'); $this->connect('omb', 'OmbQueueHandler'); $this->connect('ping', 'PingQueueHandler'); - $this->connect('distrib', 'DistribQueueHandler'); if (common_config('sms', 'enabled')) { $this->connect('sms', 'SmsQueueHandler'); } // XMPP output handlers... - $this->connect('jabber', 'JabberQueueHandler'); - $this->connect('public', 'PublicQueueHandler'); - // @fixme this should get an actual queue - //$this->connect('confirm', 'XmppConfirmHandler'); + if (common_config('xmpp', 'enabled')) { + // Delivery prep, read by queuedaemon.php: + $this->connect('jabber', 'JabberQueueHandler'); + $this->connect('public', 'PublicQueueHandler'); + + // Raw output, read by xmppdaemon.php: + $this->connect('xmppout', 'XmppOutQueueHandler', 'xmpp'); + } // For compat with old plugins not registering their own handlers. $this->connect('plugin', 'PluginQueueHandler'); - - $this->connect('xmppout', 'XmppOutQueueHandler', 'xmppdaemon'); - } Event::handle('EndInitializeQueueManager', array($this)); } @@ -276,25 +286,41 @@ abstract class QueueManager extends IoManager * @param string $class * @param string $group */ - public function connect($transport, $class, $group='queuedaemon') + public function connect($transport, $class, $group='main') { $this->handlers[$transport] = $class; $this->groups[$group][$transport] = $class; + $this->groupsByTransport[$transport] = $group; } /** - * @return string queue group to use for this request + * Set the active group which will be used for listening. + * @param string $group */ - function activeGroup() + function setActiveGroup($group) { - $group = 'queuedaemon'; - if ($this->master) { - // hack hack - if ($this->master instanceof XmppMaster) { - return 'xmppdaemon'; - } + $this->activeGroups = array($group); + } + + /** + * Set the active group(s) which will be used for listening. + * @param array $groups + */ + function setActiveGroups($groups) + { + $this->activeGroups = $groups; + } + + /** + * @return string queue group for this queue + */ + function queueGroup($queue) + { + if (isset($this->groupsByTransport[$queue])) { + return $this->groupsByTransport[$queue]; + } else { + throw new Exception("Requested group for unregistered transport $queue"); } - return $group; } /** @@ -318,4 +344,15 @@ abstract class QueueManager extends IoManager $monitor->stats($key, $owners); } } + + protected function _log($level, $msg) + { + $class = get_class($this); + if ($this->activeGroups) { + $groups = ' (' . implode(',', $this->activeGroups) . ')'; + } else { + $groups = ''; + } + common_log($level, "$class$groups: $msg"); + } } diff --git a/lib/spawningdaemon.php b/lib/spawningdaemon.php index 862cbb4fa..fd9ae4355 100644 --- a/lib/spawningdaemon.php +++ b/lib/spawningdaemon.php @@ -90,18 +90,24 @@ abstract class SpawningDaemon extends Daemon while (count($children) > 0) { $status = null; $pid = pcntl_wait($status); - if ($pid > 0 && pcntl_wifexited($status)) { - $exitCode = pcntl_wexitstatus($status); - + if ($pid > 0) { $i = array_search($pid, $children); if ($i === false) { - $this->log(LOG_ERR, "Unrecognized child pid $pid exited with status $exitCode"); + $this->log(LOG_ERR, "Ignoring exit of unrecognized child pid $pid"); continue; } + if (pcntl_wifexited($status)) { + $exitCode = pcntl_wexitstatus($status); + $info = "status $exitCode"; + } else if (pcntl_wifsignaled($status)) { + $exitCode = self::EXIT_ERR; + $signal = pcntl_wtermsig($status); + $info = "signal $signal"; + } unset($children[$i]); if ($this->shouldRespawn($exitCode)) { - $this->log(LOG_INFO, "Thread $i pid $pid exited with status $exitCode; respawing."); + $this->log(LOG_INFO, "Thread $i pid $pid exited with $info; respawing."); $pid = pcntl_fork(); if ($pid < 0) { diff --git a/lib/statusnet.php b/lib/statusnet.php index 29e903026..9c7ede5a5 100644 --- a/lib/statusnet.php +++ b/lib/statusnet.php @@ -101,6 +101,60 @@ class StatusNet self::initPlugins(); } + /** + * Get identifier of the currently active site configuration + * @return string + */ + public static function currentSite() + { + return common_config('site', 'nickname'); + } + + /** + * Change site configuration to site specified by nickname, + * if set up via Status_network. If not, sites other than + * the current will fail horribly. + * + * May throw exception or trigger a fatal error if the given + * site is missing or configured incorrectly. + * + * @param string $nickname + */ + public static function switchSite($nickname) + { + if ($nickname == StatusNet::currentSite()) { + return true; + } + + $sn = Status_network::staticGet($nickname); + if (empty($sn)) { + return false; + throw new Exception("No such site nickname '$nickname'"); + } + + $server = $sn->getServerName(); + StatusNet::init($server); + } + + /** + * Pull all local sites from status_network table. + * + * Behavior undefined if site is not configured via Status_network. + * + * @return array of nicknames + */ + public static function findAllSites() + { + $sites = array(); + $sn = new Status_network(); + $sn->find(); + while ($sn->fetch()) { + $sites[] = $sn->nickname; + } + return $sites; + } + + /** * Fire initialization events for all instantiated plugins. */ diff --git a/lib/stompqueuemanager.php b/lib/stompqueuemanager.php index cd62c25bd..bfeeb23b7 100644 --- a/lib/stompqueuemanager.php +++ b/lib/stompqueuemanager.php @@ -63,6 +63,7 @@ class StompQueueManager extends QueueManager $this->password = common_config('queue', 'stomp_password'); $this->base = common_config('queue', 'queue_basename'); $this->control = common_config('queue', 'control_channel'); + $this->subscriptions = array($this->control => $this->control); } /** @@ -75,17 +76,25 @@ class StompQueueManager extends QueueManager } /** - * Record each site we'll be handling input for in this process, - * so we can listen to the necessary queues for it. - * - * @fixme possibly actually do subscription here to save another - * loop over all sites later? - * @fixme possibly don't assume it's the current site + * Record queue subscriptions we'll need to handle the current site. */ - public function addSite($server) + public function addSite() { - $this->sites[] = $server; + $this->sites[] = StatusNet::currentSite(); + + // Set up handlers active for this site... $this->initialize(); + + foreach ($this->activeGroups as $group) { + if (isset($this->groups[$group])) { + // Actual queues may be broken out or consolidated... + // Subscribe to all the target queues we'll need. + foreach ($this->groups[$group] as $transport => $class) { + $target = $this->queueName($transport); + $this->subscriptions[$target] = $target; + } + } + } } /** @@ -121,59 +130,11 @@ class StompQueueManager extends QueueManager } /** - * Instantiate the appropriate QueueHandler class for the given queue. + * Saves an object into the queue item table. * + * @param mixed $object * @param string $queue - * @return mixed QueueHandler or null - */ - function getHandler($queue) - { - $handlers = $this->handlers[$this->currentSite()]; - if (isset($handlers[$queue])) { - $class = $handlers[$queue]; - if (class_exists($class)) { - return new $class(); - } else { - common_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'"); - } - } else { - common_log(LOG_ERR, "Requested handler for unkown queue '$queue'"); - } - return null; - } - - /** - * Get a list of all registered queue transport names. - * - * @return array of strings - */ - function getQueues() - { - $group = $this->activeGroup(); - $site = $this->currentSite(); - if (empty($this->groups[$site][$group])) { - return array(); - } else { - return array_keys($this->groups[$site][$group]); - } - } - - /** - * Register a queue transport name and handler class for your plugin. - * Only registered transports will be reliably picked up! * - * @param string $transport - * @param string $class - * @param string $group - */ - public function connect($transport, $class, $group='queuedaemon') - { - $this->handlers[$this->currentSite()][$transport] = $class; - $this->groups[$this->currentSite()][$group][$transport] = $class; - } - - /** - * Saves a notice object reference into the queue item table. * @return boolean true on success * @throws StompException on connection or send error */ @@ -192,8 +153,11 @@ class StompQueueManager extends QueueManager */ protected function _doEnqueue($object, $queue, $idx) { - $msg = $this->encode($object); $rep = $this->logrep($object); + $envelope = array('site' => common_config('site', 'nickname'), + 'handler' => $queue, + 'payload' => $this->encode($object)); + $msg = serialize($envelope); $props = array('created' => common_sql_now()); if ($this->isPersistent($queue)) { @@ -205,11 +169,11 @@ class StompQueueManager extends QueueManager $result = $con->send($this->queueName($queue), $msg, $props); if (!$result) { - common_log(LOG_ERR, "Error sending $rep to $queue queue on $host"); + $this->_log(LOG_ERR, "Error sending $rep to $queue queue on $host"); return false; } - common_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host"); + $this->_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host"); $this->stats('enqueued', $queue); return true; } @@ -275,12 +239,14 @@ class StompQueueManager extends QueueManager $idx = $this->connectionFromSocket($socket); $con = $this->cons[$idx]; $host = $con->getServer(); + $this->defaultIdx = $idx; $ok = true; try { $frames = $con->readFrames(); } catch (StompException $e) { - common_log(LOG_ERR, "Lost connection to $host: " . $e->getMessage()); + $this->_log(LOG_ERR, "Lost connection to $host: " . $e->getMessage()); + fclose($socket); // ??? $this->cons[$idx] = null; $this->transaction[$idx] = null; $this->disconnect[$idx] = time(); @@ -289,14 +255,17 @@ class StompQueueManager extends QueueManager foreach ($frames as $frame) { $dest = $frame->headers['destination']; if ($dest == $this->control) { - if (!$this->handleControlSignal($idx, $frame)) { + if (!$this->handleControlSignal($frame)) { // We got a control event that requests a shutdown; // close out and stop handling anything else! break; } } else { - $ok = $ok && $this->handleItem($idx, $frame); + $ok = $this->handleItem($frame) && $ok; } + $this->ack($idx, $frame); + $this->commit($idx); + $this->begin($idx); } return $ok; } @@ -333,22 +302,9 @@ class StompQueueManager extends QueueManager parent::start($master); $this->_connectAll(); - common_log(LOG_INFO, "Subscribing to $this->control"); - foreach ($this->cons as $con) { - if ($con) { - $con->subscribe($this->control); - } - } - if ($this->sites) { - foreach ($this->sites as $server) { - StatusNet::init($server); - $this->doSubscribe(); - } - } else { - $this->doSubscribe(); - } foreach ($this->cons as $i => $con) { if ($con) { + $this->doSubscribe($con); $this->begin($i); } } @@ -356,9 +312,7 @@ class StompQueueManager extends QueueManager } /** - * Subscribe to all the queues we're going to need to handle... - * - * Side effects: in multi-site mode, may reset site configuration. + * Close out any active connections. * * @return bool return false on failure */ @@ -376,15 +330,6 @@ class StompQueueManager extends QueueManager return true; } - /** - * Get identifier of the currently active site configuration - * @return string - */ - protected function currentSite() - { - return common_config('site', 'server'); // @fixme switch to nickname - } - /** * Lazy open a single connection to Stomp queue server. * If multiple servers are configured, we let the Stomp client library @@ -441,6 +386,10 @@ class StompQueueManager extends QueueManager } } + /** + * Attempt to manually reconnect to the Stomp server for the given + * slot. If successful, set up our subscriptions on it. + */ protected function _reconnect($idx) { try { @@ -453,17 +402,7 @@ class StompQueueManager extends QueueManager $this->cons[$idx] = $con; $this->disconnect[$idx] = null; - // now we have to listen to everything... - // @fixme refactor this nicer. :P - $host = $con->getServer(); - $this->_log(LOG_INFO, "Resubscribing to $this->control on $host"); - $con->subscribe($this->control); - foreach ($this->subscriptions as $site => $queues) { - foreach ($queues as $queue) { - $this->_log(LOG_INFO, "Resubscribing to $queue on $host"); - $con->subscribe($queue); - } - } + $this->doSubscribe($con); $this->begin($idx); } else { // Try again later... @@ -487,41 +426,15 @@ class StompQueueManager extends QueueManager } /** - * Subscribe to all enabled notice queues for the current site. - */ - protected function doSubscribe() - { - $site = $this->currentSite(); - $this->_connect(); - foreach ($this->getQueues() as $queue) { - $rawqueue = $this->queueName($queue); - $this->subscriptions[$site][$queue] = $rawqueue; - $this->_log(LOG_INFO, "Subscribing to $rawqueue"); - foreach ($this->cons as $con) { - if ($con) { - $con->subscribe($rawqueue); - } - } - } - } - - /** - * Subscribe from all enabled notice queues for the current site. + * Set up all our raw queue subscriptions on the given connection + * @param LiberalStomp $con */ - protected function doUnsubscribe() + protected function doSubscribe(LiberalStomp $con) { - $site = $this->currentSite(); - $this->_connect(); - if (!empty($this->subscriptions[$site])) { - foreach ($this->subscriptions[$site] as $queue => $rawqueue) { - $this->_log(LOG_INFO, "Unsubscribing from $rawqueue"); - foreach ($this->cons as $con) { - if ($con) { - $con->unsubscribe($rawqueue); - } - } - unset($this->subscriptions[$site][$queue]); - } + $host = $con->getServer(); + foreach ($this->subscriptions as $queue) { + $this->_log(LOG_INFO, "Subscribing to $queue on $host"); + $con->subscribe($queue); } } @@ -534,25 +447,29 @@ class StompQueueManager extends QueueManager * Side effects: in multi-site mode, may reset site configuration to * match the site that queued the event. * - * @param int $idx connection index * @param StompFrame $frame - * @return bool + * @return bool success */ - protected function handleItem($idx, $frame) + protected function handleItem($frame) { - $this->defaultIdx = $idx; + $host = $this->cons[$this->defaultIdx]->getServer(); + $message = unserialize($frame->body); + $site = $message['site']; + $queue = $message['handler']; - list($site, $queue) = $this->parseDestination($frame->headers['destination']); - if ($site != $this->currentSite()) { - $this->stats('switch'); - StatusNet::init($site); + if ($this->isDeadletter($frame, $message)) { + $this->stats('deadletter', $queue); + return false; } - $host = $this->cons[$idx]->getServer(); - $item = $this->decode($frame->body); + // @fixme detect failing site switches + $this->switchSite($site); + + $item = $this->decode($message['payload']); if (empty($item)) { $this->_log(LOG_ERR, "Skipping empty or deleted item in queue $queue from $host"); - return true; + $this->stats('baditem', $queue); + return false; } $info = $this->logrep($item) . " posted at " . $frame->headers['created'] . " in queue $queue from $host"; @@ -561,16 +478,10 @@ class StompQueueManager extends QueueManager $handler = $this->getHandler($queue); if (!$handler) { $this->_log(LOG_ERR, "Missing handler class; skipping $info"); - $this->ack($idx, $frame); - $this->commit($idx); - $this->begin($idx); $this->stats('badhandler', $queue); return false; } - // If there's an exception when handling, - // log the error and let it get requeued. - try { $ok = $handler->handle($item); } catch (Exception $e) { @@ -578,25 +489,80 @@ class StompQueueManager extends QueueManager $ok = false; } - if (!$ok) { + if ($ok) { + $this->_log(LOG_INFO, "Successfully handled $info"); + $this->stats('handled', $queue); + } else { $this->_log(LOG_WARNING, "Failed handling $info"); - // FIXME we probably shouldn't have to do - // this kind of queue management ourselves; - // if we don't ack, it should resend... - $this->ack($idx, $frame); + // Requeing moves the item to the end of the line for its next try. + // @fixme add a manual retry count $this->enqueue($item, $queue); - $this->commit($idx); - $this->begin($idx); $this->stats('requeued', $queue); - return false; } - $this->_log(LOG_INFO, "Successfully handled $info"); - $this->ack($idx, $frame); - $this->commit($idx); - $this->begin($idx); - $this->stats('handled', $queue); - return true; + return $ok; + } + + /** + * Check if a redelivered message has been run through enough + * that we're going to give up on it. + * + * @param StompFrame $frame + * @param array $message unserialized message body + * @return boolean true if we should discard + */ + protected function isDeadLetter($frame, $message) + { + if (isset($frame->headers['redelivered']) && $frame->headers['redelivered'] == 'true') { + // Message was redelivered, possibly indicating a previous failure. + $msgId = $frame->headers['message-id']; + $site = $message['site']; + $queue = $message['handler']; + $msgInfo = "message $msgId for $site in queue $queue"; + + $deliveries = $this->incDeliveryCount($msgId); + if ($deliveries > common_config('queue', 'max_retries')) { + $info = "DEAD-LETTER FILE: Gave up after retry $deliveries on $msgInfo"; + + $outdir = common_config('queue', 'dead_letter_dir'); + if ($outdir) { + $filename = $outdir . "/$site-$queue-" . rawurlencode($msgId); + $info .= ": dumping to $filename"; + file_put_contents($filename, $message['payload']); + } + + common_log(LOG_ERR, $info); + return true; + } else { + common_log(LOG_INFO, "retry $deliveries on $msgInfo"); + } + } + return false; + } + + /** + * Update count of times we've re-encountered this message recently, + * triggered when we get a message marked as 'redelivered'. + * + * Requires a CLI-friendly cache configuration. + * + * @param string $msgId message-id header from message + * @return int number of retries recorded + */ + function incDeliveryCount($msgId) + { + $count = 0; + $cache = common_memcache(); + if ($cache) { + $key = 'statusnet:stomp:message-retries:' . $msgId; + $count = $cache->increment($key); + if (!$count) { + $count = 1; + $cache->set($key, $count, null, 3600); + $got = $cache->get($key); + } + } + return $count; } /** @@ -629,13 +595,22 @@ class StompQueueManager extends QueueManager } else { $this->_log(LOG_ERR, "Ignoring unrecognized control message: $message"); } - - $this->ack($idx, $frame); - $this->commit($idx); - $this->begin($idx); return $shutdown; } + /** + * Switch site, if necessary, and reset current handler assignments + * @param string $site + */ + function switchSite($site) + { + if ($site != StatusNet::currentSite()) { + $this->stats('switch'); + StatusNet::switchSite($site); + $this->initialize(); + } + } + /** * Set us up with queue subscriptions for a new site added at runtime, * triggered by a broadcast to the 'statusnet-control' topic. @@ -648,22 +623,17 @@ class StompQueueManager extends QueueManager if (empty($this->sites)) { if ($nickname == common_config('site', 'nickname')) { StatusNet::init(common_config('site', 'server')); - $this->doUnsubscribe(); - $this->doSubscribe(); } else { $this->_log(LOG_INFO, "Ignoring update ping for other site $nickname"); } } else { $sn = Status_network::staticGet($nickname); if ($sn) { - $server = $sn->getServerName(); // @fixme do config-by-nick - StatusNet::init($server); - if (empty($this->sites[$server])) { - $this->addSite($server); + $this->switchSite($nickname); + if (!in_array($nickname, $this->sites)) { + $this->addSite(); } - $this->_log(LOG_INFO, "(Re)subscribing to queues for site $nickname / $server"); - $this->doUnsubscribe(); - $this->doSubscribe(); + // @fixme update subscriptions, if applicable $this->stats('siteupdate'); } else { $this->_log(LOG_ERR, "Ignoring ping for unrecognized new site $nickname"); @@ -673,42 +643,47 @@ class StompQueueManager extends QueueManager /** * Combines the queue_basename from configuration with the - * site server name and queue name to give eg: + * group name for this queue to give eg: * - * /queue/statusnet/identi.ca/sms + * /queue/statusnet/main * * @param string $queue * @return string */ protected function queueName($queue) { - return common_config('queue', 'queue_basename') . - $this->currentSite() . '/' . $queue; + $base = common_config('queue', 'queue_basename'); + $group = $this->queueGroup($queue); + $breakout = $this->breakoutMode($queue); + if ($breakout == 'shared') { + return $base . "$group"; + } else if ($breakout == 'handler') { + return $base . "$group/$queue"; + } else if ($breakout == 'site') { + $site = StatusNet::currentSite(); + return $base . "$group/$queue/$site"; + } + throw Exception("Unrecognized queue breakout mode '$breakout' for '$queue'"); } /** - * Returns the site and queue name from the server-side queue. + * Get the breakout mode for the given queue on the current site. * - * @param string queue destination (eg '/queue/statusnet/identi.ca/sms') - * @return array of site and queue: ('identi.ca','sms') or false if unrecognized + * @param string $queue + * @return string one of 'shared', 'handler', 'site' */ - protected function parseDestination($dest) + protected function breakoutMode($queue) { - $prefix = common_config('queue', 'queue_basename'); - if (substr($dest, 0, strlen($prefix)) == $prefix) { - $rest = substr($dest, strlen($prefix)); - return explode("/", $rest, 2); + $breakout = common_config('queue', 'breakout'); + if (isset($breakout[$queue])) { + return $breakout[$queue]; + } else if (isset($breakout['*'])) { + return $breakout['*']; } else { - common_log(LOG_ERR, "Got a message from unrecognized stomp queue: $dest"); - return array(false, false); + return 'shared'; } } - function _log($level, $msg) - { - common_log($level, 'StompQueueManager: '.$msg); - } - protected function begin($idx) { if ($this->useTransactions) { diff --git a/lib/xmppmanager.php b/lib/xmppmanager.php index 985e7c32e..f37635855 100644 --- a/lib/xmppmanager.php +++ b/lib/xmppmanager.php @@ -48,7 +48,7 @@ class XmppManager extends IoManager public static function get() { if (common_config('xmpp', 'enabled')) { - $site = common_config('site', 'server'); + $site = StatusNet::currentSite(); if (empty(self::$singletons[$site])) { self::$singletons[$site] = new XmppManager(); } @@ -69,7 +69,7 @@ class XmppManager extends IoManager function __construct() { - $this->site = common_config('site', 'server'); + $this->site = StatusNet::currentSite(); $this->resource = common_config('xmpp', 'resource') . 'daemon'; } @@ -476,10 +476,10 @@ class XmppManager extends IoManager */ protected function switchSite() { - if ($this->site != common_config('site', 'server')) { + if ($this->site != StatusNet::currentSite()) { common_log(LOG_DEBUG, __METHOD__ . ": switching to site $this->site"); $this->stats('switch'); - StatusNet::init($this->site); + StatusNet::switchSite($this->site); } } } diff --git a/plugins/MemcachePlugin.php b/plugins/MemcachePlugin.php index 2bc4b892b..93a0583fe 100644 --- a/plugins/MemcachePlugin.php +++ b/plugins/MemcachePlugin.php @@ -51,6 +51,8 @@ if (!defined('STATUSNET')) { class MemcachePlugin extends Plugin { + static $cacheInitialized = false; + private $_conn = null; public $servers = array('127.0.0.1;11211'); @@ -71,10 +73,21 @@ class MemcachePlugin extends Plugin function onInitializePlugin() { - if (is_null($this->persistent)) { + if (self::$cacheInitialized) { + $this->persistent = true; + } else { + // If we're a parent command-line process we need + // to be able to close out the connection after + // forking, so disable persistence. + // + // We'll turn it back on again the second time + // through which will either be in a child process, + // or a single-process script which is switching + // configurations. $this->persistent = (php_sapi_name() == 'cli') ? false : true; } $this->_ensureConn(); + self::$cacheInitialized = true; return true; } @@ -121,6 +134,24 @@ class MemcachePlugin extends Plugin return false; } + /** + * Atomically increment an existing numeric key value. + * Existing expiration time will not be changed. + * + * @param string &$key in; Key to use for lookups + * @param int &$step in; Amount to increment (default 1) + * @param mixed &$value out; Incremented value, or false if key not set. + * + * @return boolean hook success + */ + function onStartCacheIncrement(&$key, &$step, &$value) + { + $this->_ensureConn(); + $value = $this->_conn->increment($key, $step); + Event::handle('EndCacheIncrement', array($key, $step, $value)); + return false; + } + /** * Delete a value associated with a key * diff --git a/scripts/queuedaemon.php b/scripts/queuedaemon.php index 30a8a9602..d372d898f 100755 --- a/scripts/queuedaemon.php +++ b/scripts/queuedaemon.php @@ -74,8 +74,6 @@ require_once(INSTALLDIR.'/lib/daemon.php'); require_once(INSTALLDIR.'/classes/Queue_item.php'); require_once(INSTALLDIR.'/classes/Notice.php'); -define('CLAIM_TIMEOUT', 1200); - /** * Queue handling daemon... * @@ -92,7 +90,7 @@ class QueueDaemon extends SpawningDaemon function __construct($id=null, $daemonize=true, $threads=1, $allsites=false) { parent::__construct($id, $daemonize, $threads); - $this->all = $allsites; + $this->allsites = $allsites; } /** @@ -108,7 +106,7 @@ class QueueDaemon extends SpawningDaemon $this->log(LOG_INFO, 'checking for queued notices'); $master = new QueueMaster($this->get_id()); - $master->init($this->all); + $master->init($this->allsites); try { $master->service(); } catch (Exception $e) { @@ -133,14 +131,16 @@ class QueueMaster extends IoMaster */ function initManagers() { - $classes = array(); - if (Event::handle('StartQueueDaemonIoManagers', array(&$classes))) { - $classes[] = 'QueueManager'; + $managers = array(); + if (Event::handle('StartQueueDaemonIoManagers', array(&$managers))) { + $qm = QueueManager::get(); + $qm->setActiveGroup('main'); + $managers[] = $qm; } - Event::handle('EndQueueDaemonIoManagers', array(&$classes)); + Event::handle('EndQueueDaemonIoManagers', array(&$managers)); - foreach ($classes as $class) { - $this->instantiate($class); + foreach ($managers as $manager) { + $this->instantiate($manager); } } } diff --git a/scripts/xmppdaemon.php b/scripts/xmppdaemon.php index 46dd9b90c..9302f0c43 100755 --- a/scripts/xmppdaemon.php +++ b/scripts/xmppdaemon.php @@ -20,13 +20,15 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); -$shortoptions = 'fi::'; -$longoptions = array('id::', 'foreground'); +$shortoptions = 'fi::a'; +$longoptions = array('id::', 'foreground', 'all'); $helptext = <<allsites = $allsites; } function runThread() @@ -51,7 +56,7 @@ class XMPPDaemon extends SpawningDaemon common_log(LOG_INFO, 'Waiting to listen to XMPP and queues'); $master = new XmppMaster($this->get_id()); - $master->init(); + $master->init($this->allsites); $master->service(); common_log(LOG_INFO, 'terminating normally'); @@ -69,15 +74,19 @@ class XmppMaster extends IoMaster */ function initManagers() { - // @fixme right now there's a hack in QueueManager to determine - // which queues to subscribe to based on the master class. - $this->instantiate('QueueManager'); - $this->instantiate('XmppManager'); + if (common_config('xmpp', 'enabled')) { + $qm = QueueManager::get(); + $qm->setActiveGroup('xmpp'); + $this->instantiate($qm); + $this->instantiate(XmppManager::get()); + } } } // Abort immediately if xmpp is not enabled, otherwise the daemon chews up // lots of CPU trying to connect to unconfigured servers +// @fixme do this check after we've run through the site list so we +// don't have to find an XMPP site to start up when using --all mode. if (common_config('xmpp','enabled')==false) { print "Aborting daemon - xmpp is disabled\n"; exit(); @@ -92,7 +101,8 @@ if (have_option('i', 'id')) { } $foreground = have_option('f', 'foreground'); +$all = have_option('a') || have_option('--all'); -$daemon = new XMPPDaemon($id, !$foreground); +$daemon = new XMPPDaemon($id, !$foreground, 1, $all); $daemon->runOnce(); -- cgit v1.2.3-54-g00ecf