Merge branch '0.9.x' of gitorious.org:statusnet/mainline into 0.9.x

4 files changed, 610 insertions, 301 deletions

diff --git a/plugins/OStatus/classes/FeedSub.php b/plugins/OStatus/classes/FeedSub.php
index b848b6b1d..b10509dae 100644
--- a/plugins/OStatus/classes/FeedSub.php
+++ b/plugins/OStatus/classes/FeedSub.php
@@ -61,7 +61,7 @@ class FeedSub extends Memcached_DataObject
     public $__table = 'feedsub';
 
     public $id;
-    public $feeduri;
+    public $uri;
 
     // PuSH subscription data
     public $huburi;
@@ -110,7 +110,7 @@ class FeedSub extends Memcached_DataObject
                                    /*size*/ null,
                                    /*nullable*/ false,
                                    /*key*/ 'PRI',
-                                   /*default*/ '0',
+                                   /*default*/ null,
                                    /*extra*/ null,
                                    /*auto_increment*/ true),
                      new ColumnDef('uri', 'varchar',
@@ -238,7 +238,7 @@ class FeedSub extends Memcached_DataObject
     public function subscribe($mode='subscribe')
     {
         if ($this->sub_state && $this->sub_state != 'inactive') {
-            throw new ServerException("Attempting to start PuSH subscription to feed in state $this->sub_state");
+            common_log(LOG_WARNING, "Attempting to (re)start PuSH subscription to $this->uri in unexpected state $this->sub_state");
         }
         if (empty($this->huburi)) {
             if (common_config('feedsub', 'nohub')) {
@@ -261,7 +261,7 @@ class FeedSub extends Memcached_DataObject
      */
     public function unsubscribe() {
         if ($this->sub_state != 'active') {
-            throw new ServerException("Attempting to end PuSH subscription to feed in state $this->sub_state");
+            common_log(LOG_WARNING, "Attempting to (re)end PuSH subscription to $this->uri in unexpected state $this->sub_state");
         }
         if (empty($this->huburi)) {
             if (common_config('feedsub', 'nohub')) {
@@ -450,3 +450,4 @@ class FeedSub extends Memcached_DataObject
     }
 
 }
+
diff --git a/plugins/OStatus/classes/HubSub.php b/plugins/OStatus/classes/HubSub.php
index 3120a70f9..7db528a4e 100644
--- a/plugins/OStatus/classes/HubSub.php
+++ b/plugins/OStatus/classes/HubSub.php
@@ -77,7 +77,7 @@ class HubSub extends Memcached_DataObject
                      new ColumnDef('topic', 'varchar',
                                    /*size*/255,
                                    /*nullable*/false,
-                                   /*key*/'KEY'),
+                                   /*key*/'MUL'),
                      new ColumnDef('callback', 'varchar',
                                    255, false),
                      new ColumnDef('secret', 'text',
@@ -192,7 +192,7 @@ class HubSub extends Memcached_DataObject
 
         // Any existing query string parameters must be preserved
         $url = $this->callback;
-        if (strpos('?', $url) !== false) {
+        if (strpos($url, '?') !== false) {
             $url .= '&';
         } else {
             $url .= '?';
@@ -260,6 +260,37 @@ class HubSub extends Memcached_DataObject
             $retries = intval(common_config('ostatus', 'hub_retries'));
         }
 
+        if (common_config('ostatus', 'local_push_bypass')) {
+            // If target is a local site, bypass the web server and drop the
+            // item directly into the target's input queue.
+            $url = parse_url($this->callback);
+            $wildcard = common_config('ostatus', 'local_wildcard');
+            $site = Status_network::getFromHostname($url['host'], $wildcard);
+
+            if ($site) {
+                if ($this->secret) {
+                    $hmac = 'sha1=' . hash_hmac('sha1', $atom, $this->secret);
+                } else {
+                    $hmac = '';
+                }
+
+                // Hack: at the moment we stick the subscription ID in the callback
+                // URL so we don't have to look inside the Atom to route the subscription.
+                // For now this means we need to extract that from the target URL
+                // so we can include it in the data.
+                $parts = explode('/', $url['path']);
+                $subId = intval(array_pop($parts));
+
+                $data = array('feedsub_id' => $subId,
+                              'post' => $atom,
+                              'hmac' => $hmac);
+                common_log(LOG_DEBUG, "Cross-site PuSH bypass enqueueing straight to $site->nickname feed $subId");
+                $qm = QueueManager::get();
+                $qm->enqueue($data, 'pushin', $site->nickname);
+                return;
+            }
+        }
+
         // We dare not clone() as when the clone is discarded it'll
         // destroy the result data for the parent query.
         // @fixme use clone() again when it's safe to copy an
@@ -274,6 +305,26 @@ class HubSub extends Memcached_DataObject
     }
 
     /**
+     * Queue up a large batch of pushes to multiple subscribers
+     * for this same topic update.
+     * 
+     * If queues are disabled, this will run immediately.
+     * 
+     * @param string $atom well-formed Atom feed
+     * @param array $pushCallbacks list of callback URLs
+     */
+    function bulkDistribute($atom, $pushCallbacks)
+    {
+        $data = array('atom' => $atom,
+                      'topic' => $this->topic,
+                      'pushCallbacks' => $pushCallbacks);
+        common_log(LOG_INFO, "Queuing PuSH batch: $this->topic to " .
+                             count($pushCallbacks) . " sites");
+        $qm = QueueManager::get();
+        $qm->enqueue($data, 'hubprep');
+    }
+
+    /**
      * Send a 'fat ping' to the subscriber's callback endpoint
      * containing the given Atom feed chunk.
      *
diff --git a/plugins/OStatus/classes/Magicsig.php b/plugins/OStatus/classes/Magicsig.php
index 5a46aeeb6..f8c56a05f 100644
--- a/plugins/OStatus/classes/Magicsig.php
+++ b/plugins/OStatus/classes/Magicsig.php
@@ -40,8 +40,9 @@ class Magicsig extends Memcached_DataObject
     public $keypair;
     public $alg;
     
-    private $_rsa;
-
+    public $publicKey;
+    public $privateKey;
+    
     public function __construct($alg = 'RSA-SHA256')
     {
         $this->alg = $alg;
@@ -51,7 +52,15 @@ class Magicsig extends Memcached_DataObject
     {
         $obj =  parent::staticGet(__CLASS__, $k, $v);
         if (!empty($obj)) {
-            return Magicsig::fromString($obj->keypair);
+            $obj = Magicsig::fromString($obj->keypair);
+
+            // Double check keys: Crypt_RSA did not
+            // consistently generate good keypairs.
+            // We've also moved to 1024 bit keys.
+            if (strlen($obj->publicKey->modulus->toBits()) != 1024) {
+                $obj->delete();
+                return false;
+            }
         }
 
         return $obj;
@@ -70,9 +79,9 @@ class Magicsig extends Memcached_DataObject
     static function schemaDef()
     {
         return array(new ColumnDef('user_id', 'integer',
-                                   null, true, 'PRI'),
-                     new ColumnDef('keypair', 'varchar',
-                                   255, false),
+                                   null, false, 'PRI'),
+                     new ColumnDef('keypair', 'text',
+                                   false, false),
                      new ColumnDef('alg', 'varchar',
                                    64, false));
     }
@@ -99,17 +108,20 @@ class Magicsig extends Memcached_DataObject
         return parent::insert();
     }
 
-    public function generate($user_id, $key_length = 512)
+    public function generate($user_id)
     {
-        PEAR::pushErrorHandling(PEAR_ERROR_RETURN);
+        $rsa = new Crypt_RSA();
+        
+        $keypair = $rsa->createKey();
 
-        $keypair = new Crypt_RSA_KeyPair($key_length);
-        $params['public_key'] = $keypair->getPublicKey();
-        $params['private_key'] = $keypair->getPrivateKey();
+        $rsa->loadKey($keypair['privatekey']);
 
-        $this->_rsa = new Crypt_RSA($params);
-        PEAR::popErrorHandling();
+        $this->privateKey = new Crypt_RSA();
+        $this->privateKey->loadKey($keypair['privatekey']);
 
+        $this->publicKey = new Crypt_RSA();
+        $this->publicKey->loadKey($keypair['publickey']);
+        
         $this->user_id = $user_id;
         $this->insert();
     }
@@ -117,14 +129,11 @@ class Magicsig extends Memcached_DataObject
 
     public function toString($full_pair = true)
     {
-        $public_key = $this->_rsa->_public_key;
-        $private_key = $this->_rsa->_private_key;
-
-        $mod = base64_url_encode($public_key->getModulus());
-        $exp = base64_url_encode($public_key->getExponent());
+        $mod = Magicsig::base64_url_encode($this->publicKey->modulus->toBytes());
+        $exp = Magicsig::base64_url_encode($this->publicKey->exponent->toBytes());
         $private_exp = '';
-        if ($full_pair && $private_key->getExponent()) {
-            $private_exp = '.' . base64_url_encode($private_key->getExponent());
+        if ($full_pair && $this->privateKey->exponent->toBytes()) {
+            $private_exp = '.' . Magicsig::base64_url_encode($this->privateKey->exponent->toBytes());
         }
 
         return 'RSA.' . $mod . '.' . $exp . $private_exp; 
@@ -132,8 +141,6 @@ class Magicsig extends Memcached_DataObject
     
     public static function fromString($text)
     {
-        PEAR::pushErrorHandling(PEAR_ERROR_RETURN);
-
         $magic_sig = new Magicsig();
         
         // remove whitespace
@@ -144,35 +151,40 @@ class Magicsig extends Memcached_DataObject
             return false;
         }
         
-        $mod = base64_url_decode($matches[1]);
-        $exp = base64_url_decode($matches[2]);
+        $mod = $matches[1];
+        $exp = $matches[2];
         if (!empty($matches[4])) {
-            $private_exp = base64_url_decode($matches[4]);
+            $private_exp = $matches[4];
         } else {
             $private_exp = false;
         }
 
-        $params['public_key'] = new Crypt_RSA_KEY($mod, $exp, 'public');
-        if ($params['public_key']->isError()) {
-            $error = $params['public_key']->getLastError();
-            common_log(LOG_DEBUG, 'RSA Error: '. $error->getMessage());
-            return false;
-        }
+        $magic_sig->loadKey($mod, $exp, 'public');
         if ($private_exp) {
-            $params['private_key'] = new Crypt_RSA_KEY($mod, $private_exp, 'private');
-            if ($params['private_key']->isError()) {
-                $error = $params['private_key']->getLastError();
-                common_log(LOG_DEBUG, 'RSA Error: '. $error->getMessage());
-                return false;
-            }
+            $magic_sig->loadKey($mod, $private_exp, 'private');
         }
 
-        $magic_sig->_rsa = new Crypt_RSA($params);
-        PEAR::popErrorHandling();
-
         return $magic_sig;
     }
 
+    public function loadKey($mod, $exp, $type = 'public')
+    {
+        common_log(LOG_DEBUG, "Adding ".$type." key: (".$mod .', '. $exp .")");
+
+        $rsa = new Crypt_RSA();
+        $rsa->signatureMode = CRYPT_RSA_SIGNATURE_PKCS1;
+        $rsa->setHash('sha256');
+        $rsa->modulus = new Math_BigInteger(Magicsig::base64_url_decode($mod), 256);
+        $rsa->k = strlen($rsa->modulus->toBytes());
+        $rsa->exponent = new Math_BigInteger(Magicsig::base64_url_decode($exp), 256);
+
+        if ($type == 'private') {
+            $this->privateKey = $rsa;
+        } else {
+            $this->publicKey = $rsa;
+        }
+    }
+    
     public function getName()
     {
         return $this->alg;
@@ -183,51 +195,33 @@ class Magicsig extends Memcached_DataObject
         switch ($this->alg) {
 
         case 'RSA-SHA256':
-            return 'magicsig_sha256';
+            return 'sha256';
         }
 
     }
     
     public function sign($bytes)
     {
-        $hash = $this->getHash();
-        $sig = $this->_rsa->createSign($bytes, null, $hash);
-        if ($this->_rsa->isError()) {
-            $error = $this->_rsa->getLastError();
-            common_log(LOG_DEBUG, 'RSA Error: '. $error->getMessage());
-            return false;
-        }
-
-        return $sig;
+        $sig = $this->privateKey->sign($bytes);
+        return Magicsig::base64_url_encode($sig);
     }
 
     public function verify($signed_bytes, $signature)
     {
-        $hash = $this->getHash();
-        $result =  $this->_rsa->validateSign($signed_bytes, $signature, null, $hash);
-        if ($this->_rsa->isError()) {
-            $error = $this->keypair->getLastError();
-            common_log(LOG_DEBUG, 'RSA Error: '. $error->getMessage());
-            return false;
-        }
-        return $result;
+        $signature = Magicsig::base64_url_decode($signature);
+        return $this->publicKey->verify($signed_bytes, $signature);
     }
-        
-}
 
-// Define a sha256 function for hashing
-// (Crypt_RSA should really be updated to use hash() )
-function magicsig_sha256($bytes)
-{
-    return hash('sha256', $bytes);
-}
 
-function base64_url_encode($input)
-{
-    return strtr(base64_encode($input), '+/', '-_');
+    public static function base64_url_encode($input)
+    {
+        return strtr(base64_encode($input), '+/', '-_');
+    }
+
+    public static function base64_url_decode($input)
+    {
+        return base64_decode(strtr($input, '-_', '+/'));
+    }
 }
 
-function base64_url_decode($input)
-{
-    return base64_decode(strtr($input, '-_', '+/'));
-}
\ No newline at end of file
+
diff --git a/plugins/OStatus/classes/Ostatus_profile.php b/plugins/OStatus/classes/Ostatus_profile.php
index a33e95d93..5d3f37cd0 100644
--- a/plugins/OStatus/classes/Ostatus_profile.php
+++ b/plugins/OStatus/classes/Ostatus_profile.php
@@ -195,52 +195,6 @@ class Ostatus_profile extends Memcached_DataObject
     }
 
     /**
-     * Subscribe a local user to this remote user.
-     * PuSH subscription will be started if necessary, and we'll
-     * send a Salmon notification to the remote server if available
-     * notifying them of the sub.
-     *
-     * @param User $user
-     * @return boolean success
-     * @throws FeedException
-     */
-    public function subscribeLocalToRemote(User $user)
-    {
-        if ($this->isGroup()) {
-            throw new ServerException("Can't subscribe to a remote group");
-        }
-
-        if ($this->subscribe()) {
-            if ($user->subscribeTo($this->localProfile())) {
-                $this->notify($user->getProfile(), ActivityVerb::FOLLOW, $this);
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Mark this remote profile as subscribing to the given local user,
-     * and send appropriate notifications to the user.
-     *
-     * This will generally be in response to a subscription notification
-     * from a foreign site to our local Salmon response channel.
-     *
-     * @param User $user
-     * @return boolean success
-     */
-    public function subscribeRemoteToLocal(User $user)
-    {
-        if ($this->isGroup()) {
-            throw new ServerException("Remote groups can't subscribe to local users");
-        }
-
-        Subscription::start($this->localProfile(), $user->getProfile());
-
-        return true;
-    }
-
-    /**
      * Send a subscription request to the hub for this feed.
      * The hub will later send us a confirmation POST to /main/push/callback.
      *
@@ -250,12 +204,13 @@ class Ostatus_profile extends Memcached_DataObject
     public function subscribe()
     {
         $feedsub = FeedSub::ensureFeed($this->feeduri);
-        if ($feedsub->sub_state == 'active' || $feedsub->sub_state == 'subscribe') {
+        if ($feedsub->sub_state == 'active') {
+            // Active subscription, we don't need to do anything.
             return true;
-        } else if ($feedsub->sub_state == '' || $feedsub->sub_state == 'inactive') {
+        } else {
+            // Inactive or we got left in an inconsistent state.
+            // Run a subscription request to make sure we're current!
             return $feedsub->subscribe();
-        } else if ('unsubscribe') {
-            throw new FeedSubException("Unsub is pending, can't subscribe...");
         }
     }
 
@@ -268,15 +223,13 @@ class Ostatus_profile extends Memcached_DataObject
      */
     public function unsubscribe() {
         $feedsub = FeedSub::staticGet('uri', $this->feeduri);
-        if (!$feedsub) {
+        if (!$feedsub || $feedsub->sub_state == '' || $feedsub->sub_state == 'inactive') {
+            // No active PuSH subscription, we can just leave it be.
             return true;
-        }
-        if ($feedsub->sub_state == 'active') {
+        } else {
+            // PuSH subscription is either active or in an indeterminate state.
+            // Send an unsubscribe.
             return $feedsub->unsubscribe();
-        } else if ($feedsub->sub_state == '' || $feedsub->sub_state == 'inactive' || $feedsub->sub_state == 'unsubscribe') {
-            return true;
-        } else if ($feedsub->sub_state == 'subscribe') {
-            throw new FeedSubException("Feed is awaiting subscription, can't unsub...");
         }
     }
 
@@ -428,9 +381,23 @@ class Ostatus_profile extends Memcached_DataObject
      * Currently assumes that all items in the feed are new,
      * coming from a PuSH hub.
      *
-     * @param DOMDocument $feed
+     * @param DOMDocument $doc
+     * @param string $source identifier ("push")
      */
-    public function processFeed($feed, $source)
+    public function processFeed(DOMDocument $doc, $source)
+    {
+        $feed = $doc->documentElement;
+
+        if ($feed->localName == 'feed' && $feed->namespaceURI == Activity::ATOM) {
+            $this->processAtomFeed($feed, $source);
+        } else if ($feed->localName == 'rss') { // @fixme check namespace
+            $this->processRssFeed($feed, $source);
+        } else {
+            throw new Exception("Unknown feed format.");
+        }
+    }
+
+    public function processAtomFeed(DOMElement $feed, $source)
     {
         $entries = $feed->getElementsByTagNameNS(Activity::ATOM, 'entry');
         if ($entries->length == 0) {
@@ -444,16 +411,49 @@ class Ostatus_profile extends Memcached_DataObject
         }
     }
 
+    public function processRssFeed(DOMElement $rss, $source)
+    {
+        $channels = $rss->getElementsByTagName('channel');
+
+        if ($channels->length == 0) {
+            throw new Exception("RSS feed without a channel.");
+        } else if ($channels->length > 1) {
+            common_log(LOG_WARNING, __METHOD__ . ": more than one channel in an RSS feed");
+        }
+
+        $channel = $channels->item(0);
+
+        $items = $channel->getElementsByTagName('item');
+
+        for ($i = 0; $i < $items->length; $i++) {
+            $item = $items->item($i);
+            $this->processEntry($item, $channel, $source);
+        }
+    }
+
     /**
      * Process a posted entry from this feed source.
      *
      * @param DOMElement $entry
      * @param DOMElement $feed for context
+     * @param string $source identifier ("push" or "salmon")
      */
     public function processEntry($entry, $feed, $source)
     {
         $activity = new Activity($entry, $feed);
 
+        // @todo process all activity objects
+        switch ($activity->objects[0]->type) {
+        case ActivityObject::ARTICLE:
+        case ActivityObject::BLOGENTRY:
+        case ActivityObject::NOTE:
+        case ActivityObject::STATUS:
+        case ActivityObject::COMMENT:
+            break;
+        default:
+            throw new ClientException("Can't handle that kind of post.");
+        }
+
         if ($activity->verb == ActivityVerb::POST) {
             $this->processPost($activity, $source);
         } else {
@@ -480,24 +480,27 @@ class Ostatus_profile extends Memcached_DataObject
                 return false;
             }
         } else {
-            // Individual user feeds may contain only posts from themselves.
-            // Authorship is validated against the profile URI on upper layers,
-            // through PuSH setup or Salmon signature checks.
-            $actorUri = self::getActorProfileURI($activity);
-            if ($actorUri == $this->uri) {
-                // Check if profile info has changed and update it
-                $this->updateFromActivityObject($activity->actor);
+            $actor = $activity->actor;
+
+            if (empty($actor)) {
+                // OK here! assume the default
+            } else if ($actor->id == $this->uri || $actor->link == $this->uri) {
+                $this->updateFromActivityObject($actor);
             } else {
-                common_log(LOG_WARNING, "OStatus: skipping post with bad author: got $actorUri expected $this->uri");
-                return false;
+                throw new Exception("Got an actor '{$actor->title}' ({$actor->id}) on single-user feed for {$this->uri}");
             }
+
             $oprofile = $this;
         }
 
+        // It's not always an ActivityObject::NOTE, but... let's just say it is.
+
+        $note = $activity->objects[0];
+
         // The id URI will be used as a unique identifier for for the notice,
         // protecting against duplicate saves. It isn't required to be a URL;
         // tag: URIs for instance are found in Google Buzz feeds.
-        $sourceUri = $activity->object->id;
+        $sourceUri = $note->id;
         $dupe = Notice::staticGet('uri', $sourceUri);
         if ($dupe) {
             common_log(LOG_INFO, "OStatus: ignoring duplicate post: $sourceUri");
@@ -506,16 +509,30 @@ class Ostatus_profile extends Memcached_DataObject
 
         // We'll also want to save a web link to the original notice, if provided.
         $sourceUrl = null;
-        if ($activity->object->link) {
-            $sourceUrl = $activity->object->link;
+        if ($note->link) {
+            $sourceUrl = $note->link;
         } else if ($activity->link) {
             $sourceUrl = $activity->link;
-        } else if (preg_match('!^https?://!', $activity->object->id)) {
-            $sourceUrl = $activity->object->id;
+        } else if (preg_match('!^https?://!', $note->id)) {
+            $sourceUrl = $note->id;
+        }
+
+        // Use summary as fallback for content
+
+        if (!empty($note->content)) {
+            $sourceContent = $note->content;
+        } else if (!empty($note->summary)) {
+            $sourceContent = $note->summary;
+        } else if (!empty($note->title)) {
+            $sourceContent = $note->title;
+        } else {
+            // @fixme fetch from $sourceUrl?
+            throw new ClientException("No content for notice {$sourceUri}");
         }
 
         // Get (safe!) HTML and text versions of the content
-        $rendered = $this->purify($activity->object->content);
+
+        $rendered = $this->purify($sourceContent);
         $content = html_entity_decode(strip_tags($rendered));
 
         $shortened = common_shorten_links($content);
@@ -526,21 +543,29 @@ class Ostatus_profile extends Memcached_DataObject
         $attachment = null;
 
         if (Notice::contentTooLong($shortened)) {
-            $attachment = $this->saveHTMLFile($activity->object->title, $rendered);
-            $summary = $activity->object->summary;
+            $attachment = $this->saveHTMLFile($note->title, $rendered);
+            $summary = html_entity_decode(strip_tags($note->summary));
             if (empty($summary)) {
                 $summary = $content;
             }
             $shortSummary = common_shorten_links($summary);
             if (Notice::contentTooLong($shortSummary)) {
-                $url = common_shorten_url(common_local_url('attachment',
-                                                           array('attachment' => $attachment->id)));
+                $url = common_shorten_url($sourceUrl);
                 $shortSummary = substr($shortSummary,
                                        0,
                                        Notice::maxContent() - (mb_strlen($url) + 2));
-                $shortSummary .= '… ' . $url;
-                $content = $shortSummary;
-                $rendered = common_render_text($content);
+                $content = $shortSummary . ' ' . $url;
+
+                // We mark up the attachment link specially for the HTML output
+                // so we can fold-out the full version inline.
+                $attachUrl = common_local_url('attachment',
+                                              array('attachment' => $attachment->id));
+                $rendered = common_render_text($shortSummary) .
+                            '<a href="' . htmlspecialchars($attachUrl) .'"'.
+                            ' class="attachment more"' .
+                            ' title="'. htmlspecialchars(_m('Show more')) . '">' .
+                            '&#8230;' .
+                            '</a>';
             }
         }
 
@@ -550,7 +575,8 @@ class Ostatus_profile extends Memcached_DataObject
                         'rendered' => $rendered,
                         'replies' => array(),
                         'groups' => array(),
-                        'tags' => array());
+                        'tags' => array(),
+                        'urls' => array());
 
         // Check for optional attributes...
 
@@ -595,6 +621,12 @@ class Ostatus_profile extends Memcached_DataObject
             }
         }
 
+        // Atom enclosures -> attachment URLs
+        foreach ($activity->enclosures as $href) {
+            // @fixme save these locally or....?
+            $options['urls'][] = $href;
+        }
+
         try {
             $saved = Notice::saveNew($oprofile->profile_id,
                                      $content,
@@ -620,7 +652,8 @@ class Ostatus_profile extends Memcached_DataObject
     protected function purify($html)
     {
         require_once INSTALLDIR.'/extlib/htmLawed/htmLawed.php';
-        $config = array('safe' => 1);
+        $config = array('safe' => 1,
+                        'deny_attribute' => 'id,style,on*');
         return htmLawed($html, $config);
     }
 
@@ -658,13 +691,10 @@ class Ostatus_profile extends Memcached_DataObject
             }
 
             // Is the recipient a local group?
-            // @fixme we need a uri on user_group
+            // @fixme uri on user_group isn't reliable yet
             // $group = User_group::staticGet('uri', $recipient);
-            $template = common_local_url('groupbyid', array('id' => '31337'));
-            $template = preg_quote($template, '/');
-            $template = str_replace('31337', '(\d+)', $template);
-            if (preg_match("/$template/", $recipient, $matches)) {
-                $id = $matches[1];
+            $id = OStatusPlugin::localGroupFromUrl($recipient);
+            if ($id) {
                 $group = User_group::staticGet('id', $id);
                 if ($group) {
                     // Deliver to all members of this local group if allowed.
@@ -690,22 +720,148 @@ class Ostatus_profile extends Memcached_DataObject
     }
 
     /**
+     * Look up and if necessary create an Ostatus_profile for the remote entity
+     * with the given profile page URL. This should never return null -- you
+     * will either get an object or an exception will be thrown.
+     *
      * @param string $profile_url
      * @return Ostatus_profile
-     * @throws FeedSubException
+     * @throws Exception on various error conditions
+     * @throws OStatusShadowException if this reference would obscure a local user/group
      */
-    public static function ensureProfile($profile_uri, $hints=array())
+
+    public static function ensureProfileURL($profile_url, $hints=array())
     {
-        // Get the canonical feed URI and check it
+        $oprofile = self::getFromProfileURL($profile_url);
+
+        if (!empty($oprofile)) {
+            return $oprofile;
+        }
+
+        $hints['profileurl'] = $profile_url;
+
+        // Fetch the URL
+        // XXX: HTTP caching
+
+        $client = new HTTPClient();
+        $client->setHeader('Accept', 'text/html,application/xhtml+xml');
+        $response = $client->get($profile_url);
+
+        if (!$response->isOk()) {
+            throw new Exception("Could not reach profile page: " . $profile_url);
+        }
+
+        // Check if we have a non-canonical URL
+
+        $finalUrl = $response->getUrl();
+
+        if ($finalUrl != $profile_url) {
+
+            $hints['profileurl'] = $finalUrl;
+
+            $oprofile = self::getFromProfileURL($finalUrl);
+
+            if (!empty($oprofile)) {
+                return $oprofile;
+            }
+        }
+
+        // Try to get some hCard data
+
+        $body = $response->getBody();
+
+        $hcardHints = DiscoveryHints::hcardHints($body, $finalUrl);
+
+        if (!empty($hcardHints)) {
+            $hints = array_merge($hints, $hcardHints);
+        }
+
+        // Check if they've got an LRDD header
+
+        $lrdd = LinkHeader::getLink($response, 'lrdd', 'application/xrd+xml');
+
+        if (!empty($lrdd)) {
+
+            $xrd = Discovery::fetchXrd($lrdd);
+            $xrdHints = DiscoveryHints::fromXRD($xrd);
+
+            $hints = array_merge($hints, $xrdHints);
+        }
+
+        // If discovery found a feedurl (probably from LRDD), use it.
+
+        if (array_key_exists('feedurl', $hints)) {
+            return self::ensureFeedURL($hints['feedurl'], $hints);
+        }
+
+        // Get the feed URL from HTML
+
         $discover = new FeedDiscovery();
-        if (isset($hints['feedurl'])) {
-            $feeduri = $hints['feedurl'];
-            $feeduri = $discover->discoverFromFeedURL($feeduri);
-        } else {
-            $feeduri = $discover->discoverFromURL($profile_uri);
-            $hints['feedurl'] = $feeduri;
+
+        $feedurl = $discover->discoverFromHTML($finalUrl, $body);
+
+        if (!empty($feedurl)) {
+            $hints['feedurl'] = $feedurl;
+            return self::ensureFeedURL($feedurl, $hints);
         }
 
+        throw new Exception("Could not find a feed URL for profile page " . $finalUrl);
+    }
+
+    /**
+     * Look up the Ostatus_profile, if present, for a remote entity with the
+     * given profile page URL. Will return null for both unknown and invalid
+     * remote profiles.
+     *
+     * @return mixed Ostatus_profile or null
+     * @throws OStatusShadowException for local profiles
+     */
+    static function getFromProfileURL($profile_url)
+    {
+        $profile = Profile::staticGet('profileurl', $profile_url);
+
+        if (empty($profile)) {
+            return null;
+        }
+
+        // Is it a known Ostatus profile?
+
+        $oprofile = Ostatus_profile::staticGet('profile_id', $profile->id);
+
+        if (!empty($oprofile)) {
+            return $oprofile;
+        }
+
+        // Is it a local user?
+
+        $user = User::staticGet('id', $profile->id);
+
+        if (!empty($user)) {
+            throw new OStatusShadowException($profile, "'$profile_url' is the profile for local user '{$user->nickname}'.");
+        }
+
+        // Continue discovery; it's a remote profile
+        // for OMB or some other protocol, may also
+        // support OStatus
+
+        return null;
+    }
+
+    /**
+     * Look up and if necessary create an Ostatus_profile for remote entity
+     * with the given update feed. This should never return null -- you will
+     * either get an object or an exception will be thrown.
+     *
+     * @return Ostatus_profile
+     * @throws Exception
+     */
+    public static function ensureFeedURL($feed_url, $hints=array())
+    {
+        $discover = new FeedDiscovery();
+
+        $feeduri = $discover->discoverFromFeedURL($feed_url);
+        $hints['feedurl'] = $feeduri;
+
         $huburi = $discover->getAtomLink('hub');
         $hints['hub'] = $huburi;
         $salmonuri = $discover->getAtomLink(Salmon::NS_REPLIES);
@@ -716,9 +872,32 @@ class Ostatus_profile extends Memcached_DataObject
             throw new FeedSubNoHubException();
         }
 
-        // Try to get a profile from the feed activity:subject
+        $feedEl = $discover->root;
+
+        if ($feedEl->tagName == 'feed') {
+            return self::ensureAtomFeed($feedEl, $hints);
+        } else if ($feedEl->tagName == 'channel') {
+            return self::ensureRssChannel($feedEl, $hints);
+        } else {
+            throw new FeedSubBadXmlException($feeduri);
+        }
+    }
 
-        $feedEl = $discover->feed->documentElement;
+    /**
+     * Look up and, if necessary, create an Ostatus_profile for the remote
+     * profile with the given Atom feed - actually loaded from the feed.
+     * This should never return null -- you will either get an object or
+     * an exception will be thrown.
+     *
+     * @param DOMElement $feedEl root element of a loaded Atom feed
+     * @param array $hints additional discovery information passed from higher levels
+     * @fixme should this be marked public?
+     * @return Ostatus_profile
+     * @throws Exception
+     */
+    public static function ensureAtomFeed($feedEl, $hints)
+    {
+        // Try to get a profile from the feed activity:subject
 
         $subject = ActivityUtils::child($feedEl, Activity::SUBJECT, Activity::SPEC);
 
@@ -739,7 +918,7 @@ class Ostatus_profile extends Memcached_DataObject
         // Sheesh. Not a very nice feed! Let's try fingerpoken in the
         // entries.
 
-        $entries = $discover->feed->getElementsByTagNameNS(Activity::ATOM, 'entry');
+        $entries = $feedEl->getElementsByTagNameNS(Activity::ATOM, 'entry');
 
         if (!empty($entries) && $entries->length > 0) {
 
@@ -767,8 +946,51 @@ class Ostatus_profile extends Memcached_DataObject
     }
 
     /**
+     * Look up and, if necessary, create an Ostatus_profile for the remote
+     * profile with the given RSS feed - actually loaded from the feed.
+     * This should never return null -- you will either get an object or
+     * an exception will be thrown.
      *
+     * @param DOMElement $feedEl root element of a loaded RSS feed
+     * @param array $hints additional discovery information passed from higher levels
+     * @fixme should this be marked public?
+     * @return Ostatus_profile
+     * @throws Exception
+     */
+    public static function ensureRssChannel($feedEl, $hints)
+    {
+        // Special-case for Posterous. They have some nice metadata in their
+        // posterous:author elements. We should use them instead of the channel.
+
+        $items = $feedEl->getElementsByTagName('item');
+
+        if ($items->length > 0) {
+            $item = $items->item(0);
+            $authorEl = ActivityUtils::child($item, ActivityObject::AUTHOR, ActivityObject::POSTEROUS);
+            if (!empty($authorEl)) {
+                $obj = ActivityObject::fromPosterousAuthor($authorEl);
+                // Posterous has multiple authors per feed, and multiple feeds
+                // per author. We check if this is the "main" feed for this author.
+                if (array_key_exists('profileurl', $hints) &&
+                    !empty($obj->poco) &&
+                    common_url_to_nickname($hints['profileurl']) == $obj->poco->preferredUsername) {
+                    return self::ensureActivityObjectProfile($obj, $hints);
+                }
+            }
+        }
+
+        // @fixme we should check whether this feed has elements
+        // with different <author> or <dc:creator> elements, and... I dunno.
+        // Do something about that.
+
+        $obj = ActivityObject::fromRssChannel($feedEl);
+
+        return self::ensureActivityObjectProfile($obj, $hints);
+    }
+
+    /**
      * Download and update given avatar image
+     *
      * @param string $url
      * @throws Exception in various failure cases
      */
@@ -778,6 +1000,9 @@ class Ostatus_profile extends Memcached_DataObject
             // We've already got this one.
             return;
         }
+        if (!common_valid_http_url($url)) {
+            throw new ServerException(sprintf(_m("Invalid avatar URL %s"), $url));
+        }
 
         if ($this->isGroup()) {
             $self = $this->localGroup();
@@ -895,11 +1120,14 @@ class Ostatus_profile extends Memcached_DataObject
     /**
      * Fetch, or build if necessary, an Ostatus_profile for the actor
      * in a given Activity Streams activity.
+     * This should never return null -- you will either get an object or
+     * an exception will be thrown.
      *
      * @param Activity $activity
      * @param string $feeduri if we already know the canonical feed URI!
      * @param string $salmonuri if we already know the salmon return channel URI
      * @return Ostatus_profile
+     * @throws Exception
      */
 
     public static function ensureActorProfile($activity, $hints=array())
@@ -907,6 +1135,18 @@ class Ostatus_profile extends Memcached_DataObject
         return self::ensureActivityObjectProfile($activity->actor, $hints);
     }
 
+    /**
+     * Fetch, or build if necessary, an Ostatus_profile for the profile
+     * in a given Activity Streams object (can be subject, actor, or object).
+     * This should never return null -- you will either get an object or
+     * an exception will be thrown.
+     *
+     * @param ActivityObject $object
+     * @param array $hints additional discovery information passed from higher levels
+     * @return Ostatus_profile
+     * @throws Exception
+     */
+
     public static function ensureActivityObjectProfile($object, $hints=array())
     {
         $profile = self::getActivityObjectProfile($object);
@@ -921,35 +1161,45 @@ class Ostatus_profile extends Memcached_DataObject
     /**
      * @param Activity $activity
      * @return mixed matching Ostatus_profile or false if none known
+     * @throws ServerException if feed info invalid
      */
     public static function getActorProfile($activity)
     {
         return self::getActivityObjectProfile($activity->actor);
     }
 
+    /**
+     * @param ActivityObject $activity
+     * @return mixed matching Ostatus_profile or false if none known
+     * @throws ServerException if feed info invalid
+     */
     protected static function getActivityObjectProfile($object)
     {
         $uri = self::getActivityObjectProfileURI($object);
         return Ostatus_profile::staticGet('uri', $uri);
     }
 
-    protected static function getActorProfileURI($activity)
-    {
-        return self::getActivityObjectProfileURI($activity->actor);
-    }
-
     /**
-     * @param Activity $activity
+     * Get the identifier URI for the remote entity described
+     * by this ActivityObject. This URI is *not* guaranteed to be
+     * a resolvable HTTP/HTTPS URL.
+     *
+     * @param ActivityObject $object
      * @return string
-     * @throws ServerException
+     * @throws ServerException if feed info invalid
      */
     protected static function getActivityObjectProfileURI($object)
     {
-        $opts = array('allowed_schemes' => array('http', 'https'));
-        if ($object->id && Validate::uri($object->id, $opts)) {
-            return $object->id;
+        if ($object->id) {
+            if (ActivityUtils::validateUri($object->id)) {
+                return $object->id;
+            }
         }
-        if ($object->link && Validate::uri($object->link, $opts)) {
+
+        // If the id is missing or invalid (we've seen feeds mistakenly listing
+        // things like local usernames in that field) then we'll use the profile
+        // page link, if valid.
+        if ($object->link && common_valid_http_url($object->link)) {
             return $object->link;
         }
         throw new ServerException("No author ID URI found");
@@ -962,6 +1212,8 @@ class Ostatus_profile extends Memcached_DataObject
     /**
      * Create local ostatus_profile and profile/user_group entries for
      * the provided remote user or group.
+     * This should never return null -- you will either get an object or
+     * an exception will be thrown.
      *
      * @param ActivityObject $object
      * @param array $hints
@@ -975,7 +1227,16 @@ class Ostatus_profile extends Memcached_DataObject
 
         if (!$homeuri) {
             common_log(LOG_DEBUG, __METHOD__ . " empty actor profile URI: " . var_export($activity, true));
-            throw new ServerException("No profile URI");
+            throw new Exception("No profile URI");
+        }
+
+        $user = User::staticGet('uri', $homeuri);
+        if ($user) {
+            throw new Exception("Local user can't be referenced as remote.");
+        }
+
+        if (OStatusPlugin::localGroupFromUrl($homeuri)) {
+            throw new Exception("Local group can't be referenced as remote.");
         }
 
         if (array_key_exists('feedurl', $hints)) {
@@ -1042,15 +1303,23 @@ class Ostatus_profile extends Memcached_DataObject
 
         $ok = $oprofile->insert();
 
-        if ($ok) {
-            $avatar = self::getActivityObjectAvatar($object, $hints);
-            if ($avatar) {
+        if (!$ok) {
+            throw new ServerException("Can't save OStatus profile");
+        }
+
+        $avatar = self::getActivityObjectAvatar($object, $hints);
+
+        if ($avatar) {
+            try {
                 $oprofile->updateAvatar($avatar);
+            } catch (Exception $ex) {
+                // Profile is saved, but Avatar is messed up. We're
+                // just going to continue.
+                common_log(LOG_WARNING, "Exception saving OStatus profile avatar: ". $ex->getMessage());
             }
-            return $oprofile;
-        } else {
-            throw new ServerException("Can't save OStatus profile");
         }
+
+        return $oprofile;
     }
 
     /**
@@ -1069,7 +1338,11 @@ class Ostatus_profile extends Memcached_DataObject
         }
         $avatar = self::getActivityObjectAvatar($object, $hints);
         if ($avatar) {
-            $this->updateAvatar($avatar);
+            try {
+                $this->updateAvatar($avatar);
+            } catch (Exception $ex) {
+                common_log(LOG_WARNING, "Exception saving OStatus profile avatar: " . $ex->getMessage());
+            }
         }
     }
 
@@ -1217,9 +1490,19 @@ class Ostatus_profile extends Memcached_DataObject
             return $hints['nickname'];
         }
 
-        // Try the definitive ID
+        // Try the profile url (like foo.example.com or example.com/user/foo)
+
+        $profileUrl = ($object->link) ? $object->link : $hints['profileurl'];
+
+        if (!empty($profileUrl)) {
+            $nickname = self::nicknameFromURI($profileUrl);
+        }
+
+        // Try the URI (may be a tag:, http:, acct:, ...
 
-        $nickname = self::nicknameFromURI($object->id);
+        if (empty($nickname)) {
+            $nickname = self::nicknameFromURI($object->id);
+        }
 
         // Try a Webfinger if one was passed (way) down
 
@@ -1259,6 +1542,17 @@ class Ostatus_profile extends Memcached_DataObject
         }
     }
 
+    /**
+     * Look up, and if necessary create, an Ostatus_profile for the remote
+     * entity with the given webfinger address.
+     * This should never return null -- you will either get an object or
+     * an exception will be thrown.
+     *
+     * @param string $addr webfinger address
+     * @return Ostatus_profile
+     * @throws Exception on error conditions
+     * @throws OStatusShadowException if this reference would obscure a local user/group
+     */
     public static function ensureWebfinger($addr)
     {
         // First, try the cache
@@ -1267,7 +1561,8 @@ class Ostatus_profile extends Memcached_DataObject
 
         if ($uri !== false) {
             if (is_null($uri)) {
-                return null;
+                // Negative cache entry
+                throw new Exception('Not a valid webfinger address.');
             }
             $oprofile = Ostatus_profile::staticGet('uri', $uri);
             if (!empty($oprofile)) {
@@ -1275,7 +1570,7 @@ class Ostatus_profile extends Memcached_DataObject
             }
         }
 
-        // First, look it up
+        // Try looking it up
 
         $oprofile = Ostatus_profile::staticGet('uri', 'acct:'.$addr);
 
@@ -1289,49 +1584,36 @@ class Ostatus_profile extends Memcached_DataObject
         $disco = new Discovery();
 
         try {
-            $result = $disco->lookup($addr);
+            $xrd = $disco->lookup($addr);
         } catch (Exception $e) {
+            // Save negative cache entry so we don't waste time looking it up again.
+            // @fixme distinguish temporary failures?
             self::cacheSet(sprintf('ostatus_profile:webfinger:%s', $addr), null);
-            return null;
+            throw new Exception('Not a valid webfinger address.');
         }
 
-        foreach ($result->links as $link) {
-            switch ($link['rel']) {
-            case Discovery::PROFILEPAGE:
-                $profileUrl = $link['href'];
-                break;
-            case Salmon::NS_REPLIES:
-                $salmonEndpoint = $link['href'];
-                break;
-            case Discovery::UPDATESFROM:
-                $feedUrl = $link['href'];
-                break;
-            case Discovery::HCARD:
-                $hcardUrl = $link['href'];
-                break;
-            default:
-                common_log(LOG_NOTICE, "Don't know what to do with rel = '{$link['rel']}'");
-                break;
-            }
-        }
+        $hints = array('webfinger' => $addr);
 
-        $hints = array('webfinger' => $addr,
-                       'profileurl' => $profileUrl,
-                       'feedurl' => $feedUrl,
-                       'salmon' => $salmonEndpoint);
+        $dhints = DiscoveryHints::fromXRD($xrd);
 
-        if (isset($hcardUrl)) {
-            $hcardHints = self::slurpHcard($hcardUrl);
-            // Note: Webfinger > hcard
-            $hints = array_merge($hcardHints, $hints);
+        $hints = array_merge($hints, $dhints);
+
+        // If there's an Hcard, let's grab its info
+
+        if (array_key_exists('hcard', $hints)) {
+            if (!array_key_exists('profileurl', $hints) ||
+                $hints['hcard'] != $hints['profileurl']) {
+                $hcardHints = DiscoveryHints::fromHcardUrl($hints['hcard']);
+                $hints = array_merge($hcardHints, $hints);
+            }
         }
 
         // If we got a feed URL, try that
 
-        if (isset($feedUrl)) {
+        if (array_key_exists('feedurl', $hints)) {
             try {
-                common_log(LOG_INFO, "Discovery on acct:$addr with feed URL $feedUrl");
-                $oprofile = self::ensureProfile($feedUrl, $hints);
+                common_log(LOG_INFO, "Discovery on acct:$addr with feed URL " . $hints['feedurl']);
+                $oprofile = self::ensureFeedURL($hints['feedurl'], $hints);
                 self::cacheSet(sprintf('ostatus_profile:webfinger:%s', $addr), $oprofile->uri);
                 return $oprofile;
             } catch (Exception $e) {
@@ -1342,22 +1624,33 @@ class Ostatus_profile extends Memcached_DataObject
 
         // If we got a profile page, try that!
 
-        if (isset($profileUrl)) {
+        if (array_key_exists('profileurl', $hints)) {
             try {
                 common_log(LOG_INFO, "Discovery on acct:$addr with profile URL $profileUrl");
-                $oprofile = self::ensureProfile($profileUrl, $hints);
+                $oprofile = self::ensureProfileURL($hints['profileurl'], $hints);
                 self::cacheSet(sprintf('ostatus_profile:webfinger:%s', $addr), $oprofile->uri);
                 return $oprofile;
+            } catch (OStatusShadowException $e) {
+                // We've ended up with a remote reference to a local user or group.
+                // @fixme ideally we should be able to say who it was so we can
+                // go back and refer to it the regular way
+                throw $e;
             } catch (Exception $e) {
                 common_log(LOG_WARNING, "Failed creating profile from profile URL '$profileUrl': " . $e->getMessage());
                 // keep looking
+                //
+                // @fixme this means an error discovering from profile page
+                // may give us a corrupt entry using the webfinger URI, which
+                // will obscure the correct page-keyed profile later on.
             }
         }
 
         // XXX: try hcard
         // XXX: try FOAF
 
-        if (isset($salmonEndpoint)) {
+        if (array_key_exists('salmon', $hints)) {
+
+            $salmonEndpoint = $hints['salmon'];
 
             // An account URL, a salmon endpoint, and a dream? Not much to go
             // on, but let's give it a try
@@ -1402,13 +1695,25 @@ class Ostatus_profile extends Memcached_DataObject
             return $oprofile;
         }
 
-        return null;
+        throw new Exception("Couldn't find a valid profile for '$addr'");
     }
 
+    /**
+     * Store the full-length scrubbed HTML of a remote notice to an attachment
+     * file on our server. We'll link to this at the end of the cropped version.
+     *
+     * @param string $title plaintext for HTML page's title
+     * @param string $rendered HTML fragment for HTML page's body
+     * @return File
+     */
     function saveHTMLFile($title, $rendered)
     {
-        $final = sprintf("<!DOCTYPE html>\n<html><head><title>%s</title></head>".
-                         '<body><div>%s</div></body></html>',
+        $final = sprintf("<!DOCTYPE html>\n" .
+                         '<html><head>' .
+                         '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">' .
+                         '<title>%s</title>' .
+                         '</head>' .
+                         '<body>%s</body></html>',
                          htmlspecialchars($title),
                          $rendered);
 
@@ -1437,67 +1742,25 @@ class Ostatus_profile extends Memcached_DataObject
 
         return $file;
     }
+}
 
-    protected static function slurpHcard($url)
-    {
-        set_include_path(get_include_path() . PATH_SEPARATOR . INSTALLDIR . '/plugins/OStatus/extlib/hkit/');
-        require_once('hkit.class.php');
-
-        $h	= new hKit;
-
-        // Google Buzz hcards need to be tidied. Probably others too.
-
-        $h->tidy_mode = 'proxy'; // 'proxy', 'exec', 'php' or 'none'
-
-        // Get by URL
-        $hcards = $h->getByURL('hcard', $url);
-
-        if (empty($hcards)) {
-            return array();
-        }
-
-        // @fixme more intelligent guess on multi-hcard pages
-        $hcard = $hcards[0];
-
-        $hints = array();
-
-        $hints['profileurl'] = $url;
-
-        if (array_key_exists('nickname', $hcard)) {
-            $hints['nickname'] = $hcard['nickname'];
-        }
-
-        if (array_key_exists('fn', $hcard)) {
-            $hints['fullname'] = $hcard['fn'];
-        } else if (array_key_exists('n', $hcard)) {
-            $hints['fullname'] = implode(' ', $hcard['n']);
-        }
-
-        if (array_key_exists('photo', $hcard)) {
-            $hints['avatar'] = $hcard['photo'];
-        }
-
-        if (array_key_exists('note', $hcard)) {
-            $hints['bio'] = $hcard['note'];
-        }
-
-        if (array_key_exists('adr', $hcard)) {
-            if (is_string($hcard['adr'])) {
-                $hints['location'] = $hcard['adr'];
-            } else if (is_array($hcard['adr'])) {
-                $hints['location'] = implode(' ', $hcard['adr']);
-            }
-        }
-
-        if (array_key_exists('url', $hcard)) {
-            if (is_string($hcard['url'])) {
-                $hints['homepage'] = $hcard['url'];
-            } else if (is_array($hcard['adr'])) {
-                // HACK get the last one; that's how our hcards look
-                $hints['homepage'] = $hcard['url'][count($hcard['url'])-1];
-            }
-        }
+/**
+ * Exception indicating we've got a remote reference to a local user,
+ * not a remote user!
+ *
+ * If we can ue a local profile after all, it's available as $e->profile.
+ */
+class OStatusShadowException extends Exception
+{
+    public $profile;
 
-        return $hints;
+    /**
+     * @param Profile $profile
+     * @param string $message
+     */
+    function __construct($profile, $message) {
+        $this->profile = $profile;
+        parent::__construct($message);
     }
 }
+