Merge remote branch 'laconica/0.8.x' into 0.9.x

Conflicts:
	lib/common.php
	lib/twitter.php

9 files changed, 564 insertions, 122 deletions

diff --git a/lib/command.php b/lib/command.php
index cae7a8298..11d40b8e1 100644
--- a/lib/command.php
+++ b/lib/command.php
@@ -114,7 +114,6 @@ class StatsCommand extends Command
 
 class FavCommand extends Command
 {
-
     var $other = null;
 
     function __construct($user, $other)
@@ -158,6 +157,108 @@ class FavCommand extends Command
 
         $channel->output($this->user, _('Notice marked as fave.'));
     }
+
+}
+class JoinCommand extends Command
+{
+    var $other = null;
+
+    function __construct($user, $other)
+    {
+        parent::__construct($user);
+        $this->other = $other;
+    }
+
+    function execute($channel)
+    {
+
+        $nickname = common_canonical_nickname($this->other);
+        $group    = User_group::staticGet('nickname', $nickname);
+        $cur      = $this->user;
+
+        if (!$group) {
+            $channel->error($cur, _('No such group.'));
+            return;
+        }
+
+        if ($cur->isMember($group)) {
+            $channel->error($cur, _('You are already a member of that group'));
+            return;
+        }
+        if (Group_block::isBlocked($group, $cur->getProfile())) {
+          $channel->error($cur, _('You have been blocked from that group by the admin.'));
+            return;
+        }
+
+        $member = new Group_member();
+
+        $member->group_id   = $group->id;
+        $member->profile_id = $cur->id;
+        $member->created    = common_sql_now();
+
+        $result = $member->insert();
+        if (!$result) {
+          common_log_db_error($member, 'INSERT', __FILE__);
+          $channel->error($cur, sprintf(_('Could not join user %s to group %s'),
+                                       $cur->nickname, $group->nickname));
+          return;
+        }
+
+        $channel->output($cur, sprintf(_('%s joined group %s'),
+                                              $cur->nickname,
+                                              $group->nickname));
+    }
+
+}
+class DropCommand extends Command
+{
+    var $other = null;
+
+    function __construct($user, $other)
+    {
+        parent::__construct($user);
+        $this->other = $other;
+    }
+
+    function execute($channel)
+    {
+
+        $nickname = common_canonical_nickname($this->other);
+        $group    = User_group::staticGet('nickname', $nickname);
+        $cur      = $this->user;
+
+        if (!$group) {
+            $channel->error($cur, _('No such group.'));
+            return;
+        }
+
+        if (!$cur->isMember($group)) {
+            $channel->error($cur, _('You are not a member of that group.'));
+            return;
+        }
+
+        $member = new Group_member();
+
+        $member->group_id   = $group->id;
+        $member->profile_id = $cur->id;
+
+        if (!$member->find(true)) {
+          $channel->error($cur,_('Could not find membership record.'));
+          return;
+        }
+        $result = $member->delete();
+        if (!$result) {
+          common_log_db_error($member, 'INSERT', __FILE__);
+          $channel->error($cur, sprintf(_('Could not remove user %s to group %s'),
+                                       $cur->nickname, $group->nickname));
+          return;
+        }
+
+        $channel->output($cur, sprintf(_('%s left group %s'),
+                                              $cur->nickname,
+                                              $group->nickname));
+    }
+
 }
 
 class WhoisCommand extends Command
@@ -396,6 +497,8 @@ class HelpCommand extends Command
                            "get <nickname> - get last notice from user\n".
                            "whois <nickname> - get profile info on user\n".
                            "fav <nickname> - add user's last notice as a 'fave'\n".
+                           "join <group> - join group\n".
+                           "drop <group> - leave group\n".
                            "stats - get your stats\n".
                            "stop - same as 'off'\n".
                            "quit - same as 'off'\n".
diff --git a/lib/commandinterpreter.php b/lib/commandinterpreter.php
index ac6bf73c8..6e4340e5d 100644
--- a/lib/commandinterpreter.php
+++ b/lib/commandinterpreter.php
@@ -70,6 +70,26 @@ class CommandInterpreter
             } else {
                 return new OffCommand($user);
             }
+         case 'join':
+             if (!$arg) {
+                return null;
+            }
+            list($other, $extra) = explode(' ', $arg, 2);
+            if ($extra) {
+                return null;
+            } else {
+                return new JoinCommand($user, $other);
+            }
+         case 'drop':
+            if (!$arg) {
+                return null;
+            }
+            list($other, $extra) = explode(' ', $arg, 2);
+            if ($extra) {
+                return null;
+            } else {
+                return new DropCommand($user, $other);
+            }
          case 'follow':
          case 'sub':
             if (!$arg) {
diff --git a/lib/mailbox.php b/lib/mailbox.php
index a09bf1060..e1d384a06 100644
--- a/lib/mailbox.php
+++ b/lib/mailbox.php
@@ -213,26 +213,20 @@ class MailboxAction extends CurrentUserDesignAction
         }
 
         $this->elementStart('div', 'entry-content');
-        $this->elementStart('dl', 'timestamp');
-        $this->element('dt', null, _('Published'));
-        $this->elementStart('dd', null);
-        $dt = common_date_iso8601($message->created);
         $this->elementStart('a', array('rel' => 'bookmark',
+                                       'class' => 'timestamp',
                                        'href' => $messageurl));
+        $dt = common_date_iso8601($message->created);
         $this->element('abbr', array('class' => 'published',
                                      'title' => $dt),
                                common_date_string($message->created));
         $this->elementEnd('a');
-        $this->elementEnd('dd');
-        $this->elementEnd('dl');
 
         if ($message->source) {
-            $this->elementStart('dl', 'device');
-            $this->elementStart('dt');
-            $this->text(_('From'));
-            $this->elementEnd('dt');
-            $this->showSource($message->source);
-            $this->elementEnd('dl');
+            $this->elementStart('span', 'source');
+            $this->text(_('from'));
+            $this->element('span', 'device', $this->showSource($message->source));
+            $this->elementEnd('span');
         }
         $this->elementEnd('div');
 
@@ -277,18 +271,18 @@ class MailboxAction extends CurrentUserDesignAction
         case 'mail':
         case 'omb':
         case 'api':
-            $this->element('dd', null, $source_name);
+            $this->element('span', 'device', $source_name);
             break;
         default:
             $ns = Notice_source::staticGet($source);
             if ($ns) {
-                $this->elementStart('dd', null);
+                $this->elementStart('span', 'device');
                 $this->element('a', array('href' => $ns->url,
-                                          'rel' => 'external'),
-                               $ns->name);
-                $this->elementEnd('dd');
+                                               'rel' => 'external'),
+                                    $ns->name);
+                $this->elementEnd('span');
             } else {
-                $this->element('dd', null, $source_name);
+                $this->out->element('span', 'device', $source_name);
             }
             break;
         }
diff --git a/lib/oauthclient.php b/lib/oauthclient.php
index cc10cea8f..f1827726e 100644
--- a/lib/oauthclient.php
+++ b/lib/oauthclient.php
@@ -22,7 +22,7 @@
  * @category  Action
  * @package   StatusNet
  * @author    Zach Copley <zach@status.net>
- * @copyright 2008 StatusNet, Inc.
+ * @copyright 2009 StatusNet, Inc.
  * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
  * @link      http://status.net/
  */
diff --git a/lib/twitter.php b/lib/twitter.php
index 2d2b08f73..b49e2e119 100644
--- a/lib/twitter.php
+++ b/lib/twitter.php
@@ -154,75 +154,126 @@ function broadcast_twitter($notice)
                                        TWITTER_SERVICE);
 
     if (is_twitter_bound($notice, $flink)) {
+        if (TwitterOAuthClient::isPackedToken($flink->credentials)) {
+            return broadcast_oauth($notice, $flink);
+        } else {
+            return broadcast_basicauth($notice, $flink);
+        }
+    }
 
-        $user = $flink->getUser();
-
-        // XXX: Hack to get around PHP cURL's use of @ being a a meta character
-        $statustxt = preg_replace('/^@/', ' @', $notice->content);
-
-        // Convert !groups to #hashes
-        $statustxt = preg_replace('/(^|\s)!([A-Za-z0-9]{1,64})/', "\\1#\\2", $statustxt);
+    return true;
+}
 
-        $token = TwitterOAuthClient::unpackToken($flink->credentials);
+function broadcast_oauth($notice, $flink) {
+    $user = $flink->getUser();
+    $statustxt = format_status($notice);
+    // Convert !groups to #hashes
+    $statustxt = preg_replace('/(^|\s)!([A-Za-z0-9]{1,64})/', "\\1#\\2", $statustxt);
+    $token = TwitterOAuthClient::unpackToken($flink->credentials);
+    $client = new TwitterOAuthClient($token->key, $token->secret);
+    $status = null;
+
+    try {
+        $status = $client->statusesUpdate($statustxt);
+    } catch (OAuthClientCurlException $e) {
+        return process_error($e, $flink);
+    }
 
-        $client = new TwitterOAuthClient($token->key, $token->secret);
+    if (empty($status)) {
 
-        $status = null;
+        // This could represent a failure posting,
+        // or the Twitter API might just be behaving flakey.
 
-        try {
-            $status = $client->statusesUpdate($statustxt);
-        } catch (OAuthClientCurlException $e) {
+        $errmsg = sprintf('Twitter bridge - No data returned by Twitter API when ' .
+                          'trying to send update for %1$s (user id %2$s).',
+                          $user->nickname, $user->id);
+        common_log(LOG_WARNING, $errmsg);
 
-            if ($e->getMessage() == 'The requested URL returned error: 401') {
+        return false;
+    }
 
-                $errmsg = sprintf('User %1$s (user id: %2$s) has an invalid ' .
-                                  'Twitter OAuth access token.',
-                                  $user->nickname, $user->id);
-                common_log(LOG_WARNING, $errmsg);
+    // Notice crossed the great divide
 
-                // Bad auth token! We need to delete the foreign_link
-                // to Twitter and inform the user.
+    $msg = sprintf('Twitter bridge - posted notice %s to Twitter using OAuth.',
+                   $notice->id);
+    common_log(LOG_INFO, $msg);
 
-                remove_twitter_link($flink);
-                return true;
+    return true;
+}
 
-            } else {
+function broadcast_basicauth($notice, $flink)
+{
+    $user = $flink->getUser();
 
-                // Some other error happened, so we should probably
-                // try to send again later.
+    $statustxt = format_status($notice);
 
-                $errmsg = sprintf('cURL error trying to send notice to Twitter ' .
-                                  'for user %1$s (user id: %2$s) - ' .
-                                  'code: %3$s message: $4$s.',
-                                  $user->nickname, $user->id,
-                                  $e->getCode(), $e->getMessage());
-                common_log(LOG_WARNING, $errmsg);
+    $client = new TwitterBasicAuthClient($flink);
+    $status = null;
 
-                return false;
-            }
-        }
+    try {
+        $status = $client->statusesUpdate($statustxt);
+    } catch (BasicAuthCurlException $e) {
+        return process_error($e, $flink);
+    }
 
-        if (empty($status)) {
+    if (empty($status)) {
 
-            // This could represent a failure posting,
-            // or the Twitter API might just be behaving flakey.
+        $errmsg = sprintf('Twitter bridge - No data returned by Twitter API when ' .
+                          'trying to send update for %1$s (user id %2$s).',
+                          $user->nickname, $user->id);
+        common_log(LOG_WARNING, $errmsg);
 
             $errmsg = sprintf('No data returned by Twitter API when ' .
                              'trying to send update for %1$s (user id %2$s).',
                              $user->nickname, $user->id);
             common_log(LOG_WARNING, $errmsg);
+        return false;
+    }
 
-            return false;
-        }
+    $msg = sprintf('Twitter bridge - posted notice %s to Twitter using basic auth.',
+                   $notice->id);
+    common_log(LOG_INFO, $msg);
 
-        // Notice crossed the great divide
+    return true;
+}
 
-        $msg = sprintf('Twitter bridge posted notice %s to Twitter.',
-                       $notice->id);
-        common_log(LOG_INFO, $msg);
+function process_error($e, $flink)
+{
+    $user        = $flink->getUser();
+    $errmsg      = $e->getMessage();
+    $delivered   = false;
+
+    switch($errmsg) {
+     case 'The requested URL returned error: 401':
+        $logmsg = sprintf('Twiter bridge - User %1$s (user id: %2$s) has an invalid ' .
+                          'Twitter screen_name/password combo or an invalid acesss token.',
+                          $user->nickname, $user->id);
+        $delivered = true;
+        remove_twitter_link($flink);
+        break;
+     case 'The requested URL returned error: 403':
+        $logmsg = sprintf('Twitter bridge - User %1$s (user id: %2$s) has exceeded ' .
+                          'his/her Twitter request limit.',
+                          $user->nickname, $user->id);
+        break;
+     default:
+        $logmsg = sprintf('Twitter bridge - cURL error trying to send notice to Twitter ' .
+                          'for user %1$s (user id: %2$s) - ' .
+                          'code: %3$s message: %4$s.',
+                          $user->nickname, $user->id,
+                          $e->getCode(), $e->getMessage());
+        break;
     }
 
-    return true;
+    common_log(LOG_WARNING, $logmsg);
+
+    return $delivered;
+}
+
+function format_status($notice)
+{
+    // XXX: Hack to get around PHP cURL's use of @ being a a meta character
+    return preg_replace('/^@/', ' @', $notice->content);
 }
 
 function remove_twitter_link($flink)
@@ -230,7 +281,7 @@ function remove_twitter_link($flink)
     $user = $flink->getUser();
 
     common_log(LOG_INFO, 'Removing Twitter bridge Foreign link for ' .
-        "user $user->nickname (user id: $user->id).");
+               "user $user->nickname (user id: $user->id).");
 
     $result = $flink->delete();
 
diff --git a/lib/twitterapi.php b/lib/twitterapi.php
index 4612f74e9..d199e4dee 100644
--- a/lib/twitterapi.php
+++ b/lib/twitterapi.php
@@ -788,6 +788,52 @@ class TwitterapiAction extends Action
         $this->end_document('xml');
     }
 
+    function show_twitter_xml_users($user)
+    {
+
+        $this->init_document('xml');
+        $this->elementStart('users', array('type' => 'array'));
+
+        if (is_array($user)) {
+            foreach ($group as $g) {
+                $twitter_user = $this->twitter_user_array($g);
+                $this->show_twitter_xml_user($twitter_user,'user');
+            }
+        } else {
+            while ($user->fetch()) {
+                $twitter_user = $this->twitter_user_array($user);
+                $this->show_twitter_xml_user($twitter_user);
+            }
+        }
+
+        $this->elementEnd('users');
+        $this->end_document('xml');
+    }
+
+    function show_json_users($user)
+    {
+
+        $this->init_document('json');
+
+        $users = array();
+
+        if (is_array($user)) {
+            foreach ($user as $u) {
+                $twitter_user = $this->twitter_user_array($u);
+                array_push($users, $twitter_user);
+            }
+        } else {
+            while ($user->fetch()) {
+                $twitter_user = $this->twitter_user_array($user);
+                array_push($users, $twitter_user);
+            }
+        }
+
+        $this->show_json_objects($users);
+
+        $this->end_document('json');
+    }
+
     function show_single_json_group($group)
     {
         $this->init_document('json');
diff --git a/lib/twitterbasicauthclient.php b/lib/twitterbasicauthclient.php
new file mode 100644
index 000000000..fd331fbdc
--- /dev/null
+++ b/lib/twitterbasicauthclient.php
@@ -0,0 +1,236 @@
+<?php
+/**
+ * StatusNet, the distributed open-source microblogging tool
+ *
+ * Class for doing OAuth calls against Twitter
+ *
+ * PHP version 5
+ *
+ * LICENCE: This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Integration
+ * @package   StatusNet
+ * @author    Zach Copley <zach@status.net>
+ * @copyright 2009 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET') && !defined('LACONICA')) {
+    exit(1);
+}
+
+/**
+ * Exception wrapper for cURL errors
+ *
+ * @category Integration
+ * @package  StatusNet
+ * @author   Zach Copley <zach@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
+ * @link     http://status.net/
+ *
+ */
+class BasicAuthCurlException extends Exception
+{
+}
+
+/**
+ * Class for talking to the Twitter API with HTTP Basic Auth.
+ *
+ * @category Integration
+ * @package  StatusNet
+ * @author   Zach Copley <zach@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
+ * @link     http://status.net/
+ *
+ */
+class TwitterBasicAuthClient
+{
+    var $screen_name = null;
+    var $password    = null;
+
+    /**
+     * constructor
+     *
+     * @param Foreign_link $flink a Foreign_link storing the
+     *                            Twitter user's password, etc.
+     */
+    function __construct($flink)
+    {
+        $fuser             = $flink->getForeignUser();
+        $this->screen_name = $fuser->nickname;
+        $this->password    = $flink->credentials;
+    }
+
+    /**
+     * Calls Twitter's /statuses/update API method
+     *
+     * @param string $status                text of the status
+     * @param int    $in_reply_to_status_id optional id of the status it's
+     *                                      a reply to
+     *
+     * @return mixed the status
+     */
+    function statusesUpdate($status, $in_reply_to_status_id = null)
+    {
+        $url      = 'https://twitter.com/statuses/update.json';
+        $params   = array('status' => $status,
+                          'source' => common_config('integration', 'source'),
+                          'in_reply_to_status_id' => $in_reply_to_status_id);
+        $response = $this->httpRequest($url, $params);
+        $status   = json_decode($response);
+        return $status;
+    }
+
+    /**
+     * Calls Twitter's /statuses/friends_timeline API method
+     *
+     * @param int $since_id show statuses after this id
+     * @param int $max_id   show statuses before this id
+     * @param int $cnt      number of statuses to show
+     * @param int $page     page number
+     *
+     * @return mixed an array of statuses
+     */
+    function statusesFriendsTimeline($since_id = null, $max_id = null,
+                                     $cnt = null, $page = null)
+    {
+        $url    = 'https://twitter.com/statuses/friends_timeline.json';
+        $params = array('since_id' => $since_id,
+                        'max_id' => $max_id,
+                        'count' => $cnt,
+                        'page' => $page);
+        $qry    = http_build_query($params);
+
+        if (!empty($qry)) {
+            $url .= "?$qry";
+        }
+
+        $response = $this->httpRequest($url);
+        $statuses = json_decode($response);
+        return $statuses;
+    }
+
+    /**
+     * Calls Twitter's /statuses/friends API method
+     *
+     * @param int $id          id of the user whom you wish to see friends of
+     * @param int $user_id     numerical user id
+     * @param int $screen_name screen name
+     * @param int $page        page number
+     *
+     * @return mixed an array of twitter users and their latest status
+     */
+    function statusesFriends($id = null, $user_id = null, $screen_name = null,
+                             $page = null)
+    {
+        $url = "https://twitter.com/statuses/friends.json";
+
+        $params = array('id' => $id,
+                        'user_id' => $user_id,
+                        'screen_name' => $screen_name,
+                        'page' => $page);
+        $qry    = http_build_query($params);
+
+        if (!empty($qry)) {
+            $url .= "?$qry";
+        }
+
+        $response = $this->httpRequest($url);
+        $friends  = json_decode($response);
+        return $friends;
+    }
+
+    /**
+     * Calls Twitter's /statuses/friends/ids API method
+     *
+     * @param int $id          id of the user whom you wish to see friends of
+     * @param int $user_id     numerical user id
+     * @param int $screen_name screen name
+     * @param int $page        page number
+     *
+     * @return mixed a list of ids, 100 per page
+     */
+    function friendsIds($id = null, $user_id = null, $screen_name = null,
+                        $page = null)
+    {
+        $url = "https://twitter.com/friends/ids.json";
+
+        $params = array('id' => $id,
+                        'user_id' => $user_id,
+                        'screen_name' => $screen_name,
+                        'page' => $page);
+        $qry    = http_build_query($params);
+
+        if (!empty($qry)) {
+            $url .= "?$qry";
+        }
+
+        $response = $this->httpRequest($url);
+        $ids      = json_decode($response);
+        return $ids;
+    }
+
+    /**
+     * Make a HTTP request using cURL.
+     *
+     * @param string $url    Where to make the request
+     * @param array  $params post parameters
+     *
+     * @return mixed the request
+     */
+    function httpRequest($url, $params = null, $auth = true)
+    {
+        $options = array(
+                         CURLOPT_RETURNTRANSFER => true,
+                         CURLOPT_FAILONERROR    => true,
+                         CURLOPT_HEADER         => false,
+                         CURLOPT_FOLLOWLOCATION => true,
+                         CURLOPT_USERAGENT      => 'StatusNet',
+                         CURLOPT_CONNECTTIMEOUT => 120,
+                         CURLOPT_TIMEOUT        => 120,
+                         CURLOPT_HTTPAUTH       => CURLAUTH_ANY,
+                         CURLOPT_SSL_VERIFYPEER => false,
+
+                         // Twitter is strict about accepting invalid "Expect" headers
+
+                         CURLOPT_HTTPHEADER => array('Expect:')
+                         );
+
+        if (isset($params)) {
+            $options[CURLOPT_POST]       = true;
+            $options[CURLOPT_POSTFIELDS] = $params;
+        }
+
+        if ($auth) {
+            $options[CURLOPT_USERPWD] = $this->screen_name .
+              ':' . $this->password;
+        }
+
+        $ch = curl_init($url);
+        curl_setopt_array($ch, $options);
+        $response = curl_exec($ch);
+
+        if ($response === false) {
+            $msg  = curl_error($ch);
+            $code = curl_errno($ch);
+            throw new BasicAuthCurlException($msg, $code);
+        }
+
+        curl_close($ch);
+
+        return $response;
+    }
+
+}
diff --git a/lib/twitteroauthclient.php b/lib/twitteroauthclient.php
index e4a89c4e9..bad2b74ca 100644
--- a/lib/twitteroauthclient.php
+++ b/lib/twitteroauthclient.php
@@ -22,7 +22,7 @@
  * @category  Integration
  * @package   StatusNet
  * @author    Zach Copley <zach@status.net>
- * @copyright 2008 StatusNet, Inc.
+ * @copyright 2009 StatusNet, Inc.
  * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
  * @link      http://status.net/
  */
@@ -81,6 +81,15 @@ class TwitterOAuthClient extends OAuthClient
         return new OAuthToken($vals[0], $vals[1]);
     }
 
+    static function isPackedToken($str)
+    {
+        if (strpos($str, chr(0)) === false) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
     /**
      * Builds a link to Twitter's endpoint for authorizing a request token
      *
diff --git a/lib/util.php b/lib/util.php
index 337113c70..9cf462515 100644
--- a/lib/util.php
+++ b/lib/util.php
@@ -413,7 +413,7 @@ function common_replace_urls_callback($text, $callback, $notice_id = null) {
     // Start off with a regex
     $regex = '#'.
     '(?:^|[\s\(\)\[\]\{\}\\\'\\\";]+)(?![\@\!\#])'.
-    '(?P<url>'.
+    '('.
         '(?:'.
             '(?:'. //Known protocols
                 '(?:'.
@@ -421,7 +421,7 @@ function common_replace_urls_callback($text, $callback, $notice_id = null) {
                     '|'.
                     '(?:(?:mailto|aim|tel|xmpp):)'.
                 ')'.
-                '(?:[\pN\pL\-\_\+]+(?::[\pN\pL\-\_\+]+)?\@)?'. //user:pass@
+                '(?:[\pN\pL\-\_\+\%\~]+(?::[\pN\pL\-\_\+\%\~]+)?\@)?'. //user:pass@
                 '(?:'.
                     '(?:'.
                         '\[[\pN\pL\-\_\:\.]+(?<![\.\:])\]'. //[dns]
@@ -434,7 +434,7 @@ function common_replace_urls_callback($text, $callback, $notice_id = null) {
             '|(?:'. //IPv6
                 '\[?(?:(?:(?:[0-9A-Fa-f]{1,4}:){7}(?:(?:[0-9A-Fa-f]{1,4})|:))|(?:(?:[0-9A-Fa-f]{1,4}:){6}(?::|(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})|(?::[0-9A-Fa-f]{1,4})))|(?:(?:[0-9A-Fa-f]{1,4}:){5}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:){4}(?::[0-9A-Fa-f]{1,4}){0,1}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:){3}(?::[0-9A-Fa-f]{1,4}){0,2}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:){2}(?::[0-9A-Fa-f]{1,4}){0,3}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:)(?::[0-9A-Fa-f]{1,4}){0,4}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?::(?::[0-9A-Fa-f]{1,4}){0,5}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})))\]?'.
             ')|(?:'. //DNS
-                '(?:[\pN\pL\-\_\+]+(?:\:[\pN\pL\-\_\+]+)?\@)?'. //user:pass@
+                '(?:[\pN\pL\-\_\+\%\~]+(?:\:[\pN\pL\-\_\+\%\~]+)?\@)?'. //user:pass@
                 '[\pN\pL\-\_]+(?:\.[\pN\pL\-\_]+)*\.'.
                 //tld list from http://data.iana.org/TLD/tlds-alpha-by-domain.txt, also added local, loc, and onion
                 '(?:AC|AD|AE|AERO|AF|AG|AI|AL|AM|AN|AO|AQ|AR|ARPA|AS|ASIA|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BIZ|BJ|BM|BN|BO|BR|BS|BT|BV|BW|BY|BZ|CA|CAT|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|COM|COOP|CR|CU|CV|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EDU|EE|EG|ER|ES|ET|EU|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GOV|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|INFO|INT|IO|IQ|IR|IS|IT|JE|JM|JO|JOBS|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MG|MH|MIL|MK|ML|MM|MN|MO|MOBI|MP|MQ|MR|MS|MT|MU|MUSEUM|MV|MW|MX|MY|MZ|NA|NAME|NC|NE|NET|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|ORG|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PRO|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|ST|SU|SV|SY|SZ|TC|TD|TEL|TF|TG|TH|TJ|TK|TL|TM|TN|TO|TP|TR|TRAVEL|TT|TV|TW|TZ|UA|UG|UK|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|XN--0ZWM56D|测试|XN--11B5BS3A9AJ6G|परीक्षा|XN--80AKHBYKNJ4F|испытание|XN--9T4B11YI5A|테스트|XN--DEBA0AD|טעסט|XN--G6W251D|測試|XN--HGBK6AJ7F53BBA|آزمایشی|XN--HLCJ6AYA9ESC7A|பரிட்சை|XN--JXALPDLP|δοκιμή|XN--KGBECHTV|إختبار|XN--ZCKZAH|テスト|YE|YT|YU|ZA|ZM|ZW|local|loc|onion)'.
@@ -442,22 +442,22 @@ function common_replace_urls_callback($text, $callback, $notice_id = null) {
         ')'.
         '(?:'.
             '(?:\:\d+)?'. //:port
-            '(?:/[\pN\pL$\[\]\,\!\(\)\.\-\_\+\/\=\&\;]*)?'. // /path
-            '(?:\?[\pN\pL\$\[\]\,\!\(\)\.\-\_\+\/\=\&\;\/]*)?'. // ?query string
-            '(?:\#[\pN\pL$\[\]\,\!\(\)\.\-\_\+\/\=\&\;\/\?\#]*)?'. // #fragment
+            '(?:/[\pN\pL$\[\]\,\!\(\)\.\:\-\_\+\/\=\&\;\%\~]*)?'. // /path
+            '(?:\?[\pN\pL\$\[\]\,\!\(\)\.\:\-\_\+\/\=\&\;\%\~\/]*)?'. // ?query string
+            '(?:\#[\pN\pL$\[\]\,\!\(\)\.\:\-\_\+\/\=\&\;\%\~\/\?\#]*)?'. // #fragment
         ')(?<![\?\.\,\#\,])'.
     ')'.
     '#ixu';
     preg_match_all($regex,$text,$matches);
     //print_r($matches);
-    return preg_replace_callback($regex, curry(callback_helper,$callback,$notice_id) ,$text);
+    return preg_replace_callback($regex, curry('callback_helper',$callback,$notice_id) ,$text);
 }
 
 function callback_helper($matches, $callback, $notice_id) {
-    $url=$matches['url'];
+    $url=$matches[1];
     $left = strpos($matches[0],$url);
     $right = $left+strlen($url);
-    
+
     $groupSymbolSets=array(
         array(
             'left'=>'(',
@@ -491,13 +491,11 @@ function callback_helper($matches, $callback, $notice_id) {
             $url=substr($url,0,-1);
         }
     }while($original_url!=$url);
-    
-    
-    
+
     if(empty($notice_id)){
         $result = call_user_func_array($callback,$url);
     }else{
-        $result = call_user_func_array($callback, array($url,$notice_id) );
+        $result = call_user_func_array($callback, array(array($url,$notice_id)) );
     }
     return substr($matches[0],0,$left) . $result . substr($matches[0],$right);
 }
@@ -508,16 +506,13 @@ function curry($fn) {
     array_shift($args);
     $id = uniqid('_partial');
     $GLOBALS[$id] = array($fn, $args);
-    return create_function(
-        '',
-        '
-        $args = func_get_args();
-        return call_user_func_array(
-        $GLOBALS["'.$id.'"][0],
-        array_merge(
-            $args,
-            $GLOBALS["'.$id.'"][1]));
-    ');
+    return create_function('',
+                           '$args = func_get_args(); '.
+                           'return call_user_func_array('.
+                           '$GLOBALS["'.$id.'"][0],'.
+                           'array_merge('.
+                           '$args,'.
+                           '$GLOBALS["'.$id.'"][1]));');
 }
 
 function common_linkify($url) {
@@ -525,7 +520,7 @@ function common_linkify($url) {
     // functions
     $url = htmlspecialchars_decode($url);
 
-   if(strpos($url, '@')!==false && strpos($url, ':')===false){
+   if(strpos($url, '@') !== false && strpos($url, ':') === false) {
        //url is an email address without the mailto: protocol
        return XMLStringer::estring('a', array('href' => "mailto:$url", 'rel' => 'external'), $url);
    }
@@ -547,42 +542,30 @@ function common_linkify($url) {
     $attachment_id = null;
     $has_thumb = false;
 
-    // Check to see whether there's a filename associated with this URL.
-    // If there is, it's an upload and qualifies as an attachment
+    // Check to see whether this is a known "attachment" URL.
 
-    $localfile = File::staticGet('url', $longurl);
+    $f = File::staticGet('url', $longurl);
 
-    if (!empty($localfile)) {
-        if (isset($localfile->filename)) {
-            $is_attachment = true;
-            $attachment_id = $localfile->id;
-        }
+    if (empty($f)) {
+        // XXX: this writes to the database. :<
+        $f = File::processNew($longurl);
     }
 
-    // if this URL is an attachment, then we set class='attachment' and id='attahcment-ID'
-    // where ID is the id of the attachment for the given URL.
-    //
-    // we need a better test telling what can be shown as an attachment
-    // we're currently picking up oembeds only.
-    // I think the best option is another file_view table in the db
-    // and associated dbobject.
-
-    $query = "select file_oembed.file_id as file_id from file join file_oembed on file.id = file_oembed.file_id where file.url='$longurl'";
-    $file = new File;
-    $file->query($query);
-    $file->fetch();
-
-    if (!empty($file->file_id)) {
-        $is_attachment = true;
-        $attachment_id = $file->file_id;
-
-        $query = "select file_thumbnail.file_id as file_id from file join file_thumbnail on file.id = file_thumbnail.file_id where file.url='$longurl'";
-        $file2 = new File;
-        $file2->query($query);
-        $file2->fetch();
-
-        if (!empty($file2)) {
-            $has_thumb = true;
+    if (!empty($f)) {
+        if (isset($f->filename)) {
+            $is_attachment = true;
+            $attachment_id = $f->id;
+        } else { // if it has OEmbed info, it's an attachment, too
+            $foe = File_oembed::staticGet('file_id', $f->id);
+            if (!empty($foe)) {
+                $is_attachment = true;
+                $attachment_id = $f->id;
+
+                $thumb = File_thumbnail::staticGet('file_id', $f->id);
+                if (!empty($thumb)) {
+                    $has_thumb = true;
+                }
+            }
         }
     }