diff options
Diffstat (limited to 'plugins/TwitterBridge/daemons')
| -rwxr-xr-x | plugins/TwitterBridge/daemons/synctwitterfriends.php | 12 | ||||
| -rwxr-xr-x | plugins/TwitterBridge/daemons/twitterstatusfetcher.php | 274 |
2 files changed, 222 insertions, 64 deletions
diff --git a/plugins/TwitterBridge/daemons/synctwitterfriends.php b/plugins/TwitterBridge/daemons/synctwitterfriends.php index df7da0943..38a8b89eb 100755 --- a/plugins/TwitterBridge/daemons/synctwitterfriends.php +++ b/plugins/TwitterBridge/daemons/synctwitterfriends.php @@ -33,7 +33,6 @@ END_OF_TRIM_HELP; require_once INSTALLDIR . '/scripts/commandline.inc'; require_once INSTALLDIR . '/lib/parallelizingdaemon.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitter.php'; -require_once INSTALLDIR . '/plugins/TwitterBridge/twitterbasicauthclient.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; /** @@ -46,7 +45,6 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @link http://status.net/ */ - class SyncTwitterFriendsDaemon extends ParallelizingDaemon { /** @@ -60,7 +58,6 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon * @return void * **/ - function __construct($id = null, $interval = 60, $max_children = 2, $debug = null) { @@ -72,7 +69,6 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon * * @return string Name of the daemon. */ - function name() { return ('synctwitterfriends.' . $this->_id); @@ -111,12 +107,10 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon } function childTask($flink) { - // Each child ps needs its own DB connection // Note: DataObject::getDatabaseConnection() creates // a new connection if there isn't one already - $conn = &$flink->getDatabaseConnection(); $this->subscribeTwitterFriends($flink); @@ -128,7 +122,6 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon // XXX: Couldn't find a less brutal way to blow // away a cached connection - global $_DB_DATAOBJECT; unset($_DB_DATAOBJECT['CONNECTIONS']); } @@ -144,8 +137,8 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon $client = new TwitterOAuthClient($token->key, $token->secret); common_debug($this->name() . '- Grabbing friends IDs with OAuth.'); } else { - $client = new TwitterBasicAuthClient($flink); - common_debug($this->name() . '- Grabbing friends IDs with basic auth.'); + common_debug("Skipping Twitter friends for {$flink->user_id} since not OAuth."); + return $friends; } try { @@ -278,4 +271,3 @@ if (have_option('d') || have_option('debug')) { $syncer = new SyncTwitterFriendsDaemon($id, 60, 2, $debug); $syncer->runOnce(); - diff --git a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php index 03a4bd3f3..cef67b180 100755 --- a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php +++ b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php @@ -40,7 +40,6 @@ require_once INSTALLDIR . '/scripts/commandline.inc'; require_once INSTALLDIR . '/lib/common.php'; require_once INSTALLDIR . '/lib/daemon.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitter.php'; -require_once INSTALLDIR . '/plugins/TwitterBridge/twitterbasicauthclient.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; /** @@ -63,7 +62,6 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @link http://status.net/ */ - class TwitterStatusFetcher extends ParallelizingDaemon { /** @@ -88,7 +86,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon * * @return string Name of the daemon. */ - function name() { return ('twitterstatusfetcher.'.$this->_id); @@ -100,11 +97,9 @@ class TwitterStatusFetcher extends ParallelizingDaemon * * @return array flinks an array of Foreign_link objects */ - function getObjects() { global $_DB_DATAOBJECT; - $flink = new Foreign_link(); $conn = &$flink->getDatabaseConnection(); @@ -135,12 +130,10 @@ class TwitterStatusFetcher extends ParallelizingDaemon } function childTask($flink) { - // Each child ps needs its own DB connection // Note: DataObject::getDatabaseConnection() creates // a new connection if there isn't one already - $conn = &$flink->getDatabaseConnection(); $this->getTimeline($flink); @@ -152,7 +145,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon // XXX: Couldn't find a less brutal way to blow // away a cached connection - global $_DB_DATAOBJECT; unset($_DB_DATAOBJECT['CONNECTIONS']); } @@ -168,10 +160,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon common_debug($this->name() . ' - Trying to get timeline for Twitter user ' . $flink->foreign_id); - // XXX: Biggest remaining issue - How do we know at which status - // to start importing? How many statuses? Right now I'm going - // with the default last 20. - $client = null; if (TwitterOAuthClient::isPackedToken($flink->credentials)) { @@ -179,14 +167,17 @@ class TwitterStatusFetcher extends ParallelizingDaemon $client = new TwitterOAuthClient($token->key, $token->secret); common_debug($this->name() . ' - Grabbing friends timeline with OAuth.'); } else { - $client = new TwitterBasicAuthClient($flink); - common_debug($this->name() . ' - Grabbing friends timeline with basic auth.'); + common_debug("Skipping friends timeline for $flink->foreign_id since not OAuth."); } $timeline = null; + $lastId = Twitter_synch_status::getLastId($flink->foreign_id, 'home_timeline'); + + common_debug("Got lastId value '{$lastId}' for foreign id '{$flink->foreign_id}' and timeline 'home_timeline'"); + try { - $timeline = $client->statusesHomeTimeline(); + $timeline = $client->statusesHomeTimeline($lastId); } catch (Exception $e) { common_log(LOG_WARNING, $this->name() . ' - Twitter client unable to get friends timeline for user ' . @@ -204,9 +195,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon // Reverse to preserve order foreach (array_reverse($timeline) as $status) { - // Hacktastic: filter out stuff coming from this StatusNet - $source = mb_strtolower(common_config('integration', 'source')); if (preg_match("/$source/", mb_strtolower($status->source))) { @@ -215,41 +204,82 @@ class TwitterStatusFetcher extends ParallelizingDaemon continue; } - $this->saveStatus($status, $flink); + // Don't save it if the user is protected + // FIXME: save it but treat it as private + if ($status->user->protected) { + continue; + } + + $notice = $this->saveStatus($status); + + if (!empty($notice)) { + Inbox::insertNotice($flink->user_id, $notice->id); + } } - // Okay, record the time we synced with Twitter for posterity + if (!empty($timeline)) { + Twitter_synch_status::setLastId($flink->foreign_id, 'home_timeline', $timeline[0]->id); + common_debug("Set lastId value '{$timeline[0]->id}' for foreign id '{$flink->foreign_id}' and timeline 'home_timeline'"); + } + // Okay, record the time we synced with Twitter for posterity $flink->last_noticesync = common_sql_now(); $flink->update(); } - function saveStatus($status, $flink) + function saveStatus($status) { $profile = $this->ensureProfile($status->user); if (empty($profile)) { common_log(LOG_ERR, $this->name() . ' - Problem saving notice. No associated Profile.'); - return; + return null; } - $statusUri = 'http://twitter.com/' - . $status->user->screen_name - . '/status/' - . $status->id; + $statusUri = $this->makeStatusURI($status->user->screen_name, $status->id); // check to see if we've already imported the status + $n2s = Notice_to_status::staticGet('status_id', $status->id); - $dupe = $this->checkDupe($profile, $statusUri); - - if (!empty($dupe)) { + if (!empty($n2s)) { common_log( LOG_INFO, $this->name() . - " - Ignoring duplicate import: $statusUri" + " - Ignoring duplicate import: {$status->id}" ); - return; + return Notice::staticGet('id', $n2s->notice_id); + } + + // If it's a retweet, save it as a repeat! + if (!empty($status->retweeted_status)) { + common_log(LOG_INFO, "Status {$status->id} is a retweet of {$status->retweeted_status->id}."); + $original = $this->saveStatus($status->retweeted_status); + if (empty($original)) { + return null; + } else { + $author = $original->getProfile(); + // TRANS: Message used to repeat a notice. RT is the abbreviation of 'retweet'. + // TRANS: %1$s is the repeated user's name, %2$s is the repeated notice. + $content = sprintf(_m('RT @%1$s %2$s'), + $author->nickname, + $original->content); + + if (Notice::contentTooLong($content)) { + $contentlimit = Notice::maxContent(); + $content = mb_substr($content, 0, $contentlimit - 4) . ' ...'; + } + + $repeat = Notice::saveNew($profile->id, + $content, + 'twitter', + array('repeat_of' => $original->id, + 'uri' => $statusUri, + 'is_local' => Notice::GATEWAY)); + common_log(LOG_INFO, "Saved {$repeat->id} as a repeat of {$original->id}"); + Notice_to_status::saveNew($repeat->id, $status->id); + return $repeat; + } } $notice = new Notice(); @@ -263,14 +293,36 @@ class TwitterStatusFetcher extends ParallelizingDaemon ); $notice->source = 'twitter'; + $notice->reply_to = null; + + if (!empty($status->in_reply_to_status_id)) { + common_log(LOG_INFO, "Status {$status->id} is a reply to status {$status->in_reply_to_status_id}"); + $n2s = Notice_to_status::staticGet('status_id', $status->in_reply_to_status_id); + if (empty($n2s)) { + common_log(LOG_INFO, "Couldn't find local notice for status {$status->in_reply_to_status_id}"); + } else { + $reply = Notice::staticGet('id', $n2s->notice_id); + if (empty($reply)) { + common_log(LOG_INFO, "Couldn't find local notice for status {$status->in_reply_to_status_id}"); + } else { + common_log(LOG_INFO, "Found local notice {$reply->id} for status {$status->in_reply_to_status_id}"); + $notice->reply_to = $reply->id; + $notice->conversation = $reply->conversation; + } + } + } + + if (empty($notice->conversation)) { + $conv = Conversation::create(); + $notice->conversation = $conv->id; + common_log(LOG_INFO, "No known conversation for status {$status->id} so making a new one {$conv->id}."); + } + $notice->is_local = Notice::GATEWAY; - $notice->content = common_shorten_links($status->text); - $notice->rendered = common_render_content( - $notice->content, - $notice - ); + $notice->content = html_entity_decode($status->text, ENT_QUOTES, 'UTF-8'); + $notice->rendered = $this->linkify($status); if (Event::handle('StartNoticeSave', array(&$notice))) { @@ -285,24 +337,31 @@ class TwitterStatusFetcher extends ParallelizingDaemon Event::handle('EndNoticeSave', array($notice)); } - $orig = clone($notice); - $conv = Conversation::create(); + Notice_to_status::saveNew($notice->id, $status->id); - $notice->conversation = $conv->id; + $this->saveStatusMentions($notice, $status); - if (!$notice->update($orig)) { - common_log_db_error($notice, 'UPDATE', __FILE__); - common_log(LOG_ERR, $this->name() . - ' - Problem saving notice.'); - } - - Inbox::insertNotice($flink->user_id, $notice->id); $notice->blowOnInsert(); return $notice; } /** + * Make an URI for a status. + * + * @param object $status status object + * + * @return string URI + */ + function makeStatusURI($username, $id) + { + return 'http://twitter.com/' + . $username + . '/status/' + . $id; + } + + /** * Look up a Profile by profileurl field. Profile::staticGet() was * not working consistently. * @@ -311,7 +370,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon * * @return mixed value the first Profile with that url, or null */ - function getProfileByUrl($nickname, $profileurl) { $profile = new Profile(); @@ -335,7 +393,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon * * @return mixed value a matching Notice or null */ - function checkDupe($profile, $statusUri) { $notice = new Notice(); @@ -354,7 +411,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon function ensureProfile($user) { // check to see if there's already a profile for this user - $profileurl = 'http://twitter.com/' . $user->screen_name; $profile = $this->getProfileByUrl($user->screen_name, $profileurl); @@ -368,7 +424,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon return $profile; } else { - common_debug($this->name() . ' - Adding profile and remote profile ' . "for Twitter user: $profileurl."); @@ -400,7 +455,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon $remote_pro = Remote_profile::staticGet('uri', $profileurl); if (empty($remote_pro)) { - $remote_pro = new Remote_profile(); $remote_pro->id = $id; @@ -547,7 +601,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon $avatar = $profile->getAvatar($sizes[$size]); // Delete the avatar, if present - if ($avatar) { $avatar->delete(); } @@ -572,10 +625,8 @@ class TwitterStatusFetcher extends ParallelizingDaemon $avatar->height = 48; break; default: - // Note: Twitter's big avatars are a different size than // StatusNet's (StatusNet's = 96) - $avatar->width = 73; $avatar->height = 73; } @@ -631,6 +682,122 @@ class TwitterStatusFetcher extends ParallelizingDaemon return true; } + + const URL = 1; + const HASHTAG = 2; + const MENTION = 3; + + function linkify($status) + { + $text = $status->text; + + if (empty($status->entities)) { + common_log(LOG_WARNING, "No entities data for {$status->id}; trying to fake up links ourselves."); + $text = common_replace_urls_callback($text, 'common_linkify'); + $text = preg_replace('/(^|\"\;|\'|\(|\[|\{|\s+)#([\pL\pN_\-\.]{1,64})/e', "'\\1#'.TwitterStatusFetcher::tagLink('\\2')", $text); + $text = preg_replace('/(^|\s+)@([a-z0-9A-Z_]{1,64})/e', "'\\1@'.TwitterStatusFetcher::atLink('\\2')", $text); + return $text; + } + + // Move all the entities into order so we can + // replace them in reverse order and thus + // not mess up their indices + + $toReplace = array(); + + if (!empty($status->entities->urls)) { + foreach ($status->entities->urls as $url) { + $toReplace[$url->indices[0]] = array(self::URL, $url); + } + } + + if (!empty($status->entities->hashtags)) { + foreach ($status->entities->hashtags as $hashtag) { + $toReplace[$hashtag->indices[0]] = array(self::HASHTAG, $hashtag); + } + } + + if (!empty($status->entities->user_mentions)) { + foreach ($status->entities->user_mentions as $mention) { + $toReplace[$mention->indices[0]] = array(self::MENTION, $mention); + } + } + + // sort in reverse order by key + + krsort($toReplace); + + foreach ($toReplace as $part) { + list($type, $object) = $part; + switch($type) { + case self::URL: + $linkText = $this->makeUrlLink($object); + break; + case self::HASHTAG: + $linkText = $this->makeHashtagLink($object); + break; + case self::MENTION: + $linkText = $this->makeMentionLink($object); + break; + default: + continue; + } + $text = mb_substr($text, 0, $object->indices[0]) . $linkText . mb_substr($text, $object->indices[1]); + } + return $text; + } + + function makeUrlLink($object) + { + return "<a href='{$object->url}' class='extlink'>{$object->url}</a>"; + } + + function makeHashtagLink($object) + { + return "#" . self::tagLink($object->text); + } + + function makeMentionLink($object) + { + return "@".self::atLink($object->screen_name, $object->name); + } + + static function tagLink($tag) + { + return "<a href='https://twitter.com/search?q=%23{$tag}' class='hashtag'>{$tag}</a>"; + } + + static function atLink($screenName, $fullName=null) + { + if (!empty($fullName)) { + return "<a href='http://twitter.com/{$screenName}' title='{$fullName}'>{$screenName}</a>"; + } else { + return "<a href='http://twitter.com/{$screenName}'>{$screenName}</a>"; + } + } + + function saveStatusMentions($notice, $status) + { + $mentions = array(); + + if (empty($status->entities) || empty($status->entities->user_mentions)) { + return; + } + + foreach ($status->entities->user_mentions as $mention) { + $flink = Foreign_link::getByForeignID($mention->id, TWITTER_SERVICE); + if (!empty($flink)) { + $user = User::staticGet('id', $flink->user_id); + if (!empty($user)) { + $reply = new Reply(); + $reply->notice_id = $notice->id; + $reply->profile_id = $user->id; + common_log(LOG_INFO, __METHOD__ . ": saving reply: notice {$notice->id} to profile {$user->id}"); + $id = $reply->insert(); + } + } + } + } } $id = null; @@ -652,4 +819,3 @@ if (have_option('d') || have_option('debug')) { $fetcher = new TwitterStatusFetcher($id, 60, 2, $debug); $fetcher->runOnce(); - |