From fe9fc152861a0131582c4aa512870d2d01bccb57 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 4 Aug 2009 02:21:18 +0000 Subject: Make TwitterStatusFetcher daemon work with OAuth --- scripts/twitterstatusfetcher.php | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'scripts/twitterstatusfetcher.php') diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index e1745cfc0..d9f035fa6 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -191,7 +191,7 @@ class TwitterStatusFetcher extends Daemon { $flink = new Foreign_link(); - $flink->service = 1; // Twitter + $flink->service = TWITTER_SERVICE; $flink->orderBy('last_noticesync'); @@ -241,35 +241,33 @@ class TwitterStatusFetcher extends Daemon function getTimeline($flink) { - if (empty($flink)) { + if (empty($flink)) { common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); return; } - $fuser = $flink->getForeignUser(); - - if (empty($fuser)) { - common_log(LOG_WARNING, "Unmatched user for ID " . - $flink->user_id); - return; - } - if (defined('SCRIPT_DEBUG')) { common_debug('Trying to get timeline for Twitter user ' . - "$fuser->nickname ($flink->foreign_id)."); + $flink->foreign_id); } // XXX: Biggest remaining issue - How do we know at which status // to start importing? How many statuses? Right now I'm going // with the default last 20. - $url = 'http://twitter.com/statuses/friends_timeline.json'; + $client = new TwitterOAuthClient($flink->token, $flink->credentials); - $timeline_json = get_twitter_data($url, $fuser->nickname, - $flink->credentials); + $timeline = null; - $timeline = json_decode($timeline_json); + try { + $timeline = $client->statuses_friends_timeline(); + } catch (OAuthClientCurlException $e) { + common_log(LOG_WARNING, + 'OAuth client unable to get friends timeline for user ' . + $flink->user_id . ' - code: ' . + $e->getCode() . 'msg: ' . $e->getMessage()); + } if (empty($timeline)) { common_log(LOG_WARNING, "Empty timeline."); @@ -303,7 +301,7 @@ class TwitterStatusFetcher extends Daemon $id = $this->ensureProfile($status->user); $profile = Profile::staticGet($id); - if (!$profile) { + if (empty($profile)) { common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); return null; @@ -318,7 +316,7 @@ class TwitterStatusFetcher extends Daemon // check to see if we've already imported the status - if (!$notice) { + if (empty($notice)) { $notice = new Notice(); -- cgit v1.2.3-54-g00ecf From 5576917662b47ef27bf7ba3e551411c3d3b5e26c Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 4 Aug 2009 17:20:28 +0000 Subject: Use empty() for checking whether DB_DataObject returned something --- scripts/twitterstatusfetcher.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts/twitterstatusfetcher.php') diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index d9f035fa6..dd139417c 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -356,7 +356,7 @@ class TwitterStatusFetcher extends Daemon $profileurl = 'http://twitter.com/' . $user->screen_name; $profile = Profile::staticGet('profileurl', $profileurl); - if ($profile) { + if (!empty($profile) { if (defined('SCRIPT_DEBUG')) { common_debug("Profile for $profile->nickname found."); } @@ -394,7 +394,7 @@ class TwitterStatusFetcher extends Daemon // check for remote profile $remote_pro = Remote_profile::staticGet('uri', $profileurl); - if (!$remote_pro) { + if (empty($remote_pro)) { $remote_pro = new Remote_profile(); -- cgit v1.2.3-54-g00ecf From cd71f9cc51f77b6b8e3b11c34cb7523f36339cea Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Wed, 5 Aug 2009 00:16:12 +0000 Subject: Better (hopefully) database connection management for child processes --- scripts/twitterstatusfetcher.php | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) (limited to 'scripts/twitterstatusfetcher.php') diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index dd139417c..67f52a3cc 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -99,13 +99,6 @@ class TwitterStatusFetcher extends Daemon foreach ($flinks as $f) { - // We have to disconnect from the DB before forking so - // each sub-process will open its own connection and - // avoid stomping on the others - - $conn = &$f->getDatabaseConnection(); - $conn->disconnect(); - $pid = pcntl_fork(); if ($pid == -1) { @@ -125,7 +118,24 @@ class TwitterStatusFetcher extends Daemon } else { // Child + + // Each child ps needs its own DB connection + + // Note: DataObject::getDatabaseConnection() creates + // a new connection if there isn't one already + + global $_DB_DATAOBJECT; + $conn = &$f->getDatabaseConnection(); + $this->getTimeline($f); + + $conn->disconnect(); + + // XXX: Couldn't find a less brutal way to blow + // away a cached connection + + unset($_DB_DATAOBJECT['CONNECTIONS']); + exit(); } @@ -189,7 +199,10 @@ class TwitterStatusFetcher extends Daemon function refreshFlinks() { + global $_DB_DATAOBJECT; + $flink = new Foreign_link(); + $conn = &$flink->getDatabaseConnection(); $flink->service = TWITTER_SERVICE; @@ -215,6 +228,9 @@ class TwitterStatusFetcher extends Daemon $flink->free(); unset($flink); + $conn->disconnect(); + unset($_DB_DATAOBJECT['CONNECTIONS']); + return $flinks; } @@ -299,6 +315,7 @@ class TwitterStatusFetcher extends Daemon function saveStatus($status, $flink) { $id = $this->ensureProfile($status->user); + $profile = Profile::staticGet($id); if (empty($profile)) { @@ -356,7 +373,7 @@ class TwitterStatusFetcher extends Daemon $profileurl = 'http://twitter.com/' . $user->screen_name; $profile = Profile::staticGet('profileurl', $profileurl); - if (!empty($profile) { + if (!empty($profile)) { if (defined('SCRIPT_DEBUG')) { common_debug("Profile for $profile->nickname found."); } -- cgit v1.2.3-54-g00ecf From c03d5932877c15eb673febabda5227df2173fbad Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 6 Aug 2009 22:52:58 +0000 Subject: Make TwitterStatusFetcher extend ParallelizingDaemon --- lib/parallelizingdaemon.php | 12 +- scripts/synctwitterfriends.php | 20 ++- scripts/twitterstatusfetcher.php | 289 ++++++++++++--------------------------- 3 files changed, 110 insertions(+), 211 deletions(-) (limited to 'scripts/twitterstatusfetcher.php') diff --git a/lib/parallelizingdaemon.php b/lib/parallelizingdaemon.php index 5ecfd98f3..dc28b5643 100644 --- a/lib/parallelizingdaemon.php +++ b/lib/parallelizingdaemon.php @@ -87,7 +87,7 @@ class ParallelizingDaemon extends Daemon function run() { if (isset($this->_debug)) { - echo $this->name() . " - debugging output enabled.\n"; + echo $this->name() . " - Debugging output enabled.\n"; } do { @@ -107,9 +107,10 @@ class ParallelizingDaemon extends Daemon if ($pid) { // Parent + if (isset($this->_debug)) { echo $this->name() . - " (parent) forked new child - pid $pid.\n"; + " - Forked new child - pid $pid.\n"; } @@ -120,22 +121,25 @@ class ParallelizingDaemon extends Daemon // Child // Do something with each object + $this->childTask($o); exit(); } // Remove child from ps list as it finishes + while (($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { if (isset($this->_debug)) { - echo $this->name() . " child $c finished.\n"; + echo $this->name() . " - Child $c finished.\n"; } $this->removePs($this->_children, $c); } // Wait! We have too many damn kids. + if (sizeof($this->_children) >= $this->_max_children) { if (isset($this->_debug)) { @@ -158,7 +162,7 @@ class ParallelizingDaemon extends Daemon while (($c = pcntl_wait($status, WUNTRACED)) > 0) { if (isset($this->_debug)) { - echo $this->name() . " child $c finished.\n"; + echo $this->name() . " - Child $c finished.\n"; } $this->removePs($this->_children, $c); diff --git a/scripts/synctwitterfriends.php b/scripts/synctwitterfriends.php index 1bd75bac1..37f7842a1 100755 --- a/scripts/synctwitterfriends.php +++ b/scripts/synctwitterfriends.php @@ -54,6 +54,18 @@ require_once INSTALLDIR . '/lib/parallelizingdaemon.php'; class SyncTwitterFriendsDaemon extends ParallelizingDaemon { + /** + * Constructor + * + * @param string $id the name/id of this daemon + * @param int $interval sleep this long before doing everything again + * @param int $max_children maximum number of child processes at a time + * @param boolean $debug debug output flag + * + * @return void + * + **/ + function __construct($id = null, $interval = 60, $max_children = 2, $debug = null) { @@ -71,6 +83,12 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon return ('synctwitterfriendsdaemon.' . $this->_id); } + /** + * Find all the Twitter foreign links for users who have requested + * automatically subscribing to their Twitter friends locally. + * + * @return array flinks an array of Foreign_link objects + */ function getObjects() { $flinks = array(); @@ -237,8 +255,6 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon } -declare(ticks = 1); - $id = null; $debug = null; diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index 67f52a3cc..fa37f894c 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -56,17 +56,23 @@ require_once INSTALLDIR . '/lib/daemon.php'; // NOTE: an Avatar path MUST be set in config.php for this // script to work: e.g.: $config['avatar']['path'] = '/laconica/avatar'; -class TwitterStatusFetcher extends Daemon +class TwitterStatusFetcher extends ParallelizingDaemon { - private $_children = array(); - - function __construct($id=null, $daemonize=true) + /** + * Constructor + * + * @param string $id the name/id of this daemon + * @param int $interval sleep this long before doing everything again + * @param int $max_children maximum number of child processes at a time + * @param boolean $debug debug output flag + * + * @return void + * + **/ + function __construct($id = null, $interval = 60, + $max_children = 2, $debug = null) { - parent::__construct($daemonize); - - if ($id) { - $this->set_id($id); - } + parent::__construct($id, $interval, $max_children, $debug); } /** @@ -81,123 +87,13 @@ class TwitterStatusFetcher extends Daemon } /** - * Run the daemon - * - * @return void - */ - - function run() - { - if (defined('SCRIPT_DEBUG')) { - common_debug($this->name() . - ': debugging log output enabled.'); - } - - do { - - $flinks = $this->refreshFlinks(); - - foreach ($flinks as $f) { - - $pid = pcntl_fork(); - - if ($pid == -1) { - die ("Couldn't fork!"); - } - - if ($pid) { - - // Parent - if (defined('SCRIPT_DEBUG')) { - common_debug("Parent: forked new status ". - " fetcher process " . $pid); - } - - $this->_children[] = $pid; - - } else { - - // Child - - // Each child ps needs its own DB connection - - // Note: DataObject::getDatabaseConnection() creates - // a new connection if there isn't one already - - global $_DB_DATAOBJECT; - $conn = &$f->getDatabaseConnection(); - - $this->getTimeline($f); - - $conn->disconnect(); - - // XXX: Couldn't find a less brutal way to blow - // away a cached connection - - unset($_DB_DATAOBJECT['CONNECTIONS']); - - exit(); - } - - // Remove child from ps list as it finishes - while (($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { - - if (defined('SCRIPT_DEBUG')) { - common_debug("Child $c finished."); - } - - $this->removePs($this->_children, $c); - } - - // Wait! We have too many damn kids. - if (sizeof($this->_children) > MAXCHILDREN) { - - if (defined('SCRIPT_DEBUG')) { - common_debug('Too many children. Waiting...'); - } - - if (($c = pcntl_wait($status, WUNTRACED)) > 0) { - - if (defined('SCRIPT_DEBUG')) { - common_debug("Finished waiting for $c"); - } - - $this->removePs($this->_children, $c); - } - } - } - - // Remove all children from the process list before restarting - while (($c = pcntl_wait($status, WUNTRACED)) > 0) { - - if (defined('SCRIPT_DEBUG')) { - common_debug("Child $c finished."); - } - - $this->removePs($this->_children, $c); - } - - // Rest for a bit before we fetch more statuses - - if (defined('SCRIPT_DEBUG')) { - common_debug('Waiting ' . POLL_INTERVAL . - ' secs before hitting Twitter again.'); - } - - if (POLL_INTERVAL > 0) { - sleep(POLL_INTERVAL); - } - - } while (true); - } - - /** - * Refresh the foreign links for this user + * Find all the Twitter foreign links for users who have requested + * importing of their friends' timelines * - * @return void + * @return array flinks an array of Foreign_link objects */ - function refreshFlinks() + function getObjects() { global $_DB_DATAOBJECT; @@ -205,15 +101,8 @@ class TwitterStatusFetcher extends Daemon $conn = &$flink->getDatabaseConnection(); $flink->service = TWITTER_SERVICE; - $flink->orderBy('last_noticesync'); - - $cnt = $flink->find(); - - if (defined('SCRIPT_DEBUG')) { - common_debug('Updating Twitter friends subscriptions' . - " for $cnt users."); - } + $flink->find(); $flinks = array(); @@ -234,39 +123,39 @@ class TwitterStatusFetcher extends Daemon return $flinks; } - /** - * Unknown - * - * @param array &$plist unknown. - * @param string $ps unknown. - * - * @return unknown - * @todo document - */ + function childTask($flink) { - function removePs(&$plist, $ps) - { - for ($i = 0; $i < sizeof($plist); $i++) { - if ($plist[$i] == $ps) { - unset($plist[$i]); - $plist = array_values($plist); - break; - } - } + // Each child ps needs its own DB connection + + // Note: DataObject::getDatabaseConnection() creates + // a new connection if there isn't one already + + $conn = &$flink->getDatabaseConnection(); + + $this->getTimeline($flink); + + $flink->last_friendsync = common_sql_now(); + $flink->update(); + + $conn->disconnect(); + + // XXX: Couldn't find a less brutal way to blow + // away a cached connection + + global $_DB_DATAOBJECT; + unset($_DB_DATAOBJECT['CONNECTIONS']); } function getTimeline($flink) { if (empty($flink)) { - common_log(LOG_WARNING, - "Can't retrieve Foreign_link for foreign ID $fid"); + common_log(LOG_WARNING, $this->name() . + " - Can't retrieve Foreign_link for foreign ID $fid"); return; } - if (defined('SCRIPT_DEBUG')) { - common_debug('Trying to get timeline for Twitter user ' . - $flink->foreign_id); - } + common_debug($this->name() . ' - Trying to get timeline for Twitter user ' . + $flink->foreign_id); // XXX: Biggest remaining issue - How do we know at which status // to start importing? How many statuses? Right now I'm going @@ -279,28 +168,28 @@ class TwitterStatusFetcher extends Daemon try { $timeline = $client->statuses_friends_timeline(); } catch (OAuthClientCurlException $e) { - common_log(LOG_WARNING, - 'OAuth client unable to get friends timeline for user ' . + common_log(LOG_WARNING, $this->name() . + ' - OAuth client unable to get friends timeline for user ' . $flink->user_id . ' - code: ' . $e->getCode() . 'msg: ' . $e->getMessage()); } if (empty($timeline)) { - common_log(LOG_WARNING, "Empty timeline."); + common_log(LOG_WARNING, $this->name . " - Empty timeline."); return; } // Reverse to preserve order + foreach (array_reverse($timeline) as $status) { // Hacktastic: filter out stuff coming from this Laconica + $source = mb_strtolower(common_config('integration', 'source')); if (preg_match("/$source/", mb_strtolower($status->source))) { - if (defined('SCRIPT_DEBUG')) { - common_debug('Skipping import of status ' . $status->id . - ' with source ' . $source); - } + common_debug($this->name() . ' - Skipping import of status ' . + $status->id . ' with source ' . $source); continue; } @@ -308,6 +197,7 @@ class TwitterStatusFetcher extends Daemon } // Okay, record the time we synced with Twitter for posterity + $flink->last_noticesync = common_sql_now(); $flink->update(); } @@ -319,8 +209,8 @@ class TwitterStatusFetcher extends Daemon $profile = Profile::staticGet($id); if (empty($profile)) { - common_log(LOG_ERR, - 'Problem saving notice. No associated Profile.'); + common_log(LOG_ERR, $this->name() . + ' - Problem saving notice. No associated Profile.'); return null; } @@ -344,7 +234,7 @@ class TwitterStatusFetcher extends Daemon $notice->content = common_shorten_links($status->text); // XXX $notice->rendered = common_render_content($notice->content, $notice); $notice->source = 'twitter'; - $notice->reply_to = null; // XXX lookup reply + $notice->reply_to = null; // XXX: lookup reply $notice->is_local = Notice::GATEWAY; if (Event::handle('StartNoticeSave', array(&$notice))) { @@ -370,24 +260,22 @@ class TwitterStatusFetcher extends Daemon function ensureProfile($user) { // check to see if there's already a profile for this user + $profileurl = 'http://twitter.com/' . $user->screen_name; $profile = Profile::staticGet('profileurl', $profileurl); if (!empty($profile)) { - if (defined('SCRIPT_DEBUG')) { - common_debug("Profile for $profile->nickname found."); - } + common_debug($this->name() . + " - Profile for $profile->nickname found."); // Check to see if the user's Avatar has changed - $this->checkAvatar($user, $profile); + $this->checkAvatar($user, $profile); return $profile->id; } else { - if (defined('SCRIPT_DEBUG')) { - common_debug('Adding profile and remote profile ' . - "for Twitter user: $profileurl"); - } + common_debug($this->name() . ' - Adding profile and remote profile ' . + "for Twitter user: $profileurl."); $profile = new Profile(); $profile->query("BEGIN"); @@ -409,6 +297,7 @@ class TwitterStatusFetcher extends Daemon } // check for remote profile + $remote_pro = Remote_profile::staticGet('uri', $profileurl); if (empty($remote_pro)) { @@ -448,23 +337,18 @@ class TwitterStatusFetcher extends Daemon $oldname = $profile->getAvatar(48)->filename; if ($newname != $oldname) { - - if (defined('SCRIPT_DEBUG')) { - common_debug('Avatar for Twitter user ' . - "$profile->nickname has changed."); - common_debug("old: $oldname new: $newname"); - } + common_debug($this->name() . ' - Avatar for Twitter user ' . + "$profile->nickname has changed."); + common_debug($this->name() . " - old: $oldname new: $newname"); $this->updateAvatars($twitter_user, $profile); } if ($this->missingAvatarFile($profile)) { - - if (defined('SCRIPT_DEBUG')) { - common_debug('Twitter user ' . $profile->nickname . - ' is missing one or more local avatars.'); - common_debug("old: $oldname new: $newname"); - } + common_debug($this->name() . ' - Twitter user ' . + $profile->nickname . + ' is missing one or more local avatars.'); + common_debug($this->name() ." - old: $oldname new: $newname"); $this->updateAvatars($twitter_user, $profile); } @@ -544,23 +428,20 @@ class TwitterStatusFetcher extends Daemon if ($this->fetchAvatar($url, $filename)) { $this->newAvatar($id, $size, $mediatype, $filename); } else { - common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); + common_log(LOG_WARNING, $this->id() . + " - Problem fetching Avatar: $url"); } } } function updateAvatar($profile_id, $size, $mediatype, $filename) { - if (defined('SCRIPT_DEBUG')) { - common_debug("Updating avatar: $size"); - } + common_debug($this->name() . " - Updating avatar: $size"); $profile = Profile::staticGet($profile_id); if (empty($profile)) { - if (defined('SCRIPT_DEBUG')) { - common_debug("Couldn't get profile: $profile_id!"); - } + common_debug($this->name() . " - Couldn't get profile: $profile_id!"); return; } @@ -568,6 +449,7 @@ class TwitterStatusFetcher extends Daemon $avatar = $profile->getAvatar($sizes[$size]); // Delete the avatar, if present + if ($avatar) { $avatar->delete(); } @@ -605,9 +487,7 @@ class TwitterStatusFetcher extends Daemon $avatar->filename = $filename; $avatar->url = Avatar::url($filename); - if (defined('SCRIPT_DEBUG')) { - common_debug("new filename: $avatar->url"); - } + common_debug($this->name() . " - New filename: $avatar->url"); $avatar->created = common_sql_now(); @@ -618,9 +498,8 @@ class TwitterStatusFetcher extends Daemon return null; } - if (defined('SCRIPT_DEBUG')) { - common_debug("Saved new $size avatar for $profile_id."); - } + common_debug($this->name() . + " - Saved new $size avatar for $profile_id."); return $id; } @@ -633,13 +512,12 @@ class TwitterStatusFetcher extends Daemon $out = fopen($avatarfile, 'wb'); if (!$out) { - common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); + common_log(LOG_WARNING, $this->name() . + " - Couldn't open file $filename"); return false; } - if (defined('SCRIPT_DEBUG')) { - common_debug("Fetching avatar: $url"); - } + common_debug($this->name() . " - Fetching Twitter avatar: $url"); $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); @@ -656,7 +534,8 @@ class TwitterStatusFetcher extends Daemon } } -declare(ticks = 1); +$id = null; +$debug = null; if (have_option('i')) { $id = get_option_value('i'); @@ -669,9 +548,9 @@ if (have_option('i')) { } if (have_option('d') || have_option('debug')) { - define('SCRIPT_DEBUG', true); + $debug = true; } -$fetcher = new TwitterStatusFetcher($id); +$fetcher = new TwitterStatusFetcher($id, 60, 2, $debug); $fetcher->runOnce(); -- cgit v1.2.3-54-g00ecf From fa8433308f5ba1209ec490d6c0835d28da18eacb Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 10 Aug 2009 06:05:43 +0000 Subject: Moved some stuff around. More comments and phpcs compliance. --- actions/twitterauthorization.php | 10 +-- lib/oauthclient.php | 144 ++++++++++++++++++++++++++++++++++---- lib/twitter.php | 2 +- lib/twitteroauthclient.php | 146 +++++++++++++++++++++++++++++++++------ scripts/synctwitterfriends.php | 4 +- scripts/twitterstatusfetcher.php | 4 +- 6 files changed, 264 insertions(+), 46 deletions(-) (limited to 'scripts/twitterstatusfetcher.php') diff --git a/actions/twitterauthorization.php b/actions/twitterauthorization.php index cf27d69cf..c40f27164 100644 --- a/actions/twitterauthorization.php +++ b/actions/twitterauthorization.php @@ -48,7 +48,6 @@ if (!defined('LACONICA')) { */ class TwitterauthorizationAction extends Action { - /** * Initialize class members. Looks for 'oauth_token' parameter. * @@ -114,12 +113,13 @@ class TwitterauthorizationAction extends Action // Get a new request token and authorize it $client = new TwitterOAuthClient(); - $req_tok = $client->getRequestToken(); + $req_tok = + $client->getRequestToken(TwitterOAuthClient::$requestTokenURL); // Sock the request token away in the session temporarily $_SESSION['twitter_request_token'] = $req_tok->key; - $_SESSION['twitter_request_token_secret'] = $req_tok->key; + $_SESSION['twitter_request_token_secret'] = $req_tok->secret; $auth_link = $client->getAuthorizeLink($req_tok); @@ -155,12 +155,12 @@ class TwitterauthorizationAction extends Action // Exchange the request token for an access token - $atok = $client->getAccessToken(); + $atok = $client->getAccessToken(TwitterOAuthClient::$accessTokenURL); // Test the access token and get the user's Twitter info $client = new TwitterOAuthClient($atok->key, $atok->secret); - $twitter_user = $client->verify_credentials(); + $twitter_user = $client->verifyCredentials(); } catch (OAuthClientException $e) { $msg = sprintf('OAuth client cURL error - code: %1$s, msg: %2$s', diff --git a/lib/oauthclient.php b/lib/oauthclient.php index 11de991c8..878e47091 100644 --- a/lib/oauthclient.php +++ b/lib/oauthclient.php @@ -1,54 +1,154 @@ . + * + * @category Action + * @package Laconica + * @author Zach Copley + * @copyright 2008 Control Yourself, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +if (!defined('LACONICA')) { + exit(1); +} -require_once('OAuth.php'); - -class OAuthClientCurlException extends Exception { } +require_once 'OAuth.php'; + +/** + * Exception wrapper for cURL errors + * + * @category Integration + * @package Laconica + * @author Zach Copley + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + * + */ +class OAuthClientCurlException extends Exception +{ +} +/** + * Base class for doing OAuth calls as a consumer + * + * @category Integration + * @package Laconica + * @author Zach Copley + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + * + */ class OAuthClient { var $consumer; var $token; + /** + * Constructor + * + * Can be initialized with just consumer key and secret for requesting new + * tokens or with additional request token or access token + * + * @param string $consumer_key consumer key + * @param string $consumer_secret consumer secret + * @param string $oauth_token user's token + * @param string $oauth_token_secret user's secret + * + * @return nothing + */ function __construct($consumer_key, $consumer_secret, $oauth_token = null, $oauth_token_secret = null) { $this->sha1_method = new OAuthSignatureMethod_HMAC_SHA1(); - $this->consumer = new OAuthConsumer($consumer_key, $consumer_secret); - $this->token = null; + $this->consumer = new OAuthConsumer($consumer_key, $consumer_secret); + $this->token = null; if (isset($oauth_token) && isset($oauth_token_secret)) { $this->token = new OAuthToken($oauth_token, $oauth_token_secret); } } - function getRequestToken() + /** + * Gets a request token from the given url + * + * @param string $url OAuth endpoint for grabbing request tokens + * + * @return OAuthToken $token the request token + */ + function getRequestToken($url) { - $response = $this->oAuthGet(TwitterOAuthClient::$requestTokenURL); + $response = $this->oAuthGet($url); parse_str($response); $token = new OAuthToken($oauth_token, $oauth_token_secret); return $token; } - function getAuthorizeLink($request_token, $oauth_callback = null) + /** + * Builds a link that can be redirected to in order to + * authorize a request token. + * + * @param string $url endpoint for authorizing request tokens + * @param OAuthToken $request_token the request token to be authorized + * @param string $oauth_callback optional callback url + * + * @return string $authorize_url the url to redirect to + */ + function getAuthorizeLink($url, $request_token, $oauth_callback = null) { - $url = TwitterOAuthClient::$authorizeURL . '?oauth_token=' . + $authorize_url = $url . '?oauth_token=' . $request_token->key; if (isset($oauth_callback)) { - $url .= '&oauth_callback=' . urlencode($oauth_callback); + $authorize_url .= '&oauth_callback=' . urlencode($oauth_callback); } - return $url; + common_debug("$authorize_url"); + + return $authorize_url; } - function getAccessToken() + /** + * Fetches an access token + * + * @param string $url OAuth endpoint for exchanging authorized request tokens + * for access tokens + * + * @return OAuthToken $token the access token + */ + function getAccessToken($url) { - $response = $this->oAuthPost(TwitterOAuthClient::$accessTokenURL); + $response = $this->oAuthPost($url); parse_str($response); $token = new OAuthToken($oauth_token, $oauth_token_secret); return $token; } + /** + * Use HTTP GET to make a signed OAuth request + * + * @param string $url OAuth endpoint + * + * @return mixed the request + */ function oAuthGet($url) { $request = OAuthRequest::from_consumer_and_token($this->consumer, @@ -59,6 +159,14 @@ class OAuthClient return $this->httpRequest($request->to_url()); } + /** + * Use HTTP POST to make a signed OAuth request + * + * @param string $url OAuth endpoint + * @param array $params additional post parameters + * + * @return mixed the request + */ function oAuthPost($url, $params = null) { $request = OAuthRequest::from_consumer_and_token($this->consumer, @@ -70,6 +178,14 @@ class OAuthClient $request->to_postdata()); } + /** + * Make a HTTP request using cURL. + * + * @param string $url Where to make the + * @param array $params post parameters + * + * @return mixed the request + */ function httpRequest($url, $params = null) { $options = array( @@ -89,7 +205,7 @@ class OAuthClient ); if (isset($params)) { - $options[CURLOPT_POST] = true; + $options[CURLOPT_POST] = true; $options[CURLOPT_POSTFIELDS] = $params; } diff --git a/lib/twitter.php b/lib/twitter.php index 345516997..4e2f67c66 100644 --- a/lib/twitter.php +++ b/lib/twitter.php @@ -165,7 +165,7 @@ function broadcast_twitter($notice) $status = null; try { - $status = $client->statuses_update($statustxt); + $status = $client->statusesUpdate($statustxt); } catch (OAuthClientCurlException $e) { if ($e->getMessage() == 'The requested URL returned error: 401') { diff --git a/lib/twitteroauthclient.php b/lib/twitteroauthclient.php index 2636a3833..c798ac877 100644 --- a/lib/twitteroauthclient.php +++ b/lib/twitteroauthclient.php @@ -1,11 +1,60 @@ . + * + * @category Integration + * @package Laconica + * @author Zach Copley + * @copyright 2008 Control Yourself, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +if (!defined('LACONICA')) { + exit(1); +} +/** + * Class for talking to the Twitter API with OAuth. + * + * @category Integration + * @package Laconica + * @author Zach Copley + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + * + */ class TwitterOAuthClient extends OAuthClient { public static $requestTokenURL = 'https://twitter.com/oauth/request_token'; public static $authorizeURL = 'https://twitter.com/oauth/authorize'; public static $accessTokenURL = 'https://twitter.com/oauth/access_token'; + /** + * Constructor + * + * @param string $oauth_token the user's token + * @param string $oauth_token_secret the user's token secret + * + * @return nothing + */ function __construct($oauth_token = null, $oauth_token_secret = null) { $consumer_key = common_config('twitter', 'consumer_key'); @@ -15,39 +64,72 @@ class TwitterOAuthClient extends OAuthClient $oauth_token, $oauth_token_secret); } - function getAuthorizeLink($request_token) { - return parent::getAuthorizeLink($request_token, + /** + * Builds a link to Twitter's endpoint for authorizing a request token + * + * @param OAuthToken $request_token token to authorize + * + * @return the link + */ + function getAuthorizeLink($request_token) + { + return parent::getAuthorizeLink(self::$authorizeURL, + $request_token, common_local_url('twitterauthorization')); - } - function verify_credentials() + /** + * Calls Twitter's /account/verify_credentials API method + * + * @return mixed the Twitter user + */ + function verifyCredentials() { - $url = 'https://twitter.com/account/verify_credentials.json'; - $response = $this->oAuthGet($url); + $url = 'https://twitter.com/account/verify_credentials.json'; + $response = $this->oAuthGet($url); $twitter_user = json_decode($response); return $twitter_user; } - function statuses_update($status, $in_reply_to_status_id = null) + /** + * Calls Twitter's /stutuses/update API method + * + * @param string $status text of the status + * @param int $in_reply_to_status_id optional id of the status it's + * a reply to + * + * @return mixed the status + */ + function statusesUpdate($status, $in_reply_to_status_id = null) { - $url = 'https://twitter.com/statuses/update.json'; - $params = array('status' => $status, + $url = 'https://twitter.com/statuses/update.json'; + $params = array('status' => $status, 'in_reply_to_status_id' => $in_reply_to_status_id); $response = $this->oAuthPost($url, $params); - $status = json_decode($response); + $status = json_decode($response); return $status; } - function statuses_friends_timeline($since_id = null, $max_id = null, - $cnt = null, $page = null) { + /** + * Calls Twitter's /stutuses/friends_timeline API method + * + * @param int $since_id show statuses after this id + * @param int $max_id show statuses before this id + * @param int $cnt number of statuses to show + * @param int $page page number + * + * @return mixed an array of statuses + */ + function statusesFriendsTimeline($since_id = null, $max_id = null, + $cnt = null, $page = null) + { - $url = 'https://twitter.com/statuses/friends_timeline.json'; + $url = 'https://twitter.com/statuses/friends_timeline.json'; $params = array('since_id' => $since_id, 'max_id' => $max_id, 'count' => $cnt, 'page' => $page); - $qry = http_build_query($params); + $qry = http_build_query($params); if (!empty($qry)) { $url .= "?$qry"; @@ -58,8 +140,18 @@ class TwitterOAuthClient extends OAuthClient return $statuses; } - function statuses_friends($id = null, $user_id = null, $screen_name = null, - $page = null) + /** + * Calls Twitter's /stutuses/friends API method + * + * @param int $id id of the user whom you wish to see friends of + * @param int $user_id numerical user id + * @param int $screen_name screen name + * @param int $page page number + * + * @return mixed an array of twitter users and their latest status + */ + function statusesFriends($id = null, $user_id = null, $screen_name = null, + $page = null) { $url = "https://twitter.com/statuses/friends.json"; @@ -67,18 +159,28 @@ class TwitterOAuthClient extends OAuthClient 'user_id' => $user_id, 'screen_name' => $screen_name, 'page' => $page); - $qry = http_build_query($params); + $qry = http_build_query($params); if (!empty($qry)) { $url .= "?$qry"; } $response = $this->oAuthGet($url); - $ids = json_decode($response); - return $ids; + $friends = json_decode($response); + return $friends; } - function friends_ids($id = null, $user_id = null, $screen_name = null, + /** + * Calls Twitter's /stutuses/friends/ids API method + * + * @param int $id id of the user whom you wish to see friends of + * @param int $user_id numerical user id + * @param int $screen_name screen name + * @param int $page page number + * + * @return mixed a list of ids, 100 per page + */ + function friendsIds($id = null, $user_id = null, $screen_name = null, $page = null) { $url = "https://twitter.com/friends/ids.json"; @@ -87,14 +189,14 @@ class TwitterOAuthClient extends OAuthClient 'user_id' => $user_id, 'screen_name' => $screen_name, 'page' => $page); - $qry = http_build_query($params); + $qry = http_build_query($params); if (!empty($qry)) { $url .= "?$qry"; } $response = $this->oAuthGet($url); - $ids = json_decode($response); + $ids = json_decode($response); return $ids; } diff --git a/scripts/synctwitterfriends.php b/scripts/synctwitterfriends.php index 37f7842a1..39b9ad612 100755 --- a/scripts/synctwitterfriends.php +++ b/scripts/synctwitterfriends.php @@ -145,7 +145,7 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon $client = new TwitterOAuthClient($flink->token, $flink->credentials); try { - $friends_ids = $client->friends_ids(); + $friends_ids = $client->friendsIds(); } catch (OAuthCurlException $e) { common_log(LOG_WARNING, $this->name() . ' - cURL error getting friend ids ' . @@ -174,7 +174,7 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon for ($i = 1; $i <= $pages; $i++) { try { - $more_friends = $client->statuses_friends(null, null, null, $i); + $more_friends = $client->statusesFriends(null, null, null, $i); } catch (OAuthCurlException $e) { common_log(LOG_WARNING, $this->name() . ' - cURL error getting Twitter statuses/friends ' . diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index fa37f894c..e04a8993f 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -166,7 +166,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon $timeline = null; try { - $timeline = $client->statuses_friends_timeline(); + $timeline = $client->statusesFriendsTimeline(); } catch (OAuthClientCurlException $e) { common_log(LOG_WARNING, $this->name() . ' - OAuth client unable to get friends timeline for user ' . @@ -175,7 +175,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon } if (empty($timeline)) { - common_log(LOG_WARNING, $this->name . " - Empty timeline."); + common_log(LOG_WARNING, $this->name() . " - Empty timeline."); return; } -- cgit v1.2.3-54-g00ecf From 27548c690350bdf0376d846a5e8e86477359297d Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 10 Aug 2009 07:00:59 +0000 Subject: I forgot that we don't do database upgrades for point releases. So I've changed Twitter OAuth to store token and token secret in the same field in foreign_link (credentials). This should be changed in 0.9. --- actions/twitterauthorization.php | 8 +++++--- lib/oauthclient.php | 2 -- lib/twitter.php | 4 +++- lib/twitteroauthclient.php | 17 +++++++++++++++++ scripts/synctwitterfriends.php | 4 +++- scripts/twitterstatusfetcher.php | 4 +++- 6 files changed, 31 insertions(+), 8 deletions(-) (limited to 'scripts/twitterstatusfetcher.php') diff --git a/actions/twitterauthorization.php b/actions/twitterauthorization.php index c40f27164..b04f35327 100644 --- a/actions/twitterauthorization.php +++ b/actions/twitterauthorization.php @@ -113,7 +113,7 @@ class TwitterauthorizationAction extends Action // Get a new request token and authorize it $client = new TwitterOAuthClient(); - $req_tok = + $req_tok = $client->getRequestToken(TwitterOAuthClient::$requestTokenURL); // Sock the request token away in the session temporarily @@ -198,8 +198,10 @@ class TwitterauthorizationAction extends Action $flink->user_id = $user->id; $flink->foreign_id = $twitter_user->id; $flink->service = TWITTER_SERVICE; - $flink->token = $access_token->key; - $flink->credentials = $access_token->secret; + + $creds = TwitterOAuthClient::packToken($access_token); + + $flink->credentials = $creds; $flink->created = common_sql_now(); // Defaults: noticesync on, everything else off diff --git a/lib/oauthclient.php b/lib/oauthclient.php index 878e47091..b66a24be4 100644 --- a/lib/oauthclient.php +++ b/lib/oauthclient.php @@ -121,8 +121,6 @@ class OAuthClient $authorize_url .= '&oauth_callback=' . urlencode($oauth_callback); } - common_debug("$authorize_url"); - return $authorize_url; } diff --git a/lib/twitter.php b/lib/twitter.php index 4e2f67c66..280cdb0a3 100644 --- a/lib/twitter.php +++ b/lib/twitter.php @@ -160,7 +160,9 @@ function broadcast_twitter($notice) // XXX: Hack to get around PHP cURL's use of @ being a a meta character $statustxt = preg_replace('/^@/', ' @', $notice->content); - $client = new TwitterOAuthClient($flink->token, $flink->credentials); + $token = TwitterOAuthClient::unpackToken($flink->credentials); + + $client = new TwitterOAuthClient($token->key, $token->secret); $status = null; diff --git a/lib/twitteroauthclient.php b/lib/twitteroauthclient.php index c798ac877..b7dc4a80c 100644 --- a/lib/twitteroauthclient.php +++ b/lib/twitteroauthclient.php @@ -64,6 +64,23 @@ class TwitterOAuthClient extends OAuthClient $oauth_token, $oauth_token_secret); } + // XXX: the following two functions are to support the horrible hack + // of using the credentils field in Foreign_link to store both + // the access token and token secret. This hack should go away with + // 0.9, in which we can make DB changes and add a new column for the + // token itself. + + static function packToken($token) + { + return implode(chr(0), array($token->key, $token->secret)); + } + + static function unpackToken($str) + { + $vals = explode(chr(0), $str); + return new OAuthToken($vals[0], $vals[1]); + } + /** * Builds a link to Twitter's endpoint for authorizing a request token * diff --git a/scripts/synctwitterfriends.php b/scripts/synctwitterfriends.php index 39b9ad612..d13500e97 100755 --- a/scripts/synctwitterfriends.php +++ b/scripts/synctwitterfriends.php @@ -142,7 +142,9 @@ class SyncTwitterFriendsDaemon extends ParallelizingDaemon { $friends = array(); - $client = new TwitterOAuthClient($flink->token, $flink->credentials); + $token = TwitterOAuthClient::unpackToken($flink->credentials); + + $client = new TwitterOAuthClient($token->key, $token->secret); try { $friends_ids = $client->friendsIds(); diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index e04a8993f..f5289c5f4 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -161,7 +161,9 @@ class TwitterStatusFetcher extends ParallelizingDaemon // to start importing? How many statuses? Right now I'm going // with the default last 20. - $client = new TwitterOAuthClient($flink->token, $flink->credentials); + $token = TwitterOAuthClient::unpackToken($flink->credentials); + + $client = new TwitterOAuthClient($token->key, $token->secret); $timeline = null; -- cgit v1.2.3-54-g00ecf