From bac3ee95c96db10ed51a907f6f9b602a7c887c0f Mon Sep 17 00:00:00 2001 From: CiaranG Date: Thu, 16 Apr 2009 21:07:59 +0100 Subject: Abort the xmpp-related daemons immediately if xmpp is disabled in the config, otherwise they chew up *lots* of CPU doing nothing --- scripts/jabberqueuehandler.php | 9 ++++++++- scripts/publicqueuehandler.php | 7 +++++++ scripts/xmppconfirmhandler.php | 7 +++++++ scripts/xmppdaemon.php | 7 +++++++ 4 files changed, 29 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/jabberqueuehandler.php b/scripts/jabberqueuehandler.php index 924fc4545..8b6e974c0 100755 --- a/scripts/jabberqueuehandler.php +++ b/scripts/jabberqueuehandler.php @@ -54,6 +54,13 @@ class JabberQueueHandler extends XmppQueueHandler } } +// Abort immediately if xmpp is not enabled, otherwise the daemon chews up +// lots of CPU trying to connect to unconfigured servers +if (common_config('xmpp','enabled')==false) { + print "Aborting daemon - xmpp is disabled\n"; + exit(); +} + ini_set("max_execution_time", "0"); ini_set("max_input_time", "0"); set_time_limit(0); @@ -63,4 +70,4 @@ $resource = ($argc > 1) ? $argv[1] : (common_config('xmpp','resource') . '-queue $handler = new JabberQueueHandler($resource); -$handler->runOnce(); \ No newline at end of file +$handler->runOnce(); diff --git a/scripts/publicqueuehandler.php b/scripts/publicqueuehandler.php index 5075c12df..b0fa22d43 100755 --- a/scripts/publicqueuehandler.php +++ b/scripts/publicqueuehandler.php @@ -52,6 +52,13 @@ class PublicQueueHandler extends XmppQueueHandler } } +// Abort immediately if xmpp is not enabled, otherwise the daemon chews up +// lots of CPU trying to connect to unconfigured servers +if (common_config('xmpp','enabled')==false) { + print "Aborting daemon - xmpp is disabled\n"; + exit(); +} + ini_set("max_execution_time", "0"); ini_set("max_input_time", "0"); set_time_limit(0); diff --git a/scripts/xmppconfirmhandler.php b/scripts/xmppconfirmhandler.php index 2b8b085ce..7f39235fe 100755 --- a/scripts/xmppconfirmhandler.php +++ b/scripts/xmppconfirmhandler.php @@ -140,6 +140,13 @@ class XmppConfirmHandler extends XmppQueueHandler } } +// Abort immediately if xmpp is not enabled, otherwise the daemon chews up +// lots of CPU trying to connect to unconfigured servers +if (common_config('xmpp','enabled')==false) { + print "Aborting daemon - xmpp is disabled\n"; + exit(); +} + ini_set("max_execution_time", "0"); ini_set("max_input_time", "0"); set_time_limit(0); diff --git a/scripts/xmppdaemon.php b/scripts/xmppdaemon.php index ef3f8c63d..0ce2f2a28 100755 --- a/scripts/xmppdaemon.php +++ b/scripts/xmppdaemon.php @@ -321,6 +321,13 @@ class XMPPDaemon extends Daemon } } +// Abort immediately if xmpp is not enabled, otherwise the daemon chews up +// lots of CPU trying to connect to unconfigured servers +if (common_config('xmpp','enabled')==false) { + print "Aborting daemon - xmpp is disabled\n"; + exit(); +} + ini_set("max_execution_time", "0"); ini_set("max_input_time", "0"); set_time_limit(0); -- cgit v1.2.3-54-g00ecf From 7196410bb0398c15fbab767a9b7cedc513e6520b Mon Sep 17 00:00:00 2001 From: Tobias Diekershoff Date: Sat, 18 Apr 2009 19:00:20 +0200 Subject: shortening links in notices from XMPP This patch enables shortening of links, that where send from XMPP. The problem was, that in util.php common_current_user() is not finding the user account from which is posted, so the service to shorten is not known, so no shortening at all... This patch cleans up the xmppdaemon a little bit and hard codes ur1.ca as shortening service _if_ the user is not set. Ugly but working. --- lib/util.php | 20 ++++++-------------- scripts/xmppdaemon.php | 17 +++++------------ 2 files changed, 11 insertions(+), 26 deletions(-) (limited to 'scripts') diff --git a/lib/util.php b/lib/util.php index 675ff51f0..ab5e99593 100644 --- a/lib/util.php +++ b/lib/util.php @@ -519,11 +519,16 @@ function common_shorten_links($text) function common_shorten_link($url, $reverse = false) { + static $url_cache = array(); if ($reverse) return isset($url_cache[$url]) ? $url_cache[$url] : $url; $user = common_current_user(); - + if (!isset($user)) { + // common current user does not find a user when called from the XMPP daemon + // therefore we'll set one here fix, so that XMPP given URLs may be shortened + $user->urlshorteningservice = 'ur1.ca'; + } $curlh = curl_init(); curl_setopt($curlh, CURLOPT_CONNECTTIMEOUT, 20); // # seconds to wait curl_setopt($curlh, CURLOPT_USERAGENT, 'Laconica'); @@ -1321,16 +1326,3 @@ function common_compatible_license($from, $to) // XXX: better compatibility check needed here! return ($from == $to); } - -/** - * returns a quoted table name, if required according to config - */ -function common_database_tablename($tablename) -{ - - if(common_config('db','quote_identifiers')) { - $tablename = '"'. $tablename .'"'; - } - //table prefixes could be added here later - return $tablename; -} \ No newline at end of file diff --git a/scripts/xmppdaemon.php b/scripts/xmppdaemon.php index ef3f8c63d..5711f715d 100755 --- a/scripts/xmppdaemon.php +++ b/scripts/xmppdaemon.php @@ -152,11 +152,6 @@ class XMPPDaemon extends Daemon $body = preg_replace('/d[\ ]*('. $to .')[\ ]*/', '', $pl['body']); $this->add_direct($user, $body, $to, $from); } else { - $len = mb_strlen($pl['body']); - if($len > 140) { - $this->from_site($from, 'Message too long - maximum is 140 characters, you sent ' . $len); - return; - } $this->add_notice($user, $pl); } @@ -255,15 +250,13 @@ class XMPPDaemon extends Daemon function add_notice(&$user, &$pl) { $body = trim($pl['body']); - $content_shortened = common_shorten_link($body); + $content_shortened = common_shorten_links($body); if (mb_strlen($content_shortened) > 140) { - $content = trim(mb_substr($body, 0, 140)); - $content_shortened = common_shorten_link($content); - } - else { - $content = $body; + $from = jabber_normalize_jid($pl['from']); + $this->from_site($from, "Message too long - maximum is 140 characters, you sent ".mb_strlen($content_shortened)); + return; } - $notice = Notice::saveNew($user->id, $content, 'xmpp'); + $notice = Notice::saveNew($user->id, $content_shortened, 'xmpp'); if (is_string($notice)) { $this->log(LOG_ERR, $notice); return; -- cgit v1.2.3-54-g00ecf From 5f82ce18f012be4fe8d64a2ae79db99c2fd64670 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 21 Apr 2009 19:09:27 -0700 Subject: Script to import friends timelines from Twitter --- scripts/statusfetcher.php | 453 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) create mode 100644 scripts/statusfetcher.php (limited to 'scripts') diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php new file mode 100644 index 000000000..d303a098c --- /dev/null +++ b/scripts/statusfetcher.php @@ -0,0 +1,453 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +// Uncomment this to get useful console output +define('SCRIPT_DEBUG', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +$flink = new Foreign_link(); +$flink->service = 1; // Twitter +$cnt = $flink->find(); + +print "Updating Twitter friends subscriptions for $cnt users.\n"; + + +while ($flink->fetch()) { + + if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { + + $user = User::staticGet($flink->user_id); + + if (empty($user)) { + common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); + print "Unmatched user for ID $flink->user_id\n"; + continue; + } + + print 'Retrieving Friends Timeline for ' . $flink->user_id . "\n"; + + getTimeline($flink); + + if (defined('SCRIPT_DEBUG')) { + print "\nDONE\n"; + } + } +} + +function getTimeline($flink) +{ + + $fuser = $flink->getForeignUser(); + + if (empty($fuser)) { + common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); + if (defined('SCRIPT_DEBUG')) { + print "Unmatched user for ID $flink->user_id\n"; + } + continue; + } + + $screenname = $fuser->nickname; + + $url = 'http://twitter.com/statuses/friends_timeline.json'; + + $timeline_json = get_twitter_data($url, $fuser->nickname, + $flink->credentials); + + $timeline = json_decode($timeline_json); + + foreach ($timeline as $status) { + + // Hacktastic: filter out stuff coming from Laconica + $source = mb_strtolower(common_config('integration', 'source')); + + if (preg_match("/$source/", mb_strtolower($status->source))) { + continue; + } + + saveStatus($status, $flink); + } + +} + +function saveStatus($status, $flink) +{ + // Do we have a profile for this Twitter user? + + $id = ensureProfile($status->user); + $profile = Profile::staticGet($id); + + if (!$profile) { + common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); + if (defined('SCRIPT_DEBUG')) { + print "Problem saving notice. No associated Profile.\n"; + } + return null; + } + + $uri = 'http://twitter.com/' . $status->user->screen_name . + '/status/' . $status->id; + + // Skip save if notice source is Laconica or Identi.ca? + + $notice = Notice::staticGet('uri', $uri); + + // check to see if we've already imported the status + if (!$notice) { + + $notice = new Notice(); + + $notice->profile_id = $id; + + $notice->query('BEGIN'); + + // XXX: figure out reply_to + $notice->reply_to = null; + + // XXX: Should this be common_sql_now() instead of status create date? + + $notice->created = strftime('%Y-%m-%d %H:%M:%S', + strtotime($status->created_at)); + $notice->content = $status->text; + $notice->rendered = common_render_content($status->text, $notice); + $notice->source = 'twitter'; + $notice->is_local = 0; + $notice->uri = $uri; + + $notice_id = $notice->insert(); + + if (!$notice_id) { + common_log_db_error($notice, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Could not save notice!\n"; + } + } + + # XXX: do we need to change this for remote users? + + $notice->saveReplies(); + + // XXX: Do we want to polute our tag cloud with hashtags from Twitter? + $notice->saveTags(); + $notice->saveGroups(); + + $notice->query('COMMIT'); + + } + + if (!Notice_inbox::staticGet('notice_id', $notice->id)) { + + // Add to inbox + $inbox = new Notice_inbox(); + $inbox->user_id = $flink->user_id; + $inbox->notice_id = $notice->id; + $inbox->created = common_sql_now(); + + $inbox->insert(); + } + +} + +function ensureProfile($user) +{ + + // check to see if there's already a profile for this user + $profileurl = 'http://twitter.com/' . $user->screen_name; + + $profile = Profile::staticGet('profileurl', $profileurl); + + if ($profile) { + + common_debug("Profile for $profile->nickname found."); + + // Check to see if the user's Avatar has changed + checkAvatar($user, $profile); + return $profile->id; + + } else { + + $debugmsg = 'Adding profile and remote profile ' . + "for Twitter user: $profileurl\n"; + common_debug($debugmsg, __FILE__); + if (defined('SCRIPT_DEBUG')) { + print $debugmsg; + } + + $profile = new Profile(); + $profile->query("BEGIN"); + + $profile->nickname = $user->screen_name; + $profile->fullname = $user->name; + $profile->homepage = $user->url; + $profile->bio = $user->description; + $profile->location = $user->location; + $profile->profileurl = $profileurl; + $profile->created = common_sql_now(); + + $id = $profile->insert(); + + if (empty($id)) { + common_log_db_error($profile, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print 'Could not insert Profile: ' . + common_log_objstring($profile) . "\n"; + } + $profile->query("ROLLBACK"); + return false; + } + + // check for remote profile + $remote_pro = Remote_profile::staticGet('uri', $profileurl); + + if (!$remote_pro) { + + $remote_pro = new Remote_profile(); + + $remote_pro->id = $id; + $remote_pro->uri = $profileurl; + $remote_pro->created = common_sql_now(); + + $rid = $remote_pro->insert(); + + if (empty($rid)) { + common_log_db_error($profile, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print 'Could not insert Remote_profile: ' . + common_log_objstring($remote_pro) . "\n"; + } + $profile->query("ROLLBACK"); + return false; + } + } + + $profile->query("COMMIT"); + $profile->free(); + unset($profile); + + saveAvatars($user, $id); + + return $id; + } +} + +function checkAvatar($user, $profile) +{ + common_debug("in check avatar"); + + $path_parts = pathinfo($user->profile_image_url); + $newname = 'Twitter_' . $user->id . '_' . + $path_parts['basename']; + + $oldname = $profile->getAvatar(48)->filename; + + if ($newname != $oldname) { + + common_debug("Avatar for Twitter user $profile->nickname has changed."); + common_debug("old: $oldname new: $newname"); + + if (defined('SCRIPT_DEBUG')) { + print "Avatar for Twitter user $user->id has changed.\n"; + print "old: $oldname\n"; + print "new: $newname\n"; + } + + $img_root = substr($path_parts['basename'], 0, -11); + $ext = $path_parts['extension']; + $mediatype = getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $user->id . '_' . + $img_root . "_$size.$ext"; + + if (fetchAvatar($url, $filename)) { + updateAvatar($profile->id, $size, $mediatype, $filename); + } + } + } + +} + +function getMediatype($ext) +{ + $mediatype = null; + + switch (strtolower($ext)) { + case 'jpg': + $mediatype = 'image/jpg'; + break; + case 'gif': + $mediatype = 'image/gif'; + break; + default: + $mediatype = 'image/png'; + } + + return $mediatype; +} + + +function saveAvatars($user, $id) +{ + $path_parts = pathinfo($user->profile_image_url); + + // basename minus '_normal.ext' + + $ext = $path_parts['extension']; + $end = strlen('_normal' . $ext); + $img_root = substr($path_parts['basename'], 0, -($end+1)); + $mediatype = getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $user->id . '_' . + $img_root . "_$size.$ext"; + + if (fetchAvatar($url, $filename)) { + newAvatar($id, $size, $mediatype, $filename); + } else { + common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Problem fetching Avatar: $url\n"; + } + } + } +} + +function updateAvatar($profile_id, $size, $mediatype, $filename) { + + common_debug("updating avatar: $size"); + + $profile = Profile::staticGet($profile_id); + + if (!$profile) { + common_debug("Couldn't get profile: $profile_id!"); + if (defined('SCRIPT_DEBUG')) { + print "Couldn't get profile: $profile_id!\n"; + } + return; + } + + $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); + $avatar = $profile->getAvatar($sizes[$size]); + + if ($avatar) { + common_debug("Deleting $size avatar for $profile->nickname."); + @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); + $avatar->delete(); + } + + newAvatar($profile->id, $size, $mediatype, $filename); +} + +function newAvatar($profile_id, $size, $mediatype, $filename) +{ + $avatar = new Avatar(); + $avatar->profile_id = $profile_id; + + switch($size) { + case 'mini': + $avatar->width = 24; + $avatar->height = 24; + break; + case 'normal': + $avatar->width = 48; + $avatar->height = 48; + break; + default: + + // Note: Twitter's big avatars are a different size than + // Laconica's (Laconica's = 96) + + $avatar->width = 73; + $avatar->height = 73; + } + + $avatar->original = 0; // we don't have the original + $avatar->mediatype = $mediatype; + $avatar->filename = $filename; + $avatar->url = Avatar::url($filename); + + common_debug("new filename: $avatar->url"); + + $avatar->created = common_sql_now(); + + $id = $avatar->insert(); + + if (!$id) { + common_log_db_error($avatar, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Could not insert avatar!\n"; + } + + return null; + } + + common_debug("Saved new $size avatar for $profile_id."); + + return $id; +} + +function fetchAvatar($url, $filename) +{ + $avatar_dir = INSTALLDIR . '/avatar/'; + + $avatarfile = $avatar_dir . $filename; + + $out = fopen($avatarfile, 'wb'); + if (!$out) { + common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Couldn't open file! $filename\n"; + } + return false; + } + + common_debug("Fetching avatar: $url", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Fetching avatar from Twitter: $url\n"; + } + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_FILE, $out); + curl_setopt($ch, CURLOPT_BINARYTRANSFER, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); + $result = curl_exec($ch); + curl_close($ch); + + fclose($out); + + return $result; +} + -- cgit v1.2.3-54-g00ecf From 640628de2d593933e810b4785dfe38923b979713 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:03:19 -0400 Subject: A queuehandler for blowing caches offline We add a queuehandler for blowing the memcached caches off-line. This should speed up the processing of new notices. --- classes/Notice.php | 25 ++++++++++++++++- lib/util.php | 14 ++++++++-- scripts/memcachedqueuehandler.php | 58 +++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 3 +- scripts/stopdaemons.sh | 2 +- 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 scripts/memcachedqueuehandler.php (limited to 'scripts') diff --git a/classes/Notice.php b/classes/Notice.php index 5fa0d79a1..fbfeb9489 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -207,7 +207,11 @@ class Notice extends Memcached_DataObject # XXX: someone clever could prepend instead of clearing the cache if (common_config('memcached', 'enabled')) { - $notice->blowCaches(); + if (common_config('queues', 'enabled')) { + $notice->blowAuthorCaches(); + } else { + $notice->blowCaches(); + } } return $notice; @@ -271,6 +275,25 @@ class Notice extends Memcached_DataObject $this->blowGroupCache($blowLast); } + function blowAuthorCaches($blowLast=false) + { + // Clear the user's cache + $cache = common_memcache(); + if ($cache) { + $user = User::staticGet($this->profile_id); + if (!empty($user)) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); + if ($blowLast) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); + } + } + $user->free(); + unset($user); + } + $this->blowNoticeCache($blowLast); + $this->blowPublicCache($blowLast); + } + function blowGroupCache($blowLast=false) { $cache = common_memcache(); diff --git a/lib/util.php b/lib/util.php index e0eda0114..12797891c 100644 --- a/lib/util.php +++ b/lib/util.php @@ -879,7 +879,17 @@ function common_broadcast_notice($notice, $remote=false) function common_enqueue_notice($notice) { - foreach (array('jabber', 'omb', 'sms', 'public', 'twitter', 'facebook', 'ping') as $transport) { + $transports = array('omb', 'sms', 'twitter', 'facebook', 'ping'); + + if (common_config('xmpp', 'enabled')) { + $transports = array_merge($transports, array('jabber', 'public')); + } + + if (common_config('memcached', 'enabled')) { + $transports[] = 'memcached'; + } + + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; $qi->transport = $transport; @@ -1332,7 +1342,7 @@ function common_compatible_license($from, $to) */ function common_database_tablename($tablename) { - + if(common_config('db','quote_identifiers')) { $tablename = '"'. $tablename .'"'; } diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php new file mode 100644 index 000000000..3fcebcfc3 --- /dev/null +++ b/scripts/memcachedqueuehandler.php @@ -0,0 +1,58 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class MemcachedQueueHandler extends QueueHandler +{ + function transport() + { + return 'memcached'; + } + + function handle_notice($notice) + { + // XXX: fork here + common_log(LOG_INFO, "Blowing memcached for $notice->id\n"; + $notice->blowCaches(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new MemcachedQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index c3729761d..08de6d954 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -24,7 +24,8 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ - twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php; do + twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ + memcachedqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index 2bb8f9ecb..e5a181cd1 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,7 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler ; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From aee45ea91dcec4736d8b9befe17e030b873d9226 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:08:48 -0400 Subject: Add an inbox queue handler Handle distributing a notice to multiple inboxes in a queue handler rather than in the Web action. --- classes/Notice.php | 5 +++- lib/util.php | 4 +++ scripts/inboxqueuehandler.php | 57 +++++++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 2 +- scripts/stopdaemons.sh | 3 ++- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 scripts/inboxqueuehandler.php (limited to 'scripts') diff --git a/classes/Notice.php b/classes/Notice.php index fbfeb9489..ff00f2a94 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -197,7 +197,10 @@ class Notice extends Memcached_DataObject $notice->saveTags(); $notice->saveGroups(); - $notice->addToInboxes(); + if (!common_config('queues', 'enabled')) { + $notice->addToInboxes(); + } + $notice->query('COMMIT'); Event::handle('EndNoticeSave', array($notice)); diff --git a/lib/util.php b/lib/util.php index 12797891c..9b6d2941a 100644 --- a/lib/util.php +++ b/lib/util.php @@ -889,6 +889,10 @@ function common_enqueue_notice($notice) $transports[] = 'memcached'; } + if (common_config('queues', 'enabled')) { + $transports[] = 'inbox'; + } + foreach ($transports as $transport) { $qi = new Queue_item(); $qi->notice_id = $notice->id; diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php new file mode 100644 index 000000000..16e334b83 --- /dev/null +++ b/scripts/inboxqueuehandler.php @@ -0,0 +1,57 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +set_error_handler('common_error_handler'); + +class InboxQueueHandler extends QueueHandler +{ + function transport() + { + return 'inbox'; + } + + function handle_notice($notice) + { + common_log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); + $notice->addToInboxes(); + return true; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$handler = new InboxQueueHandler($resource); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index 08de6d954..66f9ed4e0 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -25,7 +25,7 @@ DIR=`dirname $0` for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ - memcachedqueuehandler.php; do + memcachedqueuehandler.php inboxqueuehandler.php; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index e5a181cd1..196991de0 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,8 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler memcachedhandler; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ + memcachedhandler inboxhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From 2053bdabef929f4a095d91fbdd3fa62646e9f332 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 05:23:59 -0400 Subject: fix parse error in memcachedqueuehandler --- scripts/memcachedqueuehandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php index 3fcebcfc3..43231fa2c 100644 --- a/scripts/memcachedqueuehandler.php +++ b/scripts/memcachedqueuehandler.php @@ -42,7 +42,7 @@ class MemcachedQueueHandler extends QueueHandler function handle_notice($notice) { // XXX: fork here - common_log(LOG_INFO, "Blowing memcached for $notice->id\n"; + common_log(LOG_INFO, "Blowing memcached for $notice->id"); $notice->blowCaches(); return true; } -- cgit v1.2.3-54-g00ecf From a3e727823d01ed9fd5f440eeaf27449092309865 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 09:52:21 +0000 Subject: some basic fixes for inbox and memcached queue handlers --- scripts/inboxqueuehandler.php | 15 +++++++++++++-- scripts/memcachedqueuehandler.php | 16 ++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) mode change 100644 => 100755 scripts/inboxqueuehandler.php mode change 100644 => 100755 scripts/memcachedqueuehandler.php (limited to 'scripts') diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php old mode 100644 new mode 100755 index 16e334b83..c76b80389 --- a/scripts/inboxqueuehandler.php +++ b/scripts/inboxqueuehandler.php @@ -29,6 +29,7 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); define('LACONICA', true); require_once(INSTALLDIR . '/lib/common.php'); +require_once(INSTALLDIR . '/lib/queuehandler.php'); set_error_handler('common_error_handler'); @@ -39,12 +40,20 @@ class InboxQueueHandler extends QueueHandler return 'inbox'; } + function start() { + $this->log(LOG_INFO, "INITIALIZE"); + return true; + } + function handle_notice($notice) { - common_log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); + $this->log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); $notice->addToInboxes(); return true; } + + function finish() { + } } ini_set("max_execution_time", "0"); @@ -52,6 +61,8 @@ ini_set("max_input_time", "0"); set_time_limit(0); mb_internal_encoding('UTF-8'); -$handler = new InboxQueueHandler($resource); +$id = ($argc > 1) ? $argv[1] : null; + +$handler = new InboxQueueHandler($id); $handler->runOnce(); diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php old mode 100644 new mode 100755 index 43231fa2c..6e819b41f --- a/scripts/memcachedqueuehandler.php +++ b/scripts/memcachedqueuehandler.php @@ -29,6 +29,7 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); define('LACONICA', true); require_once(INSTALLDIR . '/lib/common.php'); +require_once(INSTALLDIR . '/lib/queuehandler.php'); set_error_handler('common_error_handler'); @@ -39,13 +40,22 @@ class MemcachedQueueHandler extends QueueHandler return 'memcached'; } + function start() { + $this->log(LOG_INFO, "INITIALIZE"); + return true; + } + function handle_notice($notice) { // XXX: fork here - common_log(LOG_INFO, "Blowing memcached for $notice->id"); + $this->log(LOG_INFO, "Blowing memcached for $notice->id"); $notice->blowCaches(); return true; } + + function finish() { + } + } ini_set("max_execution_time", "0"); @@ -53,6 +63,8 @@ ini_set("max_input_time", "0"); set_time_limit(0); mb_internal_encoding('UTF-8'); -$handler = new MemcachedQueueHandler($resource); +$id = ($argc > 1) ? $argv[1] : null; + +$handler = new MemcachedQueueHandler($id); $handler->runOnce(); -- cgit v1.2.3-54-g00ecf From 290ae7888c4d8dbcb703920960b10bca635563ed Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 10:08:51 +0000 Subject: blow subs cache after updating inboxes --- scripts/inboxqueuehandler.php | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts') diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php index c76b80389..73d31e854 100755 --- a/scripts/inboxqueuehandler.php +++ b/scripts/inboxqueuehandler.php @@ -49,6 +49,7 @@ class InboxQueueHandler extends QueueHandler { $this->log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); $notice->addToInboxes(); + $notice->blowSubsCache(); return true; } -- cgit v1.2.3-54-g00ecf From 1c0d82de3bb7f75649a017a7d5632a6e070876c2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 23 Apr 2009 10:09:08 +0000 Subject: 8-char limit on transports --- lib/util.php | 3 ++- scripts/memcachedqueuehandler.php | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/lib/util.php b/lib/util.php index d77039b74..f862d7fcc 100644 --- a/lib/util.php +++ b/lib/util.php @@ -886,7 +886,8 @@ function common_enqueue_notice($notice) } if (common_config('memcached', 'enabled')) { - $transports[] = 'memcached'; + // Note: limited to 8 chars + $transports[] = 'memcache'; } if (common_config('inboxes', 'enabled') === true || diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php index 6e819b41f..185b781f7 100755 --- a/scripts/memcachedqueuehandler.php +++ b/scripts/memcachedqueuehandler.php @@ -37,7 +37,7 @@ class MemcachedQueueHandler extends QueueHandler { function transport() { - return 'memcached'; + return 'memcache'; } function start() { -- cgit v1.2.3-54-g00ecf From 6eb5a2566f5f29a4c129111e1cc86d85d641a3ad Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 24 Apr 2009 14:27:31 -0700 Subject: Some clean up -- this still doesn't work, yet. The processes all lose their database connections, including the parent process. --- scripts/statusfetcher.php | 324 ++++++++++++++++++++++++++++------------------ 1 file changed, 198 insertions(+), 126 deletions(-) (limited to 'scripts') diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php index d303a098c..8f4b60cf7 100644 --- a/scripts/statusfetcher.php +++ b/scripts/statusfetcher.php @@ -32,38 +32,117 @@ define('SCRIPT_DEBUG', true); require_once(INSTALLDIR . '/lib/common.php'); -$flink = new Foreign_link(); -$flink->service = 1; // Twitter -$cnt = $flink->find(); +$children = array(); +$flink_ids = null; -print "Updating Twitter friends subscriptions for $cnt users.\n"; +$MAXCHILDREN = 5; +$POLL_INTERVAL = 10; // 10 seconds +do { -while ($flink->fetch()) { + $flink = new Foreign_link(); + $flink->service = 1; // Twitter + $cnt = $flink->find(); - if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { + if (defined('SCRIPT_DEBUG')) { + print "Updating Twitter friends subscriptions for $cnt users.\n"; + } - $user = User::staticGet($flink->user_id); + $flink_ids = array(); - if (empty($user)) { - common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); - print "Unmatched user for ID $flink->user_id\n"; - continue; + // XXX: This only reliably happens once. After the first interation of + // the do loop, the ->find() doesn't work ... lost DB connection? + + while ($flink->fetch()) { + + if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { + $flink_ids[] = $flink->foreign_id; + } + } + + $flink->free(); + unset($flink); + + foreach ($flink_ids as $f){ + + $pid = pcntl_fork(); + + if ($pid == -1) { + die ("Couldn't fork!"); } - - print 'Retrieving Friends Timeline for ' . $flink->user_id . "\n"; - - getTimeline($flink); - + + // Parent + if ($pid) { + if (defined('SCRIPT_DEBUG')) { + print "Parent: forked " . $pid . "\n"; + } + $children[] = $pid; + } else { + + // Child + + // XXX: Each child needs its own DB connection + + getTimeline($f); + exit(); + } + + // Remove child from ps list as it finishes + while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { + if (defined('SCRIPT_DEBUG')) { + print "Child $c finished.\n"; + } + remove_ps($children, $c); + } + + // Wait if we have too many kids + if(sizeof($children) > $MAXCHILDREN) { + if (defined('SCRIPT_DEBUG')) { + print "Too many children. Waiting...\n"; + } + if( ($c = pcntl_wait($status, WUNTRACED) ) > 0){ + if (defined('SCRIPT_DEBUG')) { + print "Finished waiting for $c\n"; + } + remove_ps($children, $c); + } + } + } + + // Remove all children from the process list before restarting + while(($c = pcntl_wait($status, WUNTRACED)) > 0) { if (defined('SCRIPT_DEBUG')) { - print "\nDONE\n"; + print "Child $c finished.\n"; + } + remove_ps($children, $c); + } + + // Rest for a bit before we fetch more statuses + if (defined('SCRIPT_DEBUG')) { + print "Waiting $POLL_INTERVAL secs before hitting Twitter again.\n"; + } + + sleep($POLL_INTERVAL); + +} while (true); + + +function remove_ps(&$plist, $ps){ + for($i = 0; $i < sizeof($plist); $i++){ + if($plist[$i] == $ps){ + unset($plist[$i]); + $plist = array_values($plist); + break; } } } -function getTimeline($flink) +function getTimeline($fid) { + // XXX: Need to reconnect to the DB here? + + $flink = Foreign_link::getByForeignID($fid, 1); $fuser = $flink->getForeignUser(); if (empty($fuser)) { @@ -71,7 +150,6 @@ function getTimeline($flink) if (defined('SCRIPT_DEBUG')) { print "Unmatched user for ID $flink->user_id\n"; } - continue; } $screenname = $fuser->nickname; @@ -82,26 +160,34 @@ function getTimeline($flink) $flink->credentials); $timeline = json_decode($timeline_json); - + + if (empty($timeline)) { + common_log(LOG_WARNING, "Empty timeline."); + if (defined('SCRIPT_DEBUG')) { + print "Empty timeline!\n"; + } + return; + } + foreach ($timeline as $status) { - + // Hacktastic: filter out stuff coming from Laconica $source = mb_strtolower(common_config('integration', 'source')); - + if (preg_match("/$source/", mb_strtolower($status->source))) { continue; } - + saveStatus($status, $flink); } - + } function saveStatus($status, $flink) { - // Do we have a profile for this Twitter user? - - $id = ensureProfile($status->user); + // Do we have a profile for this Twitter user? + + $id = ensureProfile($status->user); $profile = Profile::staticGet($id); if (!$profile) { @@ -112,34 +198,33 @@ function saveStatus($status, $flink) return null; } - $uri = 'http://twitter.com/' . $status->user->screen_name . + $uri = 'http://twitter.com/' . $status->user->screen_name . '/status/' . $status->id; - - // Skip save if notice source is Laconica or Identi.ca? - - $notice = Notice::staticGet('uri', $uri); - + + // Skip save if notice source is Laconica or Identi.ca? + + $notice = Notice::staticGet('uri', $uri); + // check to see if we've already imported the status if (!$notice) { - - $notice = new Notice(); + $notice = new Notice(); $notice->profile_id = $id; - $notice->query('BEGIN'); + $notice->query('BEGIN'); // XXX: figure out reply_to - $notice->reply_to = null; - - // XXX: Should this be common_sql_now() instead of status create date? - - $notice->created = strftime('%Y-%m-%d %H:%M:%S', - strtotime($status->created_at)); - $notice->content = $status->text; - $notice->rendered = common_render_content($status->text, $notice); - $notice->source = 'twitter'; - $notice->is_local = 0; - $notice->uri = $uri; + $notice->reply_to = null; + + // XXX: Should this be common_sql_now() instead of status create date? + + $notice->created = strftime('%Y-%m-%d %H:%M:%S', + strtotime($status->created_at)); + $notice->content = $status->text; + $notice->rendered = common_render_content($status->text, $notice); + $notice->source = 'twitter'; + $notice->is_local = 0; + $notice->uri = $uri; $notice_id = $notice->insert(); @@ -150,59 +235,54 @@ function saveStatus($status, $flink) } } - # XXX: do we need to change this for remote users? - + // XXX: Figure out a better way to link replies? $notice->saveReplies(); - + // XXX: Do we want to polute our tag cloud with hashtags from Twitter? $notice->saveTags(); - $notice->saveGroups(); - + $notice->saveGroups(); + $notice->query('COMMIT'); - + } if (!Notice_inbox::staticGet('notice_id', $notice->id)) { - + // Add to inbox $inbox = new Notice_inbox(); $inbox->user_id = $flink->user_id; $inbox->notice_id = $notice->id; $inbox->created = common_sql_now(); - - $inbox->insert(); - } + $inbox->insert(); + } } -function ensureProfile($user) +function ensureProfile($user) { - + // check to see if there's already a profile for this user $profileurl = 'http://twitter.com/' . $user->screen_name; - $profile = Profile::staticGet('profileurl', $profileurl); - + if ($profile) { - common_debug("Profile for $profile->nickname found."); - + // Check to see if the user's Avatar has changed checkAvatar($user, $profile); return $profile->id; - + } else { - $debugmsg = 'Adding profile and remote profile ' . "for Twitter user: $profileurl\n"; common_debug($debugmsg, __FILE__); if (defined('SCRIPT_DEBUG')) { print $debugmsg; } - + $profile = new Profile(); $profile->query("BEGIN"); - + $profile->nickname = $user->screen_name; $profile->fullname = $user->name; $profile->homepage = $user->url; @@ -216,37 +296,37 @@ function ensureProfile($user) if (empty($id)) { common_log_db_error($profile, 'INSERT', __FILE__); if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Profile: ' . + print 'Could not insert Profile: ' . common_log_objstring($profile) . "\n"; } $profile->query("ROLLBACK"); return false; - } - - // check for remote profile + } + + // check for remote profile $remote_pro = Remote_profile::staticGet('uri', $profileurl); - + if (!$remote_pro) { - + $remote_pro = new Remote_profile(); $remote_pro->id = $id; $remote_pro->uri = $profileurl; $remote_pro->created = common_sql_now(); - + $rid = $remote_pro->insert(); - - if (empty($rid)) { + + if (empty($rid)) { common_log_db_error($profile, 'INSERT', __FILE__); if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Remote_profile: ' . + print 'Could not insert Remote_profile: ' . common_log_objstring($remote_pro) . "\n"; } $profile->query("ROLLBACK"); return false; - } + } } - + $profile->query("COMMIT"); $profile->free(); unset($profile); @@ -258,24 +338,22 @@ function ensureProfile($user) } function checkAvatar($user, $profile) -{ - common_debug("in check avatar"); - +{ $path_parts = pathinfo($user->profile_image_url); - $newname = 'Twitter_' . $user->id . '_' . + $newname = 'Twitter_' . $user->id . '_' . $path_parts['basename']; - + $oldname = $profile->getAvatar(48)->filename; if ($newname != $oldname) { - + common_debug("Avatar for Twitter user $profile->nickname has changed."); common_debug("old: $oldname new: $newname"); - + if (defined('SCRIPT_DEBUG')) { print "Avatar for Twitter user $user->id has changed.\n"; print "old: $oldname\n"; - print "new: $newname\n"; + print "new: $newname\n"; } $img_root = substr($path_parts['basename'], 0, -11); @@ -283,23 +361,22 @@ function checkAvatar($user, $profile) $mediatype = getMediatype($ext); foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . + $url = $path_parts['dirname'] . '/' . $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . + $filename = 'Twitter_' . $user->id . '_' . $img_root . "_$size.$ext"; if (fetchAvatar($url, $filename)) { - updateAvatar($profile->id, $size, $mediatype, $filename); + updateAvatar($profile->id, $size, $mediatype, $filename); } } } - } -function getMediatype($ext) +function getMediatype($ext) { $mediatype = null; - + switch (strtolower($ext)) { case 'jpg': $mediatype = 'image/jpg'; @@ -310,26 +387,22 @@ function getMediatype($ext) default: $mediatype = 'image/png'; } - + return $mediatype; } - -function saveAvatars($user, $id) +function saveAvatars($user, $id) { $path_parts = pathinfo($user->profile_image_url); - - // basename minus '_normal.ext' - $ext = $path_parts['extension']; $end = strlen('_normal' . $ext); $img_root = substr($path_parts['basename'], 0, -($end+1)); $mediatype = getMediatype($ext); - + foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . + $url = $path_parts['dirname'] . '/' . $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . + $filename = 'Twitter_' . $user->id . '_' . $img_root . "_$size.$ext"; if (fetchAvatar($url, $filename)) { @@ -348,7 +421,7 @@ function updateAvatar($profile_id, $size, $mediatype, $filename) { common_debug("updating avatar: $size"); $profile = Profile::staticGet($profile_id); - + if (!$profile) { common_debug("Couldn't get profile: $profile_id!"); if (defined('SCRIPT_DEBUG')) { @@ -356,17 +429,17 @@ function updateAvatar($profile_id, $size, $mediatype, $filename) { } return; } - + $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); $avatar = $profile->getAvatar($sizes[$size]); - + if ($avatar) { common_debug("Deleting $size avatar for $profile->nickname."); @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); $avatar->delete(); } - - newAvatar($profile->id, $size, $mediatype, $filename); + + newAvatar($profile->id, $size, $mediatype, $filename); } function newAvatar($profile_id, $size, $mediatype, $filename) @@ -376,19 +449,19 @@ function newAvatar($profile_id, $size, $mediatype, $filename) switch($size) { case 'mini': - $avatar->width = 24; + $avatar->width = 24; $avatar->height = 24; break; case 'normal': - $avatar->width = 48; + $avatar->width = 48; $avatar->height = 48; break; default: - - // Note: Twitter's big avatars are a different size than + + // Note: Twitter's big avatars are a different size than // Laconica's (Laconica's = 96) - - $avatar->width = 73; + + $avatar->width = 73; $avatar->height = 73; } @@ -396,33 +469,33 @@ function newAvatar($profile_id, $size, $mediatype, $filename) $avatar->mediatype = $mediatype; $avatar->filename = $filename; $avatar->url = Avatar::url($filename); - + common_debug("new filename: $avatar->url"); - + $avatar->created = common_sql_now(); $id = $avatar->insert(); - if (!$id) { + if (!$id) { common_log_db_error($avatar, 'INSERT', __FILE__); if (defined('SCRIPT_DEBUG')) { print "Could not insert avatar!\n"; } - + return null; } - + common_debug("Saved new $size avatar for $profile_id."); - + return $id; } -function fetchAvatar($url, $filename) +function fetchAvatar($url, $filename) { $avatar_dir = INSTALLDIR . '/avatar/'; - + $avatarfile = $avatar_dir . $filename; - + $out = fopen($avatarfile, 'wb'); if (!$out) { common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); @@ -431,12 +504,12 @@ function fetchAvatar($url, $filename) } return false; } - + common_debug("Fetching avatar: $url", __FILE__); if (defined('SCRIPT_DEBUG')) { print "Fetching avatar from Twitter: $url\n"; } - + $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_FILE, $out); @@ -447,7 +520,6 @@ function fetchAvatar($url, $filename) curl_close($ch); fclose($out); - + return $result; } - -- cgit v1.2.3-54-g00ecf From 6a20ef71d3b2b325ce24318e2ba4483d6c8732ce Mon Sep 17 00:00:00 2001 From: CiaranG Date: Tue, 28 Apr 2009 13:05:48 +0100 Subject: Fixed typo in stopdaemons.sh - was not stopping the new memcached queue handler --- scripts/stopdaemons.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index 196991de0..f6d71eddf 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -25,7 +25,7 @@ DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ - memcachedhandler inboxhandler; do + memcachehandler inboxhandler; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do -- cgit v1.2.3-54-g00ecf From 5b78f95e972f9f19ea46607e8b9544b8f7c4207a Mon Sep 17 00:00:00 2001 From: CiaranG Date: Tue, 28 Apr 2009 13:30:54 +0100 Subject: Only start daemons that are required, according to the site config. There is the potential to not start some more - see the checks in getvaliddaemons.php --- scripts/getvaliddaemons.php | 52 +++++++++++++++++++++++++++++++++++++++++++++ scripts/startdaemons.sh | 6 ++---- 2 files changed, 54 insertions(+), 4 deletions(-) create mode 100755 scripts/getvaliddaemons.php (limited to 'scripts') diff --git a/scripts/getvaliddaemons.php b/scripts/getvaliddaemons.php new file mode 100755 index 000000000..482e63af7 --- /dev/null +++ b/scripts/getvaliddaemons.php @@ -0,0 +1,52 @@ +#!/usr/bin/env php +. + */ + +/** + * Utility script to get a list of daemons that should run, based on the + * current configuration. This is used by startdaemons.sh to determine what + * it should and shouldn't start up. The output is a list of space-separated + * daemon names. + */ + + +# Abort if called from a web server +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +if(common_config('xmpp','enabled')) { + echo "xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php "; + echo "xmppconfirmhandler.php "; +} +if(common_config('memcached','enabled')) { + echo "memcachedqueuehandler.php "; +} +echo "ombqueuehandler.php "; +echo "twitterqueuehandler.php "; +echo "facebookqueuehandler.php "; +echo "pingqueuehandler.php "; +echo "inboxqueuehandler.php "; +echo "smsqueuehandler.php "; diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index 66f9ed4e0..3869e95c4 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -21,11 +21,9 @@ # Note that the 'maildaemon' needs to run as a mail filter. DIR=`dirname $0` +DAEMONS=`php $DIR/getvaliddaemons.php` -for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ - xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ - twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php \ - memcachedqueuehandler.php inboxqueuehandler.php; do +for f in $DAEMONS; do echo -n "Starting $f..."; php $DIR/$f -- cgit v1.2.3-54-g00ecf From 7f417cfee023b372a281f5e45a7593c09e279233 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 5 May 2009 19:28:57 +0000 Subject: More work on 2-way Twitter sync. Works better now with lastest version of DB_DataObject that automatically reconnects to the DB, but forked processes still lose connections occassionally. --- scripts/statusfetcher.php | 135 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 39 deletions(-) (limited to 'scripts') diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php index 8f4b60cf7..8c3ee4330 100644 --- a/scripts/statusfetcher.php +++ b/scripts/statusfetcher.php @@ -27,41 +27,20 @@ if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); define('LACONICA', true); +// Tune number of processes and how often to poll Twitter +define('MAXCHILDREN', 5); +define('POLL_INTERVAL', 60 * 5); // in seconds + // Uncomment this to get useful console output define('SCRIPT_DEBUG', true); require_once(INSTALLDIR . '/lib/common.php'); $children = array(); -$flink_ids = null; - -$MAXCHILDREN = 5; -$POLL_INTERVAL = 10; // 10 seconds do { - $flink = new Foreign_link(); - $flink->service = 1; // Twitter - $cnt = $flink->find(); - - if (defined('SCRIPT_DEBUG')) { - print "Updating Twitter friends subscriptions for $cnt users.\n"; - } - - $flink_ids = array(); - - // XXX: This only reliably happens once. After the first interation of - // the do loop, the ->find() doesn't work ... lost DB connection? - - while ($flink->fetch()) { - - if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { - $flink_ids[] = $flink->foreign_id; - } - } - - $flink->free(); - unset($flink); + $flink_ids = refreshFlinks(); foreach ($flink_ids as $f){ @@ -82,7 +61,6 @@ do { // Child // XXX: Each child needs its own DB connection - getTimeline($f); exit(); } @@ -96,11 +74,11 @@ do { } // Wait if we have too many kids - if(sizeof($children) > $MAXCHILDREN) { + if(sizeof($children) > MAXCHILDREN) { if (defined('SCRIPT_DEBUG')) { print "Too many children. Waiting...\n"; } - if( ($c = pcntl_wait($status, WUNTRACED) ) > 0){ + if(($c = pcntl_wait($status, WUNTRACED)) > 0){ if (defined('SCRIPT_DEBUG')) { print "Finished waiting for $c\n"; } @@ -119,14 +97,44 @@ do { // Rest for a bit before we fetch more statuses if (defined('SCRIPT_DEBUG')) { - print "Waiting $POLL_INTERVAL secs before hitting Twitter again.\n"; + print 'Waiting ' . POLL_INTERVAL . + " secs before hitting Twitter again.\n"; } - sleep($POLL_INTERVAL); + sleep(POLL_INTERVAL); } while (true); +function refreshFlinks() { + + global $config; + + $flink = new Foreign_link(); + $flink->service = 1; // Twitter + $flink->orderBy('last_noticesync'); + + $cnt = $flink->find(); + + if (defined('SCRIPT_DEBUG')) { + print "Updating Twitter friends subscriptions for $cnt users.\n"; + } + + $flinks = array(); + + while ($flink->fetch()) { + + if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { + $flinks[] = clone($flink); + } + } + + $flink->free(); + unset($flink); + + return $flinks; +} + function remove_ps(&$plist, $ps){ for($i = 0; $i < sizeof($plist); $i++){ if($plist[$i] == $ps){ @@ -137,22 +145,43 @@ function remove_ps(&$plist, $ps){ } } -function getTimeline($fid) +function getTimeline($flink) { - // XXX: Need to reconnect to the DB here? + global $config; + $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); + require_once(INSTALLDIR . '/lib/common.php'); - $flink = Foreign_link::getByForeignID($fid, 1); - $fuser = $flink->getForeignUser(); + if (defined('SCRIPT_DEBUG')) { + print "Trying to get timeline for $flink->foreign_id\n"; + } + + if (empty($flink)) { + common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); + if (defined('SCRIPT_DEBUG')) { + print "Can't retrieve Foreign_link for foreign ID $fid\n"; + } + return; + } + + $fuser = new Foreign_user(); + $fuser->service = 1; + $fuser->id = $flink->foreign_id; + $fuser->limit(1); + $fuser->find(true); if (empty($fuser)) { common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); if (defined('SCRIPT_DEBUG')) { print "Unmatched user for ID $flink->user_id\n"; } + return; } - $screenname = $fuser->nickname; + if (defined('SCRIPT_DEBUG')) { + // XXX: This is horrible and must be removed before releasing this + print 'username: ' . $fuser->nickname . ' password: ' . $flink->credentials . "\n"; + } $url = 'http://twitter.com/statuses/friends_timeline.json'; @@ -181,10 +210,19 @@ function getTimeline($fid) saveStatus($status, $flink); } + // Okay, record the time we synced with Twitter for posterity + + $flink->last_noticesync = common_sql_now(); + $flink->update(); } function saveStatus($status, $flink) { + + global $config; + $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); + require_once(INSTALLDIR . '/lib/common.php'); + // Do we have a profile for this Twitter user? $id = ensureProfile($status->user); @@ -244,6 +282,9 @@ function saveStatus($status, $flink) $notice->query('COMMIT'); + if (defined('SCRIPT_DEBUG')) { + print "Saved status $status->id as notice $notice->id.\n"; + } } if (!Notice_inbox::staticGet('notice_id', $notice->id)) { @@ -260,6 +301,7 @@ function saveStatus($status, $flink) function ensureProfile($user) { + global $config; // check to see if there's already a profile for this user $profileurl = 'http://twitter.com/' . $user->screen_name; @@ -328,8 +370,6 @@ function ensureProfile($user) } $profile->query("COMMIT"); - $profile->free(); - unset($profile); saveAvatars($user, $id); @@ -339,6 +379,8 @@ function ensureProfile($user) function checkAvatar($user, $profile) { + global $config; + $path_parts = pathinfo($user->profile_image_url); $newname = 'Twitter_' . $user->id . '_' . $path_parts['basename']; @@ -393,6 +435,8 @@ function getMediatype($ext) function saveAvatars($user, $id) { + global $config; + $path_parts = pathinfo($user->profile_image_url); $ext = $path_parts['extension']; $end = strlen('_normal' . $ext); @@ -418,7 +462,12 @@ function saveAvatars($user, $id) function updateAvatar($profile_id, $size, $mediatype, $filename) { - common_debug("updating avatar: $size"); + global $config; + + common_debug("Updating avatar: $size"); + if (defined('SCRIPT_DEBUG')) { + print "Updating avatar: $size\n"; + } $profile = Profile::staticGet($profile_id); @@ -444,6 +493,8 @@ function updateAvatar($profile_id, $size, $mediatype, $filename) { function newAvatar($profile_id, $size, $mediatype, $filename) { + global $config; + $avatar = new Avatar(); $avatar->profile_id = $profile_id; @@ -471,6 +522,9 @@ function newAvatar($profile_id, $size, $mediatype, $filename) $avatar->url = Avatar::url($filename); common_debug("new filename: $avatar->url"); + if (defined('SCRIPT_DEBUG')) { + print "New filename: $avatar->url\n"; + } $avatar->created = common_sql_now(); @@ -486,6 +540,9 @@ function newAvatar($profile_id, $size, $mediatype, $filename) } common_debug("Saved new $size avatar for $profile_id."); + if (defined('SCRIPT_DEBUG')) { + print "Saved new $size avatar for $profile_id.\n"; + } return $id; } -- cgit v1.2.3-54-g00ecf From 99e8f3235f2718f46cc95966ae39e725ee31a7df Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Wed, 6 May 2009 01:12:26 +0000 Subject: This finally works (provided the newer version of DB_DataObject that auto-reconnects to the DB). --- scripts/statusfetcher.php | 85 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php index 8c3ee4330..82ae5bfd4 100644 --- a/scripts/statusfetcher.php +++ b/scripts/statusfetcher.php @@ -29,7 +29,11 @@ define('LACONICA', true); // Tune number of processes and how often to poll Twitter define('MAXCHILDREN', 5); +<<<<<<< HEAD:scripts/statusfetcher.php +define('POLL_INTERVAL', 60 * 10); // in seconds +======= define('POLL_INTERVAL', 60 * 5); // in seconds +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php // Uncomment this to get useful console output define('SCRIPT_DEBUG', true); @@ -40,9 +44,20 @@ $children = array(); do { +<<<<<<< HEAD:scripts/statusfetcher.php + $flinks = refreshFlinks(); + + foreach ($flinks as $f){ + + // We have to disconnect from the DB before forking so + // each process will open its own connection and + // avoid stomping on each other +======= $flink_ids = refreshFlinks(); +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php - foreach ($flink_ids as $f){ + $conn = &$f->getDatabaseConnection(); + $conn->disconnect(); $pid = pcntl_fork(); @@ -50,38 +65,60 @@ do { die ("Couldn't fork!"); } - // Parent if ($pid) { + + // Parent + if (defined('SCRIPT_DEBUG')) { print "Parent: forked " . $pid . "\n"; } + $children[] = $pid; + } else { // Child +<<<<<<< HEAD:scripts/statusfetcher.php + getTimeline($f, $child_db_name); +======= // XXX: Each child needs its own DB connection getTimeline($f); +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php exit(); } // Remove child from ps list as it finishes while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { + if (defined('SCRIPT_DEBUG')) { print "Child $c finished.\n"; } + remove_ps($children, $c); } // Wait if we have too many kids +<<<<<<< HEAD:scripts/statusfetcher.php + if (sizeof($children) > MAXCHILDREN) { + + if (defined('SCRIPT_DEBUG')) { + print "Too many children. Waiting...\n"; + } + + if (($c = pcntl_wait($status, WUNTRACED)) > 0){ + +======= if(sizeof($children) > MAXCHILDREN) { if (defined('SCRIPT_DEBUG')) { print "Too many children. Waiting...\n"; } if(($c = pcntl_wait($status, WUNTRACED)) > 0){ +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php if (defined('SCRIPT_DEBUG')) { print "Finished waiting for $c\n"; } + remove_ps($children, $c); } } @@ -89,13 +126,17 @@ do { // Remove all children from the process list before restarting while(($c = pcntl_wait($status, WUNTRACED)) > 0) { + if (defined('SCRIPT_DEBUG')) { print "Child $c finished.\n"; } + remove_ps($children, $c); } // Rest for a bit before we fetch more statuses + common_debug('Waiting ' . POLL_INTERVAL . + ' secs before hitting Twitter again.'); if (defined('SCRIPT_DEBUG')) { print 'Waiting ' . POLL_INTERVAL . " secs before hitting Twitter again.\n"; @@ -108,8 +149,11 @@ do { function refreshFlinks() { +<<<<<<< HEAD:scripts/statusfetcher.php +======= global $config; +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php $flink = new Foreign_link(); $flink->service = 1; // Twitter $flink->orderBy('last_noticesync'); @@ -136,8 +180,8 @@ function refreshFlinks() { } function remove_ps(&$plist, $ps){ - for($i = 0; $i < sizeof($plist); $i++){ - if($plist[$i] == $ps){ + for ($i = 0; $i < sizeof($plist); $i++) { + if ($plist[$i] == $ps) { unset($plist[$i]); $plist = array_values($plist); break; @@ -148,6 +192,17 @@ function remove_ps(&$plist, $ps){ function getTimeline($flink) { +<<<<<<< HEAD:scripts/statusfetcher.php + if (empty($flink)) { + common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); + if (defined('SCRIPT_DEBUG')) { + print "Can't retrieve Foreign_link for foreign ID $fid\n"; + } + return; + } + + $fuser = $flink->getForeignUser(); +======= global $config; $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); require_once(INSTALLDIR . '/lib/common.php'); @@ -169,6 +224,7 @@ function getTimeline($flink) $fuser->id = $flink->foreign_id; $fuser->limit(1); $fuser->find(true); +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php if (empty($fuser)) { common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); @@ -178,9 +234,17 @@ function getTimeline($flink) return; } +<<<<<<< HEAD:scripts/statusfetcher.php + common_debug('Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id)."); + if (defined('SCRIPT_DEBUG')) { + print 'Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id).\n"; +======= if (defined('SCRIPT_DEBUG')) { // XXX: This is horrible and must be removed before releasing this print 'username: ' . $fuser->nickname . ' password: ' . $flink->credentials . "\n"; +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php } $url = 'http://twitter.com/statuses/friends_timeline.json'; @@ -218,6 +282,8 @@ function getTimeline($flink) function saveStatus($status, $flink) { +<<<<<<< HEAD:scripts/statusfetcher.php +======= global $config; $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); @@ -225,6 +291,7 @@ function saveStatus($status, $flink) // Do we have a profile for this Twitter user? +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php $id = ensureProfile($status->user); $profile = Profile::staticGet($id); @@ -282,6 +349,10 @@ function saveStatus($status, $flink) $notice->query('COMMIT'); +<<<<<<< HEAD:scripts/statusfetcher.php + common_debug("Saved status $status->id as notice $notice->id."); +======= +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php if (defined('SCRIPT_DEBUG')) { print "Saved status $status->id as notice $notice->id.\n"; } @@ -301,8 +372,11 @@ function saveStatus($status, $flink) function ensureProfile($user) { +<<<<<<< HEAD:scripts/statusfetcher.php +======= global $config; +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php // check to see if there's already a profile for this user $profileurl = 'http://twitter.com/' . $user->screen_name; $profile = Profile::staticGet('profileurl', $profileurl); @@ -462,8 +536,11 @@ function saveAvatars($user, $id) function updateAvatar($profile_id, $size, $mediatype, $filename) { +<<<<<<< HEAD:scripts/statusfetcher.php +======= global $config; +>>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php common_debug("Updating avatar: $size"); if (defined('SCRIPT_DEBUG')) { print "Updating avatar: $size\n"; -- cgit v1.2.3-54-g00ecf From b291cb8a1be0eb272e1663fbf6c6dea17bdb71db Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Wed, 6 May 2009 01:26:06 +0000 Subject: Fix for previous bad patch I pushed (had conflict markers) Sorry about that. --- scripts/statusfetcher.php | 80 ----------------------------------------------- 1 file changed, 80 deletions(-) (limited to 'scripts') diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php index 82ae5bfd4..5518e3aa8 100644 --- a/scripts/statusfetcher.php +++ b/scripts/statusfetcher.php @@ -29,11 +29,7 @@ define('LACONICA', true); // Tune number of processes and how often to poll Twitter define('MAXCHILDREN', 5); -<<<<<<< HEAD:scripts/statusfetcher.php define('POLL_INTERVAL', 60 * 10); // in seconds -======= -define('POLL_INTERVAL', 60 * 5); // in seconds ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php // Uncomment this to get useful console output define('SCRIPT_DEBUG', true); @@ -44,7 +40,6 @@ $children = array(); do { -<<<<<<< HEAD:scripts/statusfetcher.php $flinks = refreshFlinks(); foreach ($flinks as $f){ @@ -52,9 +47,6 @@ do { // We have to disconnect from the DB before forking so // each process will open its own connection and // avoid stomping on each other -======= - $flink_ids = refreshFlinks(); ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php $conn = &$f->getDatabaseConnection(); $conn->disconnect(); @@ -79,12 +71,7 @@ do { // Child -<<<<<<< HEAD:scripts/statusfetcher.php getTimeline($f, $child_db_name); -======= - // XXX: Each child needs its own DB connection - getTimeline($f); ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php exit(); } @@ -99,7 +86,6 @@ do { } // Wait if we have too many kids -<<<<<<< HEAD:scripts/statusfetcher.php if (sizeof($children) > MAXCHILDREN) { if (defined('SCRIPT_DEBUG')) { @@ -108,13 +94,6 @@ do { if (($c = pcntl_wait($status, WUNTRACED)) > 0){ -======= - if(sizeof($children) > MAXCHILDREN) { - if (defined('SCRIPT_DEBUG')) { - print "Too many children. Waiting...\n"; - } - if(($c = pcntl_wait($status, WUNTRACED)) > 0){ ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php if (defined('SCRIPT_DEBUG')) { print "Finished waiting for $c\n"; } @@ -149,11 +128,6 @@ do { function refreshFlinks() { -<<<<<<< HEAD:scripts/statusfetcher.php -======= - global $config; - ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php $flink = new Foreign_link(); $flink->service = 1; // Twitter $flink->orderBy('last_noticesync'); @@ -192,7 +166,6 @@ function remove_ps(&$plist, $ps){ function getTimeline($flink) { -<<<<<<< HEAD:scripts/statusfetcher.php if (empty($flink)) { common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); if (defined('SCRIPT_DEBUG')) { @@ -202,29 +175,6 @@ function getTimeline($flink) } $fuser = $flink->getForeignUser(); -======= - global $config; - $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); - require_once(INSTALLDIR . '/lib/common.php'); - - if (defined('SCRIPT_DEBUG')) { - print "Trying to get timeline for $flink->foreign_id\n"; - } - - if (empty($flink)) { - common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); - if (defined('SCRIPT_DEBUG')) { - print "Can't retrieve Foreign_link for foreign ID $fid\n"; - } - return; - } - - $fuser = new Foreign_user(); - $fuser->service = 1; - $fuser->id = $flink->foreign_id; - $fuser->limit(1); - $fuser->find(true); ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php if (empty($fuser)) { common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); @@ -234,17 +184,11 @@ function getTimeline($flink) return; } -<<<<<<< HEAD:scripts/statusfetcher.php common_debug('Trying to get timeline for Twitter user ' . "$fuser->nickname ($flink->foreign_id)."); if (defined('SCRIPT_DEBUG')) { print 'Trying to get timeline for Twitter user ' . "$fuser->nickname ($flink->foreign_id).\n"; -======= - if (defined('SCRIPT_DEBUG')) { - // XXX: This is horrible and must be removed before releasing this - print 'username: ' . $fuser->nickname . ' password: ' . $flink->credentials . "\n"; ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php } $url = 'http://twitter.com/statuses/friends_timeline.json'; @@ -282,16 +226,6 @@ function getTimeline($flink) function saveStatus($status, $flink) { -<<<<<<< HEAD:scripts/statusfetcher.php -======= - - global $config; - $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); - require_once(INSTALLDIR . '/lib/common.php'); - - // Do we have a profile for this Twitter user? - ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php $id = ensureProfile($status->user); $profile = Profile::staticGet($id); @@ -349,10 +283,6 @@ function saveStatus($status, $flink) $notice->query('COMMIT'); -<<<<<<< HEAD:scripts/statusfetcher.php - common_debug("Saved status $status->id as notice $notice->id."); -======= ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php if (defined('SCRIPT_DEBUG')) { print "Saved status $status->id as notice $notice->id.\n"; } @@ -372,11 +302,6 @@ function saveStatus($status, $flink) function ensureProfile($user) { -<<<<<<< HEAD:scripts/statusfetcher.php -======= - global $config; - ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php // check to see if there's already a profile for this user $profileurl = 'http://twitter.com/' . $user->screen_name; $profile = Profile::staticGet('profileurl', $profileurl); @@ -536,11 +461,6 @@ function saveAvatars($user, $id) function updateAvatar($profile_id, $size, $mediatype, $filename) { -<<<<<<< HEAD:scripts/statusfetcher.php -======= - global $config; - ->>>>>>> b8c700a7454db825b3867eadfa22afa1e5eb4f6c:scripts/statusfetcher.php common_debug("Updating avatar: $size"); if (defined('SCRIPT_DEBUG')) { print "Updating avatar: $size\n"; -- cgit v1.2.3-54-g00ecf From 48226e0c48e9bb2a7d97dbfd8f048ae299fbb7bf Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 7 May 2009 00:25:15 -0700 Subject: Properly daemonized 2-way Twitter bridge code --- scripts/statusfetcher.php | 795 ++++++++++++++++++++++++---------------------- 1 file changed, 413 insertions(+), 382 deletions(-) (limited to 'scripts') diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php index 5518e3aa8..5275a4575 100644 --- a/scripts/statusfetcher.php +++ b/scripts/statusfetcher.php @@ -28,378 +28,439 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); define('LACONICA', true); // Tune number of processes and how often to poll Twitter -define('MAXCHILDREN', 5); -define('POLL_INTERVAL', 60 * 10); // in seconds +// XXX: Should these things be in config.php? +define('MAXCHILDREN', 2); +define('POLL_INTERVAL', 60); // in seconds // Uncomment this to get useful console output define('SCRIPT_DEBUG', true); require_once(INSTALLDIR . '/lib/common.php'); +require_once(INSTALLDIR . '/lib/daemon.php'); -$children = array(); +class TwitterStatusFetcher extends Daemon +{ -do { + private $children = array(); - $flinks = refreshFlinks(); + function name() + { + return 'twitterstatusfetcher'; + } - foreach ($flinks as $f){ + function run() + { + do { - // We have to disconnect from the DB before forking so - // each process will open its own connection and - // avoid stomping on each other + $flinks = $this->refreshFlinks(); - $conn = &$f->getDatabaseConnection(); - $conn->disconnect(); + foreach ($flinks as $f){ - $pid = pcntl_fork(); + // We have to disconnect from the DB before forking so + // each sub-process will open its own connection and + // avoid stomping on the others - if ($pid == -1) { - die ("Couldn't fork!"); - } + $conn = &$f->getDatabaseConnection(); + $conn->disconnect(); - if ($pid) { + $pid = pcntl_fork(); - // Parent + if ($pid == -1) { + die ("Couldn't fork!"); + } - if (defined('SCRIPT_DEBUG')) { - print "Parent: forked " . $pid . "\n"; - } + if ($pid) { - $children[] = $pid; + // Parent + common_debug("Parent: forked new status fetcher process " . $pid); - } else { + if (defined('SCRIPT_DEBUG')) { + print "Parent: forked fetcher process " . $pid . "\n"; + } - // Child + $this->children[] = $pid; - getTimeline($f, $child_db_name); - exit(); - } + } else { - // Remove child from ps list as it finishes - while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { + // Child + $this->getTimeline($f); + exit(); + } - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; - } + // Remove child from ps list as it finishes + while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { - remove_ps($children, $c); - } + common_debug("Child $c finished."); - // Wait if we have too many kids - if (sizeof($children) > MAXCHILDREN) { + if (defined('SCRIPT_DEBUG')) { + print "Child $c finished.\n"; + } - if (defined('SCRIPT_DEBUG')) { - print "Too many children. Waiting...\n"; + $this->remove_ps($this->children, $c); + } + + // Wait! We have too many damn kids. + if (sizeof($this->children) > MAXCHILDREN) { + + common_debug('Too many children. Waiting...'); + + if (defined('SCRIPT_DEBUG')) { + print "Too many children. Waiting...\n"; + } + + if (($c = pcntl_wait($status, WUNTRACED)) > 0){ + + common_debug("Finished waiting for $c"); + + if (defined('SCRIPT_DEBUG')) { + print "Finished waiting for $c\n"; + } + + $this->remove_ps($this->children, $c); + } + } } - if (($c = pcntl_wait($status, WUNTRACED)) > 0){ + // Remove all children from the process list before restarting + while(($c = pcntl_wait($status, WUNTRACED)) > 0) { + + common_debug("Child $c finished."); if (defined('SCRIPT_DEBUG')) { - print "Finished waiting for $c\n"; + print "Child $c finished.\n"; } - remove_ps($children, $c); + $this->remove_ps($this->children, $c); } - } - } - - // Remove all children from the process list before restarting - while(($c = pcntl_wait($status, WUNTRACED)) > 0) { - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; - } + // Rest for a bit before we fetch more statuses + common_debug('Waiting ' . POLL_INTERVAL . + ' secs before hitting Twitter again.'); + if (defined('SCRIPT_DEBUG')) { + print 'Waiting ' . POLL_INTERVAL . + " secs before hitting Twitter again.\n"; + } - remove_ps($children, $c); - } + sleep(POLL_INTERVAL); - // Rest for a bit before we fetch more statuses - common_debug('Waiting ' . POLL_INTERVAL . - ' secs before hitting Twitter again.'); - if (defined('SCRIPT_DEBUG')) { - print 'Waiting ' . POLL_INTERVAL . - " secs before hitting Twitter again.\n"; + } while (true); } - sleep(POLL_INTERVAL); + function refreshFlinks() { -} while (true); + $flink = new Foreign_link(); + $flink->service = 1; // Twitter + $flink->orderBy('last_noticesync'); + $cnt = $flink->find(); -function refreshFlinks() { + if (defined('SCRIPT_DEBUG')) { + print "Updating Twitter friends subscriptions for $cnt users.\n"; + } - $flink = new Foreign_link(); - $flink->service = 1; // Twitter - $flink->orderBy('last_noticesync'); + $flinks = array(); - $cnt = $flink->find(); + while ($flink->fetch()) { - if (defined('SCRIPT_DEBUG')) { - print "Updating Twitter friends subscriptions for $cnt users.\n"; - } + if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { + $flinks[] = clone($flink); + } + } - $flinks = array(); + $flink->free(); + unset($flink); - while ($flink->fetch()) { + return $flinks; + } - if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { - $flinks[] = clone($flink); + function remove_ps(&$plist, $ps){ + for ($i = 0; $i < sizeof($plist); $i++) { + if ($plist[$i] == $ps) { + unset($plist[$i]); + $plist = array_values($plist); + break; + } } } - $flink->free(); - unset($flink); + function getTimeline($flink) + { - return $flinks; -} - -function remove_ps(&$plist, $ps){ - for ($i = 0; $i < sizeof($plist); $i++) { - if ($plist[$i] == $ps) { - unset($plist[$i]); - $plist = array_values($plist); - break; + if (empty($flink)) { + common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); + if (defined('SCRIPT_DEBUG')) { + print "Can't retrieve Foreign_link for foreign ID $fid\n"; + } + return; } - } -} -function getTimeline($flink) -{ + $fuser = $flink->getForeignUser(); - if (empty($flink)) { - common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); - if (defined('SCRIPT_DEBUG')) { - print "Can't retrieve Foreign_link for foreign ID $fid\n"; + if (empty($fuser)) { + common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); + if (defined('SCRIPT_DEBUG')) { + print "Unmatched user for ID $flink->user_id\n"; + } + return; } - return; - } - $fuser = $flink->getForeignUser(); - - if (empty($fuser)) { - common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); + common_debug('Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id)."); if (defined('SCRIPT_DEBUG')) { - print "Unmatched user for ID $flink->user_id\n"; + print 'Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id).\n"; } - return; - } - common_debug('Trying to get timeline for Twitter user ' . - "$fuser->nickname ($flink->foreign_id)."); - if (defined('SCRIPT_DEBUG')) { - print 'Trying to get timeline for Twitter user ' . - "$fuser->nickname ($flink->foreign_id).\n"; - } - - $url = 'http://twitter.com/statuses/friends_timeline.json'; + $url = 'http://twitter.com/statuses/friends_timeline.json'; - $timeline_json = get_twitter_data($url, $fuser->nickname, - $flink->credentials); + $timeline_json = get_twitter_data($url, $fuser->nickname, + $flink->credentials); - $timeline = json_decode($timeline_json); + $timeline = json_decode($timeline_json); - if (empty($timeline)) { - common_log(LOG_WARNING, "Empty timeline."); - if (defined('SCRIPT_DEBUG')) { - print "Empty timeline!\n"; + if (empty($timeline)) { + common_log(LOG_WARNING, "Empty timeline."); + if (defined('SCRIPT_DEBUG')) { + print "Empty timeline!\n"; + } + return; } - return; - } - foreach ($timeline as $status) { + foreach ($timeline as $status) { + + // Hacktastic: filter out stuff coming from Laconica + $source = mb_strtolower(common_config('integration', 'source')); - // Hacktastic: filter out stuff coming from Laconica - $source = mb_strtolower(common_config('integration', 'source')); + if (preg_match("/$source/", mb_strtolower($status->source))) { + continue; + } - if (preg_match("/$source/", mb_strtolower($status->source))) { - continue; + $this->saveStatus($status, $flink); } - saveStatus($status, $flink); + // Okay, record the time we synced with Twitter for posterity + + $flink->last_noticesync = common_sql_now(); + $flink->update(); } - // Okay, record the time we synced with Twitter for posterity + function saveStatus($status, $flink) + { + $id = $this->ensureProfile($status->user); + $profile = Profile::staticGet($id); - $flink->last_noticesync = common_sql_now(); - $flink->update(); -} - -function saveStatus($status, $flink) -{ - $id = ensureProfile($status->user); - $profile = Profile::staticGet($id); - - if (!$profile) { - common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); - if (defined('SCRIPT_DEBUG')) { - print "Problem saving notice. No associated Profile.\n"; + if (!$profile) { + common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); + if (defined('SCRIPT_DEBUG')) { + print "Problem saving notice. No associated Profile.\n"; + } + return null; } - return null; - } - $uri = 'http://twitter.com/' . $status->user->screen_name . - '/status/' . $status->id; + $uri = 'http://twitter.com/' . $status->user->screen_name . + '/status/' . $status->id; - // Skip save if notice source is Laconica or Identi.ca? + // Skip save if notice source is Laconica or Identi.ca? - $notice = Notice::staticGet('uri', $uri); + $notice = Notice::staticGet('uri', $uri); - // check to see if we've already imported the status - if (!$notice) { + // check to see if we've already imported the status + if (!$notice) { - $notice = new Notice(); - $notice->profile_id = $id; + $notice = new Notice(); + $notice->profile_id = $id; - $notice->query('BEGIN'); + $notice->query('BEGIN'); - // XXX: figure out reply_to - $notice->reply_to = null; + // XXX: figure out reply_to + $notice->reply_to = null; - // XXX: Should this be common_sql_now() instead of status create date? + // XXX: Should this be common_sql_now() instead of status create date? - $notice->created = strftime('%Y-%m-%d %H:%M:%S', - strtotime($status->created_at)); - $notice->content = $status->text; - $notice->rendered = common_render_content($status->text, $notice); - $notice->source = 'twitter'; - $notice->is_local = 0; - $notice->uri = $uri; + $notice->created = strftime('%Y-%m-%d %H:%M:%S', + strtotime($status->created_at)); + $notice->content = $status->text; + $notice->rendered = common_render_content($status->text, $notice); + $notice->source = 'twitter'; + $notice->is_local = 0; + $notice->uri = $uri; - $notice_id = $notice->insert(); + $notice_id = $notice->insert(); - if (!$notice_id) { - common_log_db_error($notice, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Could not save notice!\n"; + if (!$notice_id) { + common_log_db_error($notice, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Could not save notice!\n"; + } } - } - // XXX: Figure out a better way to link replies? - $notice->saveReplies(); + // XXX: Figure out a better way to link replies? + $notice->saveReplies(); - // XXX: Do we want to polute our tag cloud with hashtags from Twitter? - $notice->saveTags(); - $notice->saveGroups(); + // XXX: Do we want to polute our tag cloud with hashtags from Twitter? + $notice->saveTags(); + $notice->saveGroups(); - $notice->query('COMMIT'); + $notice->query('COMMIT'); - if (defined('SCRIPT_DEBUG')) { - print "Saved status $status->id as notice $notice->id.\n"; + if (defined('SCRIPT_DEBUG')) { + print "Saved status $status->id as notice $notice->id.\n"; + } } - } - if (!Notice_inbox::staticGet('notice_id', $notice->id)) { + if (!Notice_inbox::staticGet('notice_id', $notice->id)) { - // Add to inbox - $inbox = new Notice_inbox(); - $inbox->user_id = $flink->user_id; - $inbox->notice_id = $notice->id; - $inbox->created = common_sql_now(); + // Add to inbox + $inbox = new Notice_inbox(); + $inbox->user_id = $flink->user_id; + $inbox->notice_id = $notice->id; + $inbox->created = common_sql_now(); - $inbox->insert(); + $inbox->insert(); + } } -} -function ensureProfile($user) -{ - // check to see if there's already a profile for this user - $profileurl = 'http://twitter.com/' . $user->screen_name; - $profile = Profile::staticGet('profileurl', $profileurl); + function ensureProfile($user) + { + // check to see if there's already a profile for this user + $profileurl = 'http://twitter.com/' . $user->screen_name; + $profile = Profile::staticGet('profileurl', $profileurl); - if ($profile) { - common_debug("Profile for $profile->nickname found."); + if ($profile) { + common_debug("Profile for $profile->nickname found."); - // Check to see if the user's Avatar has changed - checkAvatar($user, $profile); - return $profile->id; + // Check to see if the user's Avatar has changed + $this->checkAvatar($user, $profile); + return $profile->id; - } else { - $debugmsg = 'Adding profile and remote profile ' . - "for Twitter user: $profileurl\n"; - common_debug($debugmsg, __FILE__); - if (defined('SCRIPT_DEBUG')) { - print $debugmsg; - } + } else { + $debugmsg = 'Adding profile and remote profile ' . + "for Twitter user: $profileurl\n"; + common_debug($debugmsg, __FILE__); + if (defined('SCRIPT_DEBUG')) { + print $debugmsg; + } - $profile = new Profile(); - $profile->query("BEGIN"); + $profile = new Profile(); + $profile->query("BEGIN"); - $profile->nickname = $user->screen_name; - $profile->fullname = $user->name; - $profile->homepage = $user->url; - $profile->bio = $user->description; - $profile->location = $user->location; - $profile->profileurl = $profileurl; - $profile->created = common_sql_now(); + $profile->nickname = $user->screen_name; + $profile->fullname = $user->name; + $profile->homepage = $user->url; + $profile->bio = $user->description; + $profile->location = $user->location; + $profile->profileurl = $profileurl; + $profile->created = common_sql_now(); - $id = $profile->insert(); + $id = $profile->insert(); - if (empty($id)) { - common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Profile: ' . - common_log_objstring($profile) . "\n"; + if (empty($id)) { + common_log_db_error($profile, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print 'Could not insert Profile: ' . + common_log_objstring($profile) . "\n"; + } + $profile->query("ROLLBACK"); + return false; } - $profile->query("ROLLBACK"); - return false; - } - // check for remote profile - $remote_pro = Remote_profile::staticGet('uri', $profileurl); + // check for remote profile + $remote_pro = Remote_profile::staticGet('uri', $profileurl); - if (!$remote_pro) { + if (!$remote_pro) { - $remote_pro = new Remote_profile(); + $remote_pro = new Remote_profile(); - $remote_pro->id = $id; - $remote_pro->uri = $profileurl; - $remote_pro->created = common_sql_now(); + $remote_pro->id = $id; + $remote_pro->uri = $profileurl; + $remote_pro->created = common_sql_now(); - $rid = $remote_pro->insert(); + $rid = $remote_pro->insert(); - if (empty($rid)) { - common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Remote_profile: ' . - common_log_objstring($remote_pro) . "\n"; + if (empty($rid)) { + common_log_db_error($profile, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print 'Could not insert Remote_profile: ' . + common_log_objstring($remote_pro) . "\n"; + } + $profile->query("ROLLBACK"); + return false; } - $profile->query("ROLLBACK"); - return false; } - } - $profile->query("COMMIT"); + $profile->query("COMMIT"); - saveAvatars($user, $id); + $this->saveAvatars($user, $id); - return $id; + return $id; + } } -} -function checkAvatar($user, $profile) -{ - global $config; + function checkAvatar($user, $profile) + { + global $config; - $path_parts = pathinfo($user->profile_image_url); - $newname = 'Twitter_' . $user->id . '_' . - $path_parts['basename']; + $path_parts = pathinfo($user->profile_image_url); + $newname = 'Twitter_' . $user->id . '_' . + $path_parts['basename']; - $oldname = $profile->getAvatar(48)->filename; + $oldname = $profile->getAvatar(48)->filename; - if ($newname != $oldname) { + if ($newname != $oldname) { - common_debug("Avatar for Twitter user $profile->nickname has changed."); - common_debug("old: $oldname new: $newname"); + common_debug("Avatar for Twitter user $profile->nickname has changed."); + common_debug("old: $oldname new: $newname"); - if (defined('SCRIPT_DEBUG')) { - print "Avatar for Twitter user $user->id has changed.\n"; - print "old: $oldname\n"; - print "new: $newname\n"; + if (defined('SCRIPT_DEBUG')) { + print "Avatar for Twitter user $user->id has changed.\n"; + print "old: $oldname\n"; + print "new: $newname\n"; + } + + $img_root = substr($path_parts['basename'], 0, -11); + $ext = $path_parts['extension']; + $mediatype = $this->getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $user->id . '_' . + $img_root . "_$size.$ext"; + + if ($this->fetchAvatar($url, $filename)) { + $this->updateAvatar($profile->id, $size, $mediatype, $filename); + } + } + } + } + + function getMediatype($ext) + { + $mediatype = null; + + switch (strtolower($ext)) { + case 'jpg': + $mediatype = 'image/jpg'; + break; + case 'gif': + $mediatype = 'image/gif'; + break; + default: + $mediatype = 'image/png'; } - $img_root = substr($path_parts['basename'], 0, -11); + return $mediatype; + } + + function saveAvatars($user, $id) + { + global $config; + + $path_parts = pathinfo($user->profile_image_url); $ext = $path_parts['extension']; - $mediatype = getMediatype($ext); + $end = strlen('_normal' . $ext); + $img_root = substr($path_parts['basename'], 0, -($end+1)); + $mediatype = $this->getMediatype($ext); foreach (array('mini', 'normal', 'bigger') as $size) { $url = $path_parts['dirname'] . '/' . @@ -407,173 +468,143 @@ function checkAvatar($user, $profile) $filename = 'Twitter_' . $user->id . '_' . $img_root . "_$size.$ext"; - if (fetchAvatar($url, $filename)) { - updateAvatar($profile->id, $size, $mediatype, $filename); + if ($this->fetchAvatar($url, $filename)) { + $this->newAvatar($id, $size, $mediatype, $filename); + } else { + common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Problem fetching Avatar: $url\n"; + } } } } -} -function getMediatype($ext) -{ - $mediatype = null; - - switch (strtolower($ext)) { - case 'jpg': - $mediatype = 'image/jpg'; - break; - case 'gif': - $mediatype = 'image/gif'; - break; - default: - $mediatype = 'image/png'; - } + function updateAvatar($profile_id, $size, $mediatype, $filename) { - return $mediatype; -} + common_debug("Updating avatar: $size"); + if (defined('SCRIPT_DEBUG')) { + print "Updating avatar: $size\n"; + } -function saveAvatars($user, $id) -{ - global $config; - - $path_parts = pathinfo($user->profile_image_url); - $ext = $path_parts['extension']; - $end = strlen('_normal' . $ext); - $img_root = substr($path_parts['basename'], 0, -($end+1)); - $mediatype = getMediatype($ext); - - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; - - if (fetchAvatar($url, $filename)) { - newAvatar($id, $size, $mediatype, $filename); - } else { - common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); + $profile = Profile::staticGet($profile_id); + + if (!$profile) { + common_debug("Couldn't get profile: $profile_id!"); if (defined('SCRIPT_DEBUG')) { - print "Problem fetching Avatar: $url\n"; + print "Couldn't get profile: $profile_id!\n"; } + return; } - } -} -function updateAvatar($profile_id, $size, $mediatype, $filename) { + $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); + $avatar = $profile->getAvatar($sizes[$size]); - common_debug("Updating avatar: $size"); - if (defined('SCRIPT_DEBUG')) { - print "Updating avatar: $size\n"; + if ($avatar) { + common_debug("Deleting $size avatar for $profile->nickname."); + @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); + $avatar->delete(); + } + + $this->newAvatar($profile->id, $size, $mediatype, $filename); } - $profile = Profile::staticGet($profile_id); + function newAvatar($profile_id, $size, $mediatype, $filename) + { + global $config; - if (!$profile) { - common_debug("Couldn't get profile: $profile_id!"); - if (defined('SCRIPT_DEBUG')) { - print "Couldn't get profile: $profile_id!\n"; - } - return; - } + $avatar = new Avatar(); + $avatar->profile_id = $profile_id; - $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); - $avatar = $profile->getAvatar($sizes[$size]); + switch($size) { + case 'mini': + $avatar->width = 24; + $avatar->height = 24; + break; + case 'normal': + $avatar->width = 48; + $avatar->height = 48; + break; + default: - if ($avatar) { - common_debug("Deleting $size avatar for $profile->nickname."); - @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); - $avatar->delete(); - } + // Note: Twitter's big avatars are a different size than + // Laconica's (Laconica's = 96) - newAvatar($profile->id, $size, $mediatype, $filename); -} + $avatar->width = 73; + $avatar->height = 73; + } -function newAvatar($profile_id, $size, $mediatype, $filename) -{ - global $config; - - $avatar = new Avatar(); - $avatar->profile_id = $profile_id; - - switch($size) { - case 'mini': - $avatar->width = 24; - $avatar->height = 24; - break; - case 'normal': - $avatar->width = 48; - $avatar->height = 48; - break; - default: - - // Note: Twitter's big avatars are a different size than - // Laconica's (Laconica's = 96) - - $avatar->width = 73; - $avatar->height = 73; - } + $avatar->original = 0; // we don't have the original + $avatar->mediatype = $mediatype; + $avatar->filename = $filename; + $avatar->url = Avatar::url($filename); + + common_debug("new filename: $avatar->url"); + if (defined('SCRIPT_DEBUG')) { + print "New filename: $avatar->url\n"; + } - $avatar->original = 0; // we don't have the original - $avatar->mediatype = $mediatype; - $avatar->filename = $filename; - $avatar->url = Avatar::url($filename); + $avatar->created = common_sql_now(); - common_debug("new filename: $avatar->url"); - if (defined('SCRIPT_DEBUG')) { - print "New filename: $avatar->url\n"; - } + $id = $avatar->insert(); - $avatar->created = common_sql_now(); + if (!$id) { + common_log_db_error($avatar, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Could not insert avatar!\n"; + } - $id = $avatar->insert(); + return null; + } - if (!$id) { - common_log_db_error($avatar, 'INSERT', __FILE__); + common_debug("Saved new $size avatar for $profile_id."); if (defined('SCRIPT_DEBUG')) { - print "Could not insert avatar!\n"; + print "Saved new $size avatar for $profile_id.\n"; } - return null; - } - - common_debug("Saved new $size avatar for $profile_id."); - if (defined('SCRIPT_DEBUG')) { - print "Saved new $size avatar for $profile_id.\n"; + return $id; } - return $id; -} + function fetchAvatar($url, $filename) + { + $avatar_dir = INSTALLDIR . '/avatar/'; -function fetchAvatar($url, $filename) -{ - $avatar_dir = INSTALLDIR . '/avatar/'; + $avatarfile = $avatar_dir . $filename; - $avatarfile = $avatar_dir . $filename; + $out = fopen($avatarfile, 'wb'); + if (!$out) { + common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Couldn't open file! $filename\n"; + } + return false; + } - $out = fopen($avatarfile, 'wb'); - if (!$out) { - common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); + common_debug("Fetching avatar: $url", __FILE__); if (defined('SCRIPT_DEBUG')) { - print "Couldn't open file! $filename\n"; + print "Fetching avatar from Twitter: $url\n"; } - return false; - } - common_debug("Fetching avatar: $url", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Fetching avatar from Twitter: $url\n"; + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_FILE, $out); + curl_setopt($ch, CURLOPT_BINARYTRANSFER, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); + $result = curl_exec($ch); + curl_close($ch); + + fclose($out); + + return $result; } +} - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_FILE, $out); - curl_setopt($ch, CURLOPT_BINARYTRANSFER, true); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); - $result = curl_exec($ch); - curl_close($ch); +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); +declare(ticks = 1); - fclose($out); +$fetcher = new TwitterStatusFetcher(); +$fetcher->runOnce(); - return $result; -} -- cgit v1.2.3-54-g00ecf From 2621a5471f9a3fa75d206ed5b3a4a91df1e28bdc Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 7 May 2009 00:26:42 -0700 Subject: Better name --- scripts/statusfetcher.php | 610 --------------------------------------- scripts/twitterstatusfetcher.php | 610 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 610 insertions(+), 610 deletions(-) delete mode 100644 scripts/statusfetcher.php create mode 100644 scripts/twitterstatusfetcher.php (limited to 'scripts') diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php deleted file mode 100644 index 5275a4575..000000000 --- a/scripts/statusfetcher.php +++ /dev/null @@ -1,610 +0,0 @@ -#!/usr/bin/env php -. - */ - -// Abort if called from a web server -if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { - print "This script must be run from the command line\n"; - exit(); -} - -define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); -define('LACONICA', true); - -// Tune number of processes and how often to poll Twitter -// XXX: Should these things be in config.php? -define('MAXCHILDREN', 2); -define('POLL_INTERVAL', 60); // in seconds - -// Uncomment this to get useful console output -define('SCRIPT_DEBUG', true); - -require_once(INSTALLDIR . '/lib/common.php'); -require_once(INSTALLDIR . '/lib/daemon.php'); - -class TwitterStatusFetcher extends Daemon -{ - - private $children = array(); - - function name() - { - return 'twitterstatusfetcher'; - } - - function run() - { - do { - - $flinks = $this->refreshFlinks(); - - foreach ($flinks as $f){ - - // We have to disconnect from the DB before forking so - // each sub-process will open its own connection and - // avoid stomping on the others - - $conn = &$f->getDatabaseConnection(); - $conn->disconnect(); - - $pid = pcntl_fork(); - - if ($pid == -1) { - die ("Couldn't fork!"); - } - - if ($pid) { - - // Parent - common_debug("Parent: forked new status fetcher process " . $pid); - - if (defined('SCRIPT_DEBUG')) { - print "Parent: forked fetcher process " . $pid . "\n"; - } - - $this->children[] = $pid; - - } else { - - // Child - $this->getTimeline($f); - exit(); - } - - // Remove child from ps list as it finishes - while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { - - common_debug("Child $c finished."); - - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; - } - - $this->remove_ps($this->children, $c); - } - - // Wait! We have too many damn kids. - if (sizeof($this->children) > MAXCHILDREN) { - - common_debug('Too many children. Waiting...'); - - if (defined('SCRIPT_DEBUG')) { - print "Too many children. Waiting...\n"; - } - - if (($c = pcntl_wait($status, WUNTRACED)) > 0){ - - common_debug("Finished waiting for $c"); - - if (defined('SCRIPT_DEBUG')) { - print "Finished waiting for $c\n"; - } - - $this->remove_ps($this->children, $c); - } - } - } - - // Remove all children from the process list before restarting - while(($c = pcntl_wait($status, WUNTRACED)) > 0) { - - common_debug("Child $c finished."); - - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; - } - - $this->remove_ps($this->children, $c); - } - - // Rest for a bit before we fetch more statuses - common_debug('Waiting ' . POLL_INTERVAL . - ' secs before hitting Twitter again.'); - if (defined('SCRIPT_DEBUG')) { - print 'Waiting ' . POLL_INTERVAL . - " secs before hitting Twitter again.\n"; - } - - sleep(POLL_INTERVAL); - - } while (true); - } - - function refreshFlinks() { - - $flink = new Foreign_link(); - $flink->service = 1; // Twitter - $flink->orderBy('last_noticesync'); - - $cnt = $flink->find(); - - if (defined('SCRIPT_DEBUG')) { - print "Updating Twitter friends subscriptions for $cnt users.\n"; - } - - $flinks = array(); - - while ($flink->fetch()) { - - if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { - $flinks[] = clone($flink); - } - } - - $flink->free(); - unset($flink); - - return $flinks; - } - - function remove_ps(&$plist, $ps){ - for ($i = 0; $i < sizeof($plist); $i++) { - if ($plist[$i] == $ps) { - unset($plist[$i]); - $plist = array_values($plist); - break; - } - } - } - - function getTimeline($flink) - { - - if (empty($flink)) { - common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); - if (defined('SCRIPT_DEBUG')) { - print "Can't retrieve Foreign_link for foreign ID $fid\n"; - } - return; - } - - $fuser = $flink->getForeignUser(); - - if (empty($fuser)) { - common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); - if (defined('SCRIPT_DEBUG')) { - print "Unmatched user for ID $flink->user_id\n"; - } - return; - } - - common_debug('Trying to get timeline for Twitter user ' . - "$fuser->nickname ($flink->foreign_id)."); - if (defined('SCRIPT_DEBUG')) { - print 'Trying to get timeline for Twitter user ' . - "$fuser->nickname ($flink->foreign_id).\n"; - } - - $url = 'http://twitter.com/statuses/friends_timeline.json'; - - $timeline_json = get_twitter_data($url, $fuser->nickname, - $flink->credentials); - - $timeline = json_decode($timeline_json); - - if (empty($timeline)) { - common_log(LOG_WARNING, "Empty timeline."); - if (defined('SCRIPT_DEBUG')) { - print "Empty timeline!\n"; - } - return; - } - - foreach ($timeline as $status) { - - // Hacktastic: filter out stuff coming from Laconica - $source = mb_strtolower(common_config('integration', 'source')); - - if (preg_match("/$source/", mb_strtolower($status->source))) { - continue; - } - - $this->saveStatus($status, $flink); - } - - // Okay, record the time we synced with Twitter for posterity - - $flink->last_noticesync = common_sql_now(); - $flink->update(); - } - - function saveStatus($status, $flink) - { - $id = $this->ensureProfile($status->user); - $profile = Profile::staticGet($id); - - if (!$profile) { - common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); - if (defined('SCRIPT_DEBUG')) { - print "Problem saving notice. No associated Profile.\n"; - } - return null; - } - - $uri = 'http://twitter.com/' . $status->user->screen_name . - '/status/' . $status->id; - - // Skip save if notice source is Laconica or Identi.ca? - - $notice = Notice::staticGet('uri', $uri); - - // check to see if we've already imported the status - if (!$notice) { - - $notice = new Notice(); - $notice->profile_id = $id; - - $notice->query('BEGIN'); - - // XXX: figure out reply_to - $notice->reply_to = null; - - // XXX: Should this be common_sql_now() instead of status create date? - - $notice->created = strftime('%Y-%m-%d %H:%M:%S', - strtotime($status->created_at)); - $notice->content = $status->text; - $notice->rendered = common_render_content($status->text, $notice); - $notice->source = 'twitter'; - $notice->is_local = 0; - $notice->uri = $uri; - - $notice_id = $notice->insert(); - - if (!$notice_id) { - common_log_db_error($notice, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Could not save notice!\n"; - } - } - - // XXX: Figure out a better way to link replies? - $notice->saveReplies(); - - // XXX: Do we want to polute our tag cloud with hashtags from Twitter? - $notice->saveTags(); - $notice->saveGroups(); - - $notice->query('COMMIT'); - - if (defined('SCRIPT_DEBUG')) { - print "Saved status $status->id as notice $notice->id.\n"; - } - } - - if (!Notice_inbox::staticGet('notice_id', $notice->id)) { - - // Add to inbox - $inbox = new Notice_inbox(); - $inbox->user_id = $flink->user_id; - $inbox->notice_id = $notice->id; - $inbox->created = common_sql_now(); - - $inbox->insert(); - } - } - - function ensureProfile($user) - { - // check to see if there's already a profile for this user - $profileurl = 'http://twitter.com/' . $user->screen_name; - $profile = Profile::staticGet('profileurl', $profileurl); - - if ($profile) { - common_debug("Profile for $profile->nickname found."); - - // Check to see if the user's Avatar has changed - $this->checkAvatar($user, $profile); - return $profile->id; - - } else { - $debugmsg = 'Adding profile and remote profile ' . - "for Twitter user: $profileurl\n"; - common_debug($debugmsg, __FILE__); - if (defined('SCRIPT_DEBUG')) { - print $debugmsg; - } - - $profile = new Profile(); - $profile->query("BEGIN"); - - $profile->nickname = $user->screen_name; - $profile->fullname = $user->name; - $profile->homepage = $user->url; - $profile->bio = $user->description; - $profile->location = $user->location; - $profile->profileurl = $profileurl; - $profile->created = common_sql_now(); - - $id = $profile->insert(); - - if (empty($id)) { - common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Profile: ' . - common_log_objstring($profile) . "\n"; - } - $profile->query("ROLLBACK"); - return false; - } - - // check for remote profile - $remote_pro = Remote_profile::staticGet('uri', $profileurl); - - if (!$remote_pro) { - - $remote_pro = new Remote_profile(); - - $remote_pro->id = $id; - $remote_pro->uri = $profileurl; - $remote_pro->created = common_sql_now(); - - $rid = $remote_pro->insert(); - - if (empty($rid)) { - common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Remote_profile: ' . - common_log_objstring($remote_pro) . "\n"; - } - $profile->query("ROLLBACK"); - return false; - } - } - - $profile->query("COMMIT"); - - $this->saveAvatars($user, $id); - - return $id; - } - } - - function checkAvatar($user, $profile) - { - global $config; - - $path_parts = pathinfo($user->profile_image_url); - $newname = 'Twitter_' . $user->id . '_' . - $path_parts['basename']; - - $oldname = $profile->getAvatar(48)->filename; - - if ($newname != $oldname) { - - common_debug("Avatar for Twitter user $profile->nickname has changed."); - common_debug("old: $oldname new: $newname"); - - if (defined('SCRIPT_DEBUG')) { - print "Avatar for Twitter user $user->id has changed.\n"; - print "old: $oldname\n"; - print "new: $newname\n"; - } - - $img_root = substr($path_parts['basename'], 0, -11); - $ext = $path_parts['extension']; - $mediatype = $this->getMediatype($ext); - - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; - - if ($this->fetchAvatar($url, $filename)) { - $this->updateAvatar($profile->id, $size, $mediatype, $filename); - } - } - } - } - - function getMediatype($ext) - { - $mediatype = null; - - switch (strtolower($ext)) { - case 'jpg': - $mediatype = 'image/jpg'; - break; - case 'gif': - $mediatype = 'image/gif'; - break; - default: - $mediatype = 'image/png'; - } - - return $mediatype; - } - - function saveAvatars($user, $id) - { - global $config; - - $path_parts = pathinfo($user->profile_image_url); - $ext = $path_parts['extension']; - $end = strlen('_normal' . $ext); - $img_root = substr($path_parts['basename'], 0, -($end+1)); - $mediatype = $this->getMediatype($ext); - - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; - - if ($this->fetchAvatar($url, $filename)) { - $this->newAvatar($id, $size, $mediatype, $filename); - } else { - common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Problem fetching Avatar: $url\n"; - } - } - } - } - - function updateAvatar($profile_id, $size, $mediatype, $filename) { - - common_debug("Updating avatar: $size"); - if (defined('SCRIPT_DEBUG')) { - print "Updating avatar: $size\n"; - } - - $profile = Profile::staticGet($profile_id); - - if (!$profile) { - common_debug("Couldn't get profile: $profile_id!"); - if (defined('SCRIPT_DEBUG')) { - print "Couldn't get profile: $profile_id!\n"; - } - return; - } - - $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); - $avatar = $profile->getAvatar($sizes[$size]); - - if ($avatar) { - common_debug("Deleting $size avatar for $profile->nickname."); - @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); - $avatar->delete(); - } - - $this->newAvatar($profile->id, $size, $mediatype, $filename); - } - - function newAvatar($profile_id, $size, $mediatype, $filename) - { - global $config; - - $avatar = new Avatar(); - $avatar->profile_id = $profile_id; - - switch($size) { - case 'mini': - $avatar->width = 24; - $avatar->height = 24; - break; - case 'normal': - $avatar->width = 48; - $avatar->height = 48; - break; - default: - - // Note: Twitter's big avatars are a different size than - // Laconica's (Laconica's = 96) - - $avatar->width = 73; - $avatar->height = 73; - } - - $avatar->original = 0; // we don't have the original - $avatar->mediatype = $mediatype; - $avatar->filename = $filename; - $avatar->url = Avatar::url($filename); - - common_debug("new filename: $avatar->url"); - if (defined('SCRIPT_DEBUG')) { - print "New filename: $avatar->url\n"; - } - - $avatar->created = common_sql_now(); - - $id = $avatar->insert(); - - if (!$id) { - common_log_db_error($avatar, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Could not insert avatar!\n"; - } - - return null; - } - - common_debug("Saved new $size avatar for $profile_id."); - if (defined('SCRIPT_DEBUG')) { - print "Saved new $size avatar for $profile_id.\n"; - } - - return $id; - } - - function fetchAvatar($url, $filename) - { - $avatar_dir = INSTALLDIR . '/avatar/'; - - $avatarfile = $avatar_dir . $filename; - - $out = fopen($avatarfile, 'wb'); - if (!$out) { - common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Couldn't open file! $filename\n"; - } - return false; - } - - common_debug("Fetching avatar: $url", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Fetching avatar from Twitter: $url\n"; - } - - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_FILE, $out); - curl_setopt($ch, CURLOPT_BINARYTRANSFER, true); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); - $result = curl_exec($ch); - curl_close($ch); - - fclose($out); - - return $result; - } -} - -ini_set("max_execution_time", "0"); -ini_set("max_input_time", "0"); -set_time_limit(0); -mb_internal_encoding('UTF-8'); -declare(ticks = 1); - -$fetcher = new TwitterStatusFetcher(); -$fetcher->runOnce(); - diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php new file mode 100644 index 000000000..5275a4575 --- /dev/null +++ b/scripts/twitterstatusfetcher.php @@ -0,0 +1,610 @@ +#!/usr/bin/env php +. + */ + +// Abort if called from a web server +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +// Tune number of processes and how often to poll Twitter +// XXX: Should these things be in config.php? +define('MAXCHILDREN', 2); +define('POLL_INTERVAL', 60); // in seconds + +// Uncomment this to get useful console output +define('SCRIPT_DEBUG', true); + +require_once(INSTALLDIR . '/lib/common.php'); +require_once(INSTALLDIR . '/lib/daemon.php'); + +class TwitterStatusFetcher extends Daemon +{ + + private $children = array(); + + function name() + { + return 'twitterstatusfetcher'; + } + + function run() + { + do { + + $flinks = $this->refreshFlinks(); + + foreach ($flinks as $f){ + + // We have to disconnect from the DB before forking so + // each sub-process will open its own connection and + // avoid stomping on the others + + $conn = &$f->getDatabaseConnection(); + $conn->disconnect(); + + $pid = pcntl_fork(); + + if ($pid == -1) { + die ("Couldn't fork!"); + } + + if ($pid) { + + // Parent + common_debug("Parent: forked new status fetcher process " . $pid); + + if (defined('SCRIPT_DEBUG')) { + print "Parent: forked fetcher process " . $pid . "\n"; + } + + $this->children[] = $pid; + + } else { + + // Child + $this->getTimeline($f); + exit(); + } + + // Remove child from ps list as it finishes + while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { + + common_debug("Child $c finished."); + + if (defined('SCRIPT_DEBUG')) { + print "Child $c finished.\n"; + } + + $this->remove_ps($this->children, $c); + } + + // Wait! We have too many damn kids. + if (sizeof($this->children) > MAXCHILDREN) { + + common_debug('Too many children. Waiting...'); + + if (defined('SCRIPT_DEBUG')) { + print "Too many children. Waiting...\n"; + } + + if (($c = pcntl_wait($status, WUNTRACED)) > 0){ + + common_debug("Finished waiting for $c"); + + if (defined('SCRIPT_DEBUG')) { + print "Finished waiting for $c\n"; + } + + $this->remove_ps($this->children, $c); + } + } + } + + // Remove all children from the process list before restarting + while(($c = pcntl_wait($status, WUNTRACED)) > 0) { + + common_debug("Child $c finished."); + + if (defined('SCRIPT_DEBUG')) { + print "Child $c finished.\n"; + } + + $this->remove_ps($this->children, $c); + } + + // Rest for a bit before we fetch more statuses + common_debug('Waiting ' . POLL_INTERVAL . + ' secs before hitting Twitter again.'); + if (defined('SCRIPT_DEBUG')) { + print 'Waiting ' . POLL_INTERVAL . + " secs before hitting Twitter again.\n"; + } + + sleep(POLL_INTERVAL); + + } while (true); + } + + function refreshFlinks() { + + $flink = new Foreign_link(); + $flink->service = 1; // Twitter + $flink->orderBy('last_noticesync'); + + $cnt = $flink->find(); + + if (defined('SCRIPT_DEBUG')) { + print "Updating Twitter friends subscriptions for $cnt users.\n"; + } + + $flinks = array(); + + while ($flink->fetch()) { + + if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { + $flinks[] = clone($flink); + } + } + + $flink->free(); + unset($flink); + + return $flinks; + } + + function remove_ps(&$plist, $ps){ + for ($i = 0; $i < sizeof($plist); $i++) { + if ($plist[$i] == $ps) { + unset($plist[$i]); + $plist = array_values($plist); + break; + } + } + } + + function getTimeline($flink) + { + + if (empty($flink)) { + common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); + if (defined('SCRIPT_DEBUG')) { + print "Can't retrieve Foreign_link for foreign ID $fid\n"; + } + return; + } + + $fuser = $flink->getForeignUser(); + + if (empty($fuser)) { + common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); + if (defined('SCRIPT_DEBUG')) { + print "Unmatched user for ID $flink->user_id\n"; + } + return; + } + + common_debug('Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id)."); + if (defined('SCRIPT_DEBUG')) { + print 'Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id).\n"; + } + + $url = 'http://twitter.com/statuses/friends_timeline.json'; + + $timeline_json = get_twitter_data($url, $fuser->nickname, + $flink->credentials); + + $timeline = json_decode($timeline_json); + + if (empty($timeline)) { + common_log(LOG_WARNING, "Empty timeline."); + if (defined('SCRIPT_DEBUG')) { + print "Empty timeline!\n"; + } + return; + } + + foreach ($timeline as $status) { + + // Hacktastic: filter out stuff coming from Laconica + $source = mb_strtolower(common_config('integration', 'source')); + + if (preg_match("/$source/", mb_strtolower($status->source))) { + continue; + } + + $this->saveStatus($status, $flink); + } + + // Okay, record the time we synced with Twitter for posterity + + $flink->last_noticesync = common_sql_now(); + $flink->update(); + } + + function saveStatus($status, $flink) + { + $id = $this->ensureProfile($status->user); + $profile = Profile::staticGet($id); + + if (!$profile) { + common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); + if (defined('SCRIPT_DEBUG')) { + print "Problem saving notice. No associated Profile.\n"; + } + return null; + } + + $uri = 'http://twitter.com/' . $status->user->screen_name . + '/status/' . $status->id; + + // Skip save if notice source is Laconica or Identi.ca? + + $notice = Notice::staticGet('uri', $uri); + + // check to see if we've already imported the status + if (!$notice) { + + $notice = new Notice(); + $notice->profile_id = $id; + + $notice->query('BEGIN'); + + // XXX: figure out reply_to + $notice->reply_to = null; + + // XXX: Should this be common_sql_now() instead of status create date? + + $notice->created = strftime('%Y-%m-%d %H:%M:%S', + strtotime($status->created_at)); + $notice->content = $status->text; + $notice->rendered = common_render_content($status->text, $notice); + $notice->source = 'twitter'; + $notice->is_local = 0; + $notice->uri = $uri; + + $notice_id = $notice->insert(); + + if (!$notice_id) { + common_log_db_error($notice, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Could not save notice!\n"; + } + } + + // XXX: Figure out a better way to link replies? + $notice->saveReplies(); + + // XXX: Do we want to polute our tag cloud with hashtags from Twitter? + $notice->saveTags(); + $notice->saveGroups(); + + $notice->query('COMMIT'); + + if (defined('SCRIPT_DEBUG')) { + print "Saved status $status->id as notice $notice->id.\n"; + } + } + + if (!Notice_inbox::staticGet('notice_id', $notice->id)) { + + // Add to inbox + $inbox = new Notice_inbox(); + $inbox->user_id = $flink->user_id; + $inbox->notice_id = $notice->id; + $inbox->created = common_sql_now(); + + $inbox->insert(); + } + } + + function ensureProfile($user) + { + // check to see if there's already a profile for this user + $profileurl = 'http://twitter.com/' . $user->screen_name; + $profile = Profile::staticGet('profileurl', $profileurl); + + if ($profile) { + common_debug("Profile for $profile->nickname found."); + + // Check to see if the user's Avatar has changed + $this->checkAvatar($user, $profile); + return $profile->id; + + } else { + $debugmsg = 'Adding profile and remote profile ' . + "for Twitter user: $profileurl\n"; + common_debug($debugmsg, __FILE__); + if (defined('SCRIPT_DEBUG')) { + print $debugmsg; + } + + $profile = new Profile(); + $profile->query("BEGIN"); + + $profile->nickname = $user->screen_name; + $profile->fullname = $user->name; + $profile->homepage = $user->url; + $profile->bio = $user->description; + $profile->location = $user->location; + $profile->profileurl = $profileurl; + $profile->created = common_sql_now(); + + $id = $profile->insert(); + + if (empty($id)) { + common_log_db_error($profile, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print 'Could not insert Profile: ' . + common_log_objstring($profile) . "\n"; + } + $profile->query("ROLLBACK"); + return false; + } + + // check for remote profile + $remote_pro = Remote_profile::staticGet('uri', $profileurl); + + if (!$remote_pro) { + + $remote_pro = new Remote_profile(); + + $remote_pro->id = $id; + $remote_pro->uri = $profileurl; + $remote_pro->created = common_sql_now(); + + $rid = $remote_pro->insert(); + + if (empty($rid)) { + common_log_db_error($profile, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print 'Could not insert Remote_profile: ' . + common_log_objstring($remote_pro) . "\n"; + } + $profile->query("ROLLBACK"); + return false; + } + } + + $profile->query("COMMIT"); + + $this->saveAvatars($user, $id); + + return $id; + } + } + + function checkAvatar($user, $profile) + { + global $config; + + $path_parts = pathinfo($user->profile_image_url); + $newname = 'Twitter_' . $user->id . '_' . + $path_parts['basename']; + + $oldname = $profile->getAvatar(48)->filename; + + if ($newname != $oldname) { + + common_debug("Avatar for Twitter user $profile->nickname has changed."); + common_debug("old: $oldname new: $newname"); + + if (defined('SCRIPT_DEBUG')) { + print "Avatar for Twitter user $user->id has changed.\n"; + print "old: $oldname\n"; + print "new: $newname\n"; + } + + $img_root = substr($path_parts['basename'], 0, -11); + $ext = $path_parts['extension']; + $mediatype = $this->getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $user->id . '_' . + $img_root . "_$size.$ext"; + + if ($this->fetchAvatar($url, $filename)) { + $this->updateAvatar($profile->id, $size, $mediatype, $filename); + } + } + } + } + + function getMediatype($ext) + { + $mediatype = null; + + switch (strtolower($ext)) { + case 'jpg': + $mediatype = 'image/jpg'; + break; + case 'gif': + $mediatype = 'image/gif'; + break; + default: + $mediatype = 'image/png'; + } + + return $mediatype; + } + + function saveAvatars($user, $id) + { + global $config; + + $path_parts = pathinfo($user->profile_image_url); + $ext = $path_parts['extension']; + $end = strlen('_normal' . $ext); + $img_root = substr($path_parts['basename'], 0, -($end+1)); + $mediatype = $this->getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $user->id . '_' . + $img_root . "_$size.$ext"; + + if ($this->fetchAvatar($url, $filename)) { + $this->newAvatar($id, $size, $mediatype, $filename); + } else { + common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Problem fetching Avatar: $url\n"; + } + } + } + } + + function updateAvatar($profile_id, $size, $mediatype, $filename) { + + common_debug("Updating avatar: $size"); + if (defined('SCRIPT_DEBUG')) { + print "Updating avatar: $size\n"; + } + + $profile = Profile::staticGet($profile_id); + + if (!$profile) { + common_debug("Couldn't get profile: $profile_id!"); + if (defined('SCRIPT_DEBUG')) { + print "Couldn't get profile: $profile_id!\n"; + } + return; + } + + $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); + $avatar = $profile->getAvatar($sizes[$size]); + + if ($avatar) { + common_debug("Deleting $size avatar for $profile->nickname."); + @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); + $avatar->delete(); + } + + $this->newAvatar($profile->id, $size, $mediatype, $filename); + } + + function newAvatar($profile_id, $size, $mediatype, $filename) + { + global $config; + + $avatar = new Avatar(); + $avatar->profile_id = $profile_id; + + switch($size) { + case 'mini': + $avatar->width = 24; + $avatar->height = 24; + break; + case 'normal': + $avatar->width = 48; + $avatar->height = 48; + break; + default: + + // Note: Twitter's big avatars are a different size than + // Laconica's (Laconica's = 96) + + $avatar->width = 73; + $avatar->height = 73; + } + + $avatar->original = 0; // we don't have the original + $avatar->mediatype = $mediatype; + $avatar->filename = $filename; + $avatar->url = Avatar::url($filename); + + common_debug("new filename: $avatar->url"); + if (defined('SCRIPT_DEBUG')) { + print "New filename: $avatar->url\n"; + } + + $avatar->created = common_sql_now(); + + $id = $avatar->insert(); + + if (!$id) { + common_log_db_error($avatar, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Could not insert avatar!\n"; + } + + return null; + } + + common_debug("Saved new $size avatar for $profile_id."); + if (defined('SCRIPT_DEBUG')) { + print "Saved new $size avatar for $profile_id.\n"; + } + + return $id; + } + + function fetchAvatar($url, $filename) + { + $avatar_dir = INSTALLDIR . '/avatar/'; + + $avatarfile = $avatar_dir . $filename; + + $out = fopen($avatarfile, 'wb'); + if (!$out) { + common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Couldn't open file! $filename\n"; + } + return false; + } + + common_debug("Fetching avatar: $url", __FILE__); + if (defined('SCRIPT_DEBUG')) { + print "Fetching avatar from Twitter: $url\n"; + } + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_FILE, $out); + curl_setopt($ch, CURLOPT_BINARYTRANSFER, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); + $result = curl_exec($ch); + curl_close($ch); + + fclose($out); + + return $result; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); +declare(ticks = 1); + +$fetcher = new TwitterStatusFetcher(); +$fetcher->runOnce(); + -- cgit v1.2.3-54-g00ecf From 856e05a08ff8d09fbd580ed35906e3dda0475a0a Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 7 May 2009 01:10:31 -0700 Subject: Less pychotic debugging statements --- scripts/twitterstatusfetcher.php | 140 +++++++++++++++------------------------ 1 file changed, 52 insertions(+), 88 deletions(-) mode change 100644 => 100755 scripts/twitterstatusfetcher.php (limited to 'scripts') diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php old mode 100644 new mode 100755 index 5275a4575..e8819f665 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -72,10 +72,8 @@ class TwitterStatusFetcher extends Daemon if ($pid) { // Parent - common_debug("Parent: forked new status fetcher process " . $pid); - if (defined('SCRIPT_DEBUG')) { - print "Parent: forked fetcher process " . $pid . "\n"; + common_debug("Parent: forked new status fetcher process " . $pid); } $this->children[] = $pid; @@ -90,10 +88,8 @@ class TwitterStatusFetcher extends Daemon // Remove child from ps list as it finishes while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { - common_debug("Child $c finished."); - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; + common_debug("Child $c finished."); } $this->remove_ps($this->children, $c); @@ -102,18 +98,14 @@ class TwitterStatusFetcher extends Daemon // Wait! We have too many damn kids. if (sizeof($this->children) > MAXCHILDREN) { - common_debug('Too many children. Waiting...'); - if (defined('SCRIPT_DEBUG')) { - print "Too many children. Waiting...\n"; + common_debug('Too many children. Waiting...'); } if (($c = pcntl_wait($status, WUNTRACED)) > 0){ - common_debug("Finished waiting for $c"); - if (defined('SCRIPT_DEBUG')) { - print "Finished waiting for $c\n"; + common_debug("Finished waiting for $c"); } $this->remove_ps($this->children, $c); @@ -124,21 +116,18 @@ class TwitterStatusFetcher extends Daemon // Remove all children from the process list before restarting while(($c = pcntl_wait($status, WUNTRACED)) > 0) { - common_debug("Child $c finished."); - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; + common_debug("Child $c finished."); } $this->remove_ps($this->children, $c); } // Rest for a bit before we fetch more statuses - common_debug('Waiting ' . POLL_INTERVAL . - ' secs before hitting Twitter again.'); + if (defined('SCRIPT_DEBUG')) { - print 'Waiting ' . POLL_INTERVAL . - " secs before hitting Twitter again.\n"; + common_debug('Waiting ' . POLL_INTERVAL . + ' secs before hitting Twitter again.'); } sleep(POLL_INTERVAL); @@ -155,14 +144,16 @@ class TwitterStatusFetcher extends Daemon $cnt = $flink->find(); if (defined('SCRIPT_DEBUG')) { - print "Updating Twitter friends subscriptions for $cnt users.\n"; + common_debug('Updating Twitter friends subscriptions' . + " for $cnt users."); } $flinks = array(); while ($flink->fetch()) { - if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { + if (($flink->noticesync & FOREIGN_NOTICE_RECV) == + FOREIGN_NOTICE_RECV) { $flinks[] = clone($flink); } } @@ -187,30 +178,28 @@ class TwitterStatusFetcher extends Daemon { if (empty($flink)) { - common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); - if (defined('SCRIPT_DEBUG')) { - print "Can't retrieve Foreign_link for foreign ID $fid\n"; - } + common_log(LOG_WARNING, + "Can't retrieve Foreign_link for foreign ID $fid"); return; } $fuser = $flink->getForeignUser(); if (empty($fuser)) { - common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); - if (defined('SCRIPT_DEBUG')) { - print "Unmatched user for ID $flink->user_id\n"; - } + common_log(LOG_WARNING, "Unmatched user for ID " . + $flink->user_id); return; } - common_debug('Trying to get timeline for Twitter user ' . - "$fuser->nickname ($flink->foreign_id)."); if (defined('SCRIPT_DEBUG')) { - print 'Trying to get timeline for Twitter user ' . - "$fuser->nickname ($flink->foreign_id).\n"; + common_debug('Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id)."); } + // XXX: Biggest remaining issue - How do we know at which status + // to start importing? How many statuses? Right now I'm going + // with the default last 20. + $url = 'http://twitter.com/statuses/friends_timeline.json'; $timeline_json = get_twitter_data($url, $fuser->nickname, @@ -220,18 +209,19 @@ class TwitterStatusFetcher extends Daemon if (empty($timeline)) { common_log(LOG_WARNING, "Empty timeline."); - if (defined('SCRIPT_DEBUG')) { - print "Empty timeline!\n"; - } return; } foreach ($timeline as $status) { - // Hacktastic: filter out stuff coming from Laconica + // Hacktastic: filter out stuff coming from this Laconica $source = mb_strtolower(common_config('integration', 'source')); if (preg_match("/$source/", mb_strtolower($status->source))) { + if (defined('SCRIPT_DEBUG')) { + common_debug('Skipping import of status ' . $status->id . + ' with source ' . $source); + } continue; } @@ -239,7 +229,6 @@ class TwitterStatusFetcher extends Daemon } // Okay, record the time we synced with Twitter for posterity - $flink->last_noticesync = common_sql_now(); $flink->update(); } @@ -250,18 +239,14 @@ class TwitterStatusFetcher extends Daemon $profile = Profile::staticGet($id); if (!$profile) { - common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); - if (defined('SCRIPT_DEBUG')) { - print "Problem saving notice. No associated Profile.\n"; - } + common_log(LOG_ERR, + 'Problem saving notice. No associated Profile.'); return null; } $uri = 'http://twitter.com/' . $status->user->screen_name . '/status/' . $status->id; - // Skip save if notice source is Laconica or Identi.ca? - $notice = Notice::staticGet('uri', $uri); // check to see if we've already imported the status @@ -290,21 +275,23 @@ class TwitterStatusFetcher extends Daemon if (!$notice_id) { common_log_db_error($notice, 'INSERT', __FILE__); if (defined('SCRIPT_DEBUG')) { - print "Could not save notice!\n"; + common_debug('Could not save notice!'); } } - // XXX: Figure out a better way to link replies? + // XXX: Figure out a better way to link Twitter replies? $notice->saveReplies(); - // XXX: Do we want to polute our tag cloud with hashtags from Twitter? + // XXX: Do we want to polute our tag cloud with + // hashtags from Twitter? $notice->saveTags(); $notice->saveGroups(); $notice->query('COMMIT'); if (defined('SCRIPT_DEBUG')) { - print "Saved status $status->id as notice $notice->id.\n"; + common_debug("Saved status $status->id" . + " as notice $notice->id."); } } @@ -327,18 +314,19 @@ class TwitterStatusFetcher extends Daemon $profile = Profile::staticGet('profileurl', $profileurl); if ($profile) { - common_debug("Profile for $profile->nickname found."); + if (defined('SCRIPT_DEBUG')) { + common_debug("Profile for $profile->nickname found."); + } // Check to see if the user's Avatar has changed $this->checkAvatar($user, $profile); + return $profile->id; } else { - $debugmsg = 'Adding profile and remote profile ' . - "for Twitter user: $profileurl\n"; - common_debug($debugmsg, __FILE__); if (defined('SCRIPT_DEBUG')) { - print $debugmsg; + common_debug('Adding profile and remote profile ' . + "for Twitter user: $profileurl"); } $profile = new Profile(); @@ -356,10 +344,6 @@ class TwitterStatusFetcher extends Daemon if (empty($id)) { common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Profile: ' . - common_log_objstring($profile) . "\n"; - } $profile->query("ROLLBACK"); return false; } @@ -379,10 +363,6 @@ class TwitterStatusFetcher extends Daemon if (empty($rid)) { common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Remote_profile: ' . - common_log_objstring($remote_pro) . "\n"; - } $profile->query("ROLLBACK"); return false; } @@ -408,13 +388,10 @@ class TwitterStatusFetcher extends Daemon if ($newname != $oldname) { - common_debug("Avatar for Twitter user $profile->nickname has changed."); - common_debug("old: $oldname new: $newname"); - if (defined('SCRIPT_DEBUG')) { - print "Avatar for Twitter user $user->id has changed.\n"; - print "old: $oldname\n"; - print "new: $newname\n"; + common_debug('Avatar for Twitter user ' . + "$profile->nickname has changed."); + common_debug("old: $oldname new: $newname"); } $img_root = substr($path_parts['basename'], 0, -11); @@ -472,26 +449,21 @@ class TwitterStatusFetcher extends Daemon $this->newAvatar($id, $size, $mediatype, $filename); } else { common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Problem fetching Avatar: $url\n"; - } } } } function updateAvatar($profile_id, $size, $mediatype, $filename) { - common_debug("Updating avatar: $size"); if (defined('SCRIPT_DEBUG')) { - print "Updating avatar: $size\n"; + common_debug("Updating avatar: $size"); } $profile = Profile::staticGet($profile_id); if (!$profile) { - common_debug("Couldn't get profile: $profile_id!"); if (defined('SCRIPT_DEBUG')) { - print "Couldn't get profile: $profile_id!\n"; + common_debug("Couldn't get profile: $profile_id!"); } return; } @@ -500,7 +472,9 @@ class TwitterStatusFetcher extends Daemon $avatar = $profile->getAvatar($sizes[$size]); if ($avatar) { - common_debug("Deleting $size avatar for $profile->nickname."); + if (defined('SCRIPT_DEBUG')) { + common_debug("Deleting $size avatar for $profile->nickname."); + } @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); $avatar->delete(); } @@ -538,9 +512,8 @@ class TwitterStatusFetcher extends Daemon $avatar->filename = $filename; $avatar->url = Avatar::url($filename); - common_debug("new filename: $avatar->url"); if (defined('SCRIPT_DEBUG')) { - print "New filename: $avatar->url\n"; + common_debug("new filename: $avatar->url"); } $avatar->created = common_sql_now(); @@ -549,16 +522,11 @@ class TwitterStatusFetcher extends Daemon if (!$id) { common_log_db_error($avatar, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Could not insert avatar!\n"; - } - return null; } - common_debug("Saved new $size avatar for $profile_id."); if (defined('SCRIPT_DEBUG')) { - print "Saved new $size avatar for $profile_id.\n"; + common_debug("Saved new $size avatar for $profile_id."); } return $id; @@ -573,15 +541,11 @@ class TwitterStatusFetcher extends Daemon $out = fopen($avatarfile, 'wb'); if (!$out) { common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Couldn't open file! $filename\n"; - } return false; } - common_debug("Fetching avatar: $url", __FILE__); if (defined('SCRIPT_DEBUG')) { - print "Fetching avatar from Twitter: $url\n"; + common_debug("Fetching avatar: $url"); } $ch = curl_init(); -- cgit v1.2.3-54-g00ecf From bc190595d1dfd56bf7e68597b3d574909eb27260 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 7 May 2009 02:07:31 -0700 Subject: Added TwitterStatusFetcher into daemon startup and shutdown subsystem --- config.php.sample | 3 +++ lib/common.php | 2 ++ scripts/getvaliddaemons.php | 3 +++ scripts/stopdaemons.sh | 2 +- scripts/twitterstatusfetcher.php | 10 ++++++---- 5 files changed, 15 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/config.php.sample b/config.php.sample index b8ed45fa8..6d6a9b533 100644 --- a/config.php.sample +++ b/config.php.sample @@ -150,6 +150,9 @@ $config['sphinx']['port'] = 3312; #$config['memcached']['server'] = 'localhost'; #$config['memcached']['port'] = 11211; +# Enable bidirectional Twitter bridge +#$config['twitterbridge']['enabled'] = true; + #Twitter integration source attribute. Note: default is Laconica #$config['integration']['source'] = 'Laconica'; diff --git a/lib/common.php b/lib/common.php index 00e5b0bc2..abdc22c0e 100644 --- a/lib/common.php +++ b/lib/common.php @@ -143,6 +143,8 @@ $config = array('piddir' => '/var/run', 'user' => false, 'group' => false), + 'twitterbridge' => + array('enabled' => false), 'integration' => array('source' => 'Laconica', # source attribute for Twitter 'taguri' => $_server.',2009'), # base for tag URIs diff --git a/scripts/getvaliddaemons.php b/scripts/getvaliddaemons.php index 482e63af7..a10233e69 100755 --- a/scripts/getvaliddaemons.php +++ b/scripts/getvaliddaemons.php @@ -44,6 +44,9 @@ if(common_config('xmpp','enabled')) { if(common_config('memcached','enabled')) { echo "memcachedqueuehandler.php "; } +if(common_config('twitterbridge','enabled')) { + echo "twitterstatusfetcher.php "; +} echo "ombqueuehandler.php "; echo "twitterqueuehandler.php "; echo "facebookqueuehandler.php "; diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index f6d71eddf..764037e8f 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -25,7 +25,7 @@ DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ - memcachehandler inboxhandler; do + memcachehandler inboxhandler twitterstatusfetcher; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index e8819f665..9dfadc760 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -32,7 +32,7 @@ define('LACONICA', true); define('MAXCHILDREN', 2); define('POLL_INTERVAL', 60); // in seconds -// Uncomment this to get useful console output +// Uncomment this to get useful logging define('SCRIPT_DEBUG', true); require_once(INSTALLDIR . '/lib/common.php'); @@ -45,7 +45,7 @@ class TwitterStatusFetcher extends Daemon function name() { - return 'twitterstatusfetcher'; + return ('twitterstatusfetcher.generic'); } function run() @@ -130,7 +130,9 @@ class TwitterStatusFetcher extends Daemon ' secs before hitting Twitter again.'); } - sleep(POLL_INTERVAL); + if (POLL_INTERVAL > 0) { + sleep(POLL_INTERVAL); + } } while (true); } @@ -282,7 +284,7 @@ class TwitterStatusFetcher extends Daemon // XXX: Figure out a better way to link Twitter replies? $notice->saveReplies(); - // XXX: Do we want to polute our tag cloud with + // XXX: Do we want to pollute our tag cloud with // hashtags from Twitter? $notice->saveTags(); $notice->saveGroups(); -- cgit v1.2.3-54-g00ecf From 11e0db8c2cec18337fd960ccda055dd14d89f9d7 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 7 May 2009 18:22:14 -0700 Subject: Twitter friends sync now does 25 users at a time and uses last_friendsync field to prioritize --- actions/twittersettings.php | 4 +++- scripts/synctwitterfriends.php | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/actions/twittersettings.php b/actions/twittersettings.php index 45725d3ff..0b98eef59 100644 --- a/actions/twittersettings.php +++ b/actions/twittersettings.php @@ -261,7 +261,7 @@ class TwittersettingsAction extends ConnectSettingsAction 'alt' => ($other->fullname) ? $other->fullname : $other->nickname)); - + $this->element('span', 'fn nickname', $other->nickname); $this->elementEnd('a'); $this->elementEnd('li'); @@ -375,6 +375,8 @@ class TwittersettingsAction extends ConnectSettingsAction if ($friendsync) { save_twitter_friends($user, $twit_user->id, $screen_name, $password); + $flink->last_friendsync = common_sql_now(); + $flink->update(); } $this->showForm(_('Twitter settings saved.'), true); diff --git a/scripts/synctwitterfriends.php b/scripts/synctwitterfriends.php index 794301f0f..bd08ba58d 100755 --- a/scripts/synctwitterfriends.php +++ b/scripts/synctwitterfriends.php @@ -32,8 +32,25 @@ define('LACONICA', true); require_once(INSTALLDIR . '/lib/common.php'); +// Make a lockfile +$lockfilename = lockFilename(); +if (!($lockfile = @fopen($lockfilename, "w"))) { + print "Already running... exiting.\n"; + exit(1); +} + +// Obtain an exlcusive lock on file (will fail if script is already going) +if (!@flock( $lockfile, LOCK_EX | LOCK_NB, &$wouldblock) || $wouldblock) { + // Script already running - abort + @fclose($lockfile); + print "Already running... exiting.\n"; + exit(1); +} + $flink = new Foreign_link(); $flink->service = 1; // Twitter +$flink->orderBy('last_friendsync'); +$flink->limit(25); // sync this many users during this run $cnt = $flink->find(); print "Updating Twitter friends subscriptions for $cnt users.\n"; @@ -60,8 +77,11 @@ while ($flink->fetch()) { continue; } - $result = save_twitter_friends($user, $fuser->id, - $fuser->nickname, $flink->credentials); + save_twitter_friends($user, $fuser->id, $fuser->nickname, $flink->credentials); + + $flink->last_friendsync = common_sql_now(); + $flink->update(); + if (defined('SCRIPT_DEBUG')) { print "\nDONE\n"; } else { @@ -70,4 +90,18 @@ while ($flink->fetch()) { } } +function lockFilename() +{ + $piddir = common_config('daemon', 'piddir'); + if (!$piddir) { + $piddir = '/var/run'; + } + + return $piddir . '/synctwitterfriends.lock'; +} + +// Cleanup +fclose($lockfile); +unlink($lockfilename); + exit(0); -- cgit v1.2.3-54-g00ecf From 1a5ef90ea0fbdfa01c47671980402d0264839f27 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Wed, 20 May 2009 22:08:08 +0000 Subject: Preserve order of imported Tweets --- scripts/twitterstatusfetcher.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index 9dfadc760..a61ce1b0d 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -214,7 +214,8 @@ class TwitterStatusFetcher extends Daemon return; } - foreach ($timeline as $status) { + // Reverse to preserve order + foreach (array_reverse($timeline) as $status) { // Hacktastic: filter out stuff coming from this Laconica $source = mb_strtolower(common_config('integration', 'source')); -- cgit v1.2.3-54-g00ecf