diff options
Diffstat (limited to 'scripts/twitterstatusfetcher.php')
| -rwxr-xr-x | scripts/twitterstatusfetcher.php | 208 |
1 files changed, 146 insertions, 62 deletions
diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index 9287b6d73..5ffdda58f 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -1,8 +1,8 @@ #!/usr/bin/env php <?php -/* +/** * Laconica - a distributed open-source microblogging tool - * Copyright (C) 2008, Controlez-Vous, Inc. + * Copyright (C) 2008, 2009, Control Yourself, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -18,43 +18,68 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -// Abort if called from a web server -if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { - print "This script must be run from the command line\n"; - exit(); -} - define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); -define('LACONICA', true); // Tune number of processes and how often to poll Twitter // XXX: Should these things be in config.php? define('MAXCHILDREN', 2); define('POLL_INTERVAL', 60); // in seconds -// Uncomment this to get useful logging -define('SCRIPT_DEBUG', true); +$helptext = <<<END_OF_TRIM_HELP +Batch script for retrieving Twitter messages from foreign service. + +END_OF_TRIM_HELP; + +require_once INSTALLDIR.'/scripts/commandline.inc'; + +require_once INSTALLDIR . '/lib/common.php'; +require_once INSTALLDIR . '/lib/daemon.php'; + +/** + * Fetcher for statuses from Twitter + * + * Fetches statuses from Twitter and inserts them as notices in local + * system. + * + * @category Twitter + * @package Laconica + * @author Zach Copley <zach@controlyourself.ca> + * @author Evan Prodromou <evan@controlyourself.ca> + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ -require_once(INSTALLDIR . '/lib/common.php'); -require_once(INSTALLDIR . '/lib/daemon.php'); +// NOTE: an Avatar path MUST be set in config.php for this +// script to work: e.g.: $config['avatar']['path'] = '/laconica/avatar'; class TwitterStatusFetcher extends Daemon { + private $_children = array(); - private $children = array(); + /** + * Name of this daemon + * + * @return string Name of the daemon. + */ function name() { return ('twitterstatusfetcher.generic'); } + /** + * Run the daemon + * + * @return void + */ + function run() { do { $flinks = $this->refreshFlinks(); - foreach ($flinks as $f){ + foreach ($flinks as $f) { // We have to disconnect from the DB before forking so // each sub-process will open its own connection and @@ -73,10 +98,11 @@ class TwitterStatusFetcher extends Daemon // Parent if (defined('SCRIPT_DEBUG')) { - common_debug("Parent: forked new status fetcher process " . $pid); + common_debug("Parent: forked new status ". + " fetcher process " . $pid); } - $this->children[] = $pid; + $this->_children[] = $pid; } else { @@ -86,41 +112,41 @@ class TwitterStatusFetcher extends Daemon } // Remove child from ps list as it finishes - while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { + while (($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { if (defined('SCRIPT_DEBUG')) { common_debug("Child $c finished."); } - $this->remove_ps($this->children, $c); + $this->removePs($this->_children, $c); } // Wait! We have too many damn kids. - if (sizeof($this->children) > MAXCHILDREN) { + if (sizeof($this->_children) > MAXCHILDREN) { if (defined('SCRIPT_DEBUG')) { common_debug('Too many children. Waiting...'); } - if (($c = pcntl_wait($status, WUNTRACED)) > 0){ + if (($c = pcntl_wait($status, WUNTRACED)) > 0) { if (defined('SCRIPT_DEBUG')) { common_debug("Finished waiting for $c"); } - $this->remove_ps($this->children, $c); + $this->removePs($this->_children, $c); } } } // Remove all children from the process list before restarting - while(($c = pcntl_wait($status, WUNTRACED)) > 0) { + while (($c = pcntl_wait($status, WUNTRACED)) > 0) { if (defined('SCRIPT_DEBUG')) { common_debug("Child $c finished."); } - $this->remove_ps($this->children, $c); + $this->removePs($this->_children, $c); } // Rest for a bit before we fetch more statuses @@ -137,10 +163,18 @@ class TwitterStatusFetcher extends Daemon } while (true); } - function refreshFlinks() { + /** + * Refresh the foreign links for this user + * + * @return void + */ + function refreshFlinks() + { $flink = new Foreign_link(); + $flink->service = 1; // Twitter + $flink->orderBy('last_noticesync'); $cnt = $flink->find(); @@ -166,7 +200,18 @@ class TwitterStatusFetcher extends Daemon return $flinks; } - function remove_ps(&$plist, $ps){ + /** + * Unknown + * + * @param array &$plist unknown. + * @param string $ps unknown. + * + * @return unknown + * @todo document + */ + + function removePs(&$plist, $ps) + { for ($i = 0; $i < sizeof($plist); $i++) { if ($plist[$i] == $ps) { unset($plist[$i]); @@ -178,7 +223,6 @@ class TwitterStatusFetcher extends Daemon function getTimeline($flink) { - if (empty($flink)) { common_log(LOG_WARNING, "Can't retrieve Foreign_link for foreign ID $fid"); @@ -247,23 +291,32 @@ class TwitterStatusFetcher extends Daemon return null; } + // XXX: change of screen name? + $uri = 'http://twitter.com/' . $status->user->screen_name . '/status/' . $status->id; $notice = Notice::staticGet('uri', $uri); // check to see if we've already imported the status - if (!$notice) { - - $created = strftime('%Y-%m-%d %H:%M:%S', - strtotime($status->created_at));; - $notice = Notice::saveNew($id, $status->text, 'twitter', - -2, null, $uri, $created); + if (!$notice) { - if (defined('SCRIPT_DEBUG')) { - common_debug("Saved status $status->id" . - " as notice $notice->id."); + $notice = new Notice(); + + $notice->profile_id = $id; + $notice->uri = $uri; + $notice->created = strftime('%Y-%m-%d %H:%M:%S', + strtotime($status->created_at)); + $notice->content = common_shorten_links($status->text); // XXX + $notice->rendered = common_render_content($notice->content, $notice); + $notice->source = 'twitter'; + $notice->reply_to = null; // XXX lookup reply + $notice->is_local = NOTICE_GATEWAY; + + if (Event::handle('StartNoticeSave', array(&$notice))) { + $id = $notice->insert(); + Event::handle('EndNoticeSave', array($notice)); } } @@ -271,9 +324,11 @@ class TwitterStatusFetcher extends Daemon 'user_id' => $flink->user_id))) { // Add to inbox $inbox = new Notice_inbox(); - $inbox->user_id = $flink->user_id; + + $inbox->user_id = $flink->user_id; $inbox->notice_id = $notice->id; - $inbox->created = $notice->created; + $inbox->created = $notice->created; + $inbox->source = NOTICE_INBOX_SOURCE_GATEWAY; // From a private source $inbox->insert(); } @@ -348,12 +403,13 @@ class TwitterStatusFetcher extends Daemon } } - function checkAvatar($user, $profile) + function checkAvatar($twitter_user, $profile) { global $config; - $path_parts = pathinfo($user->profile_image_url); - $newname = 'Twitter_' . $user->id . '_' . + $path_parts = pathinfo($twitter_user->profile_image_url); + + $newname = 'Twitter_' . $twitter_user->id . '_' . $path_parts['basename']; $oldname = $profile->getAvatar(48)->filename; @@ -366,21 +422,56 @@ class TwitterStatusFetcher extends Daemon common_debug("old: $oldname new: $newname"); } - $img_root = substr($path_parts['basename'], 0, -11); - $ext = $path_parts['extension']; - $mediatype = $this->getMediatype($ext); + $this->updateAvatars($twitter_user, $profile); + } - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; + if ($this->missingAvatarFile($profile)) { - if ($this->fetchAvatar($url, $filename)) { - $this->updateAvatar($profile->id, $size, $mediatype, $filename); - } + if (defined('SCRIPT_DEBUG')) { + common_debug('Twitter user ' . $profile->nickname . + ' is missing one or more local avatars.'); + common_debug("old: $oldname new: $newname"); } + + $this->updateAvatars($twitter_user, $profile); } + + } + + function updateAvatars($twitter_user, $profile) { + + global $config; + + $path_parts = pathinfo($twitter_user->profile_image_url); + + $img_root = substr($path_parts['basename'], 0, -11); + $ext = $path_parts['extension']; + $mediatype = $this->getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $twitter_user->id . '_' . + $img_root . "_$size.$ext"; + + $this->updateAvatar($profile->id, $size, $mediatype, $filename); + $this->fetchAvatar($url, $filename); + } + } + + function missingAvatarFile($profile) { + + foreach (array(24, 48, 73) as $size) { + + $filename = $profile->getAvatar($size)->filename; + $avatarpath = Avatar::path($filename); + + if (file_exists($avatarpath) == FALSE) { + return true; + } + } + + return false; } function getMediatype($ext) @@ -433,7 +524,7 @@ class TwitterStatusFetcher extends Daemon $profile = Profile::staticGet($profile_id); - if (!$profile) { + if (empty($profile)) { if (defined('SCRIPT_DEBUG')) { common_debug("Couldn't get profile: $profile_id!"); } @@ -443,11 +534,8 @@ class TwitterStatusFetcher extends Daemon $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); $avatar = $profile->getAvatar($sizes[$size]); + // Delete the avatar, if present if ($avatar) { - if (defined('SCRIPT_DEBUG')) { - common_debug("Deleting $size avatar for $profile->nickname."); - } - @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); $avatar->delete(); } @@ -492,7 +580,7 @@ class TwitterStatusFetcher extends Daemon $id = $avatar->insert(); - if (!$id) { + if (empty($id)) { common_log_db_error($avatar, 'INSERT', __FILE__); return null; } @@ -535,10 +623,6 @@ class TwitterStatusFetcher extends Daemon } } -ini_set("max_execution_time", "0"); -ini_set("max_input_time", "0"); -set_time_limit(0); -mb_internal_encoding('UTF-8'); declare(ticks = 1); $fetcher = new TwitterStatusFetcher(); |