diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/getvaliddaemons.php | 55 | ||||
| -rwxr-xr-x | scripts/inboxqueuehandler.php | 69 | ||||
| -rwxr-xr-x | scripts/memcachedqueuehandler.php | 70 | ||||
| -rwxr-xr-x | scripts/startdaemons.sh | 5 | ||||
| -rw-r--r-- | scripts/statusfetcher.php | 525 | ||||
| -rwxr-xr-x | scripts/stopdaemons.sh | 3 | ||||
| -rwxr-xr-x | scripts/synctwitterfriends.php | 38 | ||||
| -rwxr-xr-x | scripts/twitterstatusfetcher.php | 576 | ||||
| -rwxr-xr-x | scripts/xmppdaemon.php | 17 |
9 files changed, 815 insertions, 543 deletions
diff --git a/scripts/getvaliddaemons.php b/scripts/getvaliddaemons.php new file mode 100755 index 000000000..a10233e69 --- /dev/null +++ b/scripts/getvaliddaemons.php @@ -0,0 +1,55 @@ +#!/usr/bin/env php +<?php +/* + * Laconica - a distributed open-source microblogging tool + * Copyright (C) 2008, Controlez-Vous, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Utility script to get a list of daemons that should run, based on the + * current configuration. This is used by startdaemons.sh to determine what + * it should and shouldn't start up. The output is a list of space-separated + * daemon names. + */ + + +# Abort if called from a web server +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); + +if(common_config('xmpp','enabled')) { + echo "xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php "; + echo "xmppconfirmhandler.php "; +} +if(common_config('memcached','enabled')) { + echo "memcachedqueuehandler.php "; +} +if(common_config('twitterbridge','enabled')) { + echo "twitterstatusfetcher.php "; +} +echo "ombqueuehandler.php "; +echo "twitterqueuehandler.php "; +echo "facebookqueuehandler.php "; +echo "pingqueuehandler.php "; +echo "inboxqueuehandler.php "; +echo "smsqueuehandler.php "; diff --git a/scripts/inboxqueuehandler.php b/scripts/inboxqueuehandler.php new file mode 100755 index 000000000..73d31e854 --- /dev/null +++ b/scripts/inboxqueuehandler.php @@ -0,0 +1,69 @@ +#!/usr/bin/env php +<?php +/* + * Laconica - a distributed open-source microblogging tool + * Copyright (C) 2008,2009 Control Yourself, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); +require_once(INSTALLDIR . '/lib/queuehandler.php'); + +set_error_handler('common_error_handler'); + +class InboxQueueHandler extends QueueHandler +{ + function transport() + { + return 'inbox'; + } + + function start() { + $this->log(LOG_INFO, "INITIALIZE"); + return true; + } + + function handle_notice($notice) + { + $this->log(LOG_INFO, "Distributing notice to inboxes for $notice->id"); + $notice->addToInboxes(); + $notice->blowSubsCache(); + return true; + } + + function finish() { + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$id = ($argc > 1) ? $argv[1] : null; + +$handler = new InboxQueueHandler($id); + +$handler->runOnce(); diff --git a/scripts/memcachedqueuehandler.php b/scripts/memcachedqueuehandler.php new file mode 100755 index 000000000..185b781f7 --- /dev/null +++ b/scripts/memcachedqueuehandler.php @@ -0,0 +1,70 @@ +#!/usr/bin/env php +<?php +/* + * Laconica - a distributed open-source microblogging tool + * Copyright (C) 2008,2009 Control Yourself, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +// Abort if called from a web server + +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); +require_once(INSTALLDIR . '/lib/queuehandler.php'); + +set_error_handler('common_error_handler'); + +class MemcachedQueueHandler extends QueueHandler +{ + function transport() + { + return 'memcache'; + } + + function start() { + $this->log(LOG_INFO, "INITIALIZE"); + return true; + } + + function handle_notice($notice) + { + // XXX: fork here + $this->log(LOG_INFO, "Blowing memcached for $notice->id"); + $notice->blowCaches(); + return true; + } + + function finish() { + } + +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +$id = ($argc > 1) ? $argv[1] : null; + +$handler = new MemcachedQueueHandler($id); + +$handler->runOnce(); diff --git a/scripts/startdaemons.sh b/scripts/startdaemons.sh index c3729761d..3869e95c4 100755 --- a/scripts/startdaemons.sh +++ b/scripts/startdaemons.sh @@ -21,10 +21,9 @@ # Note that the 'maildaemon' needs to run as a mail filter. DIR=`dirname $0` +DAEMONS=`php $DIR/getvaliddaemons.php` -for f in xmppdaemon.php jabberqueuehandler.php publicqueuehandler.php \ - xmppconfirmhandler.php smsqueuehandler.php ombqueuehandler.php \ - twitterqueuehandler.php facebookqueuehandler.php pingqueuehandler.php; do +for f in $DAEMONS; do echo -n "Starting $f..."; php $DIR/$f diff --git a/scripts/statusfetcher.php b/scripts/statusfetcher.php deleted file mode 100644 index 8f4b60cf7..000000000 --- a/scripts/statusfetcher.php +++ /dev/null @@ -1,525 +0,0 @@ -#!/usr/bin/env php -<?php -/* - * Laconica - a distributed open-source microblogging tool - * Copyright (C) 2008, Controlez-Vous, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -// Abort if called from a web server -if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { - print "This script must be run from the command line\n"; - exit(); -} - -define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); -define('LACONICA', true); - -// Uncomment this to get useful console output -define('SCRIPT_DEBUG', true); - -require_once(INSTALLDIR . '/lib/common.php'); - -$children = array(); -$flink_ids = null; - -$MAXCHILDREN = 5; -$POLL_INTERVAL = 10; // 10 seconds - -do { - - $flink = new Foreign_link(); - $flink->service = 1; // Twitter - $cnt = $flink->find(); - - if (defined('SCRIPT_DEBUG')) { - print "Updating Twitter friends subscriptions for $cnt users.\n"; - } - - $flink_ids = array(); - - // XXX: This only reliably happens once. After the first interation of - // the do loop, the ->find() doesn't work ... lost DB connection? - - while ($flink->fetch()) { - - if (($flink->noticesync & FOREIGN_NOTICE_RECV) == FOREIGN_NOTICE_RECV) { - $flink_ids[] = $flink->foreign_id; - } - } - - $flink->free(); - unset($flink); - - foreach ($flink_ids as $f){ - - $pid = pcntl_fork(); - - if ($pid == -1) { - die ("Couldn't fork!"); - } - - // Parent - if ($pid) { - if (defined('SCRIPT_DEBUG')) { - print "Parent: forked " . $pid . "\n"; - } - $children[] = $pid; - } else { - - // Child - - // XXX: Each child needs its own DB connection - - getTimeline($f); - exit(); - } - - // Remove child from ps list as it finishes - while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; - } - remove_ps($children, $c); - } - - // Wait if we have too many kids - if(sizeof($children) > $MAXCHILDREN) { - if (defined('SCRIPT_DEBUG')) { - print "Too many children. Waiting...\n"; - } - if( ($c = pcntl_wait($status, WUNTRACED) ) > 0){ - if (defined('SCRIPT_DEBUG')) { - print "Finished waiting for $c\n"; - } - remove_ps($children, $c); - } - } - } - - // Remove all children from the process list before restarting - while(($c = pcntl_wait($status, WUNTRACED)) > 0) { - if (defined('SCRIPT_DEBUG')) { - print "Child $c finished.\n"; - } - remove_ps($children, $c); - } - - // Rest for a bit before we fetch more statuses - if (defined('SCRIPT_DEBUG')) { - print "Waiting $POLL_INTERVAL secs before hitting Twitter again.\n"; - } - - sleep($POLL_INTERVAL); - -} while (true); - - -function remove_ps(&$plist, $ps){ - for($i = 0; $i < sizeof($plist); $i++){ - if($plist[$i] == $ps){ - unset($plist[$i]); - $plist = array_values($plist); - break; - } - } -} - -function getTimeline($fid) -{ - - // XXX: Need to reconnect to the DB here? - - $flink = Foreign_link::getByForeignID($fid, 1); - $fuser = $flink->getForeignUser(); - - if (empty($fuser)) { - common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id); - if (defined('SCRIPT_DEBUG')) { - print "Unmatched user for ID $flink->user_id\n"; - } - } - - $screenname = $fuser->nickname; - - $url = 'http://twitter.com/statuses/friends_timeline.json'; - - $timeline_json = get_twitter_data($url, $fuser->nickname, - $flink->credentials); - - $timeline = json_decode($timeline_json); - - if (empty($timeline)) { - common_log(LOG_WARNING, "Empty timeline."); - if (defined('SCRIPT_DEBUG')) { - print "Empty timeline!\n"; - } - return; - } - - foreach ($timeline as $status) { - - // Hacktastic: filter out stuff coming from Laconica - $source = mb_strtolower(common_config('integration', 'source')); - - if (preg_match("/$source/", mb_strtolower($status->source))) { - continue; - } - - saveStatus($status, $flink); - } - -} - -function saveStatus($status, $flink) -{ - // Do we have a profile for this Twitter user? - - $id = ensureProfile($status->user); - $profile = Profile::staticGet($id); - - if (!$profile) { - common_log(LOG_ERR, 'Problem saving notice. No associated Profile.'); - if (defined('SCRIPT_DEBUG')) { - print "Problem saving notice. No associated Profile.\n"; - } - return null; - } - - $uri = 'http://twitter.com/' . $status->user->screen_name . - '/status/' . $status->id; - - // Skip save if notice source is Laconica or Identi.ca? - - $notice = Notice::staticGet('uri', $uri); - - // check to see if we've already imported the status - if (!$notice) { - - $notice = new Notice(); - $notice->profile_id = $id; - - $notice->query('BEGIN'); - - // XXX: figure out reply_to - $notice->reply_to = null; - - // XXX: Should this be common_sql_now() instead of status create date? - - $notice->created = strftime('%Y-%m-%d %H:%M:%S', - strtotime($status->created_at)); - $notice->content = $status->text; - $notice->rendered = common_render_content($status->text, $notice); - $notice->source = 'twitter'; - $notice->is_local = 0; - $notice->uri = $uri; - - $notice_id = $notice->insert(); - - if (!$notice_id) { - common_log_db_error($notice, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Could not save notice!\n"; - } - } - - // XXX: Figure out a better way to link replies? - $notice->saveReplies(); - - // XXX: Do we want to polute our tag cloud with hashtags from Twitter? - $notice->saveTags(); - $notice->saveGroups(); - - $notice->query('COMMIT'); - - } - - if (!Notice_inbox::staticGet('notice_id', $notice->id)) { - - // Add to inbox - $inbox = new Notice_inbox(); - $inbox->user_id = $flink->user_id; - $inbox->notice_id = $notice->id; - $inbox->created = common_sql_now(); - - $inbox->insert(); - } -} - -function ensureProfile($user) -{ - - // check to see if there's already a profile for this user - $profileurl = 'http://twitter.com/' . $user->screen_name; - $profile = Profile::staticGet('profileurl', $profileurl); - - if ($profile) { - common_debug("Profile for $profile->nickname found."); - - // Check to see if the user's Avatar has changed - checkAvatar($user, $profile); - return $profile->id; - - } else { - $debugmsg = 'Adding profile and remote profile ' . - "for Twitter user: $profileurl\n"; - common_debug($debugmsg, __FILE__); - if (defined('SCRIPT_DEBUG')) { - print $debugmsg; - } - - $profile = new Profile(); - $profile->query("BEGIN"); - - $profile->nickname = $user->screen_name; - $profile->fullname = $user->name; - $profile->homepage = $user->url; - $profile->bio = $user->description; - $profile->location = $user->location; - $profile->profileurl = $profileurl; - $profile->created = common_sql_now(); - - $id = $profile->insert(); - - if (empty($id)) { - common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Profile: ' . - common_log_objstring($profile) . "\n"; - } - $profile->query("ROLLBACK"); - return false; - } - - // check for remote profile - $remote_pro = Remote_profile::staticGet('uri', $profileurl); - - if (!$remote_pro) { - - $remote_pro = new Remote_profile(); - - $remote_pro->id = $id; - $remote_pro->uri = $profileurl; - $remote_pro->created = common_sql_now(); - - $rid = $remote_pro->insert(); - - if (empty($rid)) { - common_log_db_error($profile, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print 'Could not insert Remote_profile: ' . - common_log_objstring($remote_pro) . "\n"; - } - $profile->query("ROLLBACK"); - return false; - } - } - - $profile->query("COMMIT"); - $profile->free(); - unset($profile); - - saveAvatars($user, $id); - - return $id; - } -} - -function checkAvatar($user, $profile) -{ - $path_parts = pathinfo($user->profile_image_url); - $newname = 'Twitter_' . $user->id . '_' . - $path_parts['basename']; - - $oldname = $profile->getAvatar(48)->filename; - - if ($newname != $oldname) { - - common_debug("Avatar for Twitter user $profile->nickname has changed."); - common_debug("old: $oldname new: $newname"); - - if (defined('SCRIPT_DEBUG')) { - print "Avatar for Twitter user $user->id has changed.\n"; - print "old: $oldname\n"; - print "new: $newname\n"; - } - - $img_root = substr($path_parts['basename'], 0, -11); - $ext = $path_parts['extension']; - $mediatype = getMediatype($ext); - - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; - - if (fetchAvatar($url, $filename)) { - updateAvatar($profile->id, $size, $mediatype, $filename); - } - } - } -} - -function getMediatype($ext) -{ - $mediatype = null; - - switch (strtolower($ext)) { - case 'jpg': - $mediatype = 'image/jpg'; - break; - case 'gif': - $mediatype = 'image/gif'; - break; - default: - $mediatype = 'image/png'; - } - - return $mediatype; -} - -function saveAvatars($user, $id) -{ - $path_parts = pathinfo($user->profile_image_url); - $ext = $path_parts['extension']; - $end = strlen('_normal' . $ext); - $img_root = substr($path_parts['basename'], 0, -($end+1)); - $mediatype = getMediatype($ext); - - foreach (array('mini', 'normal', 'bigger') as $size) { - $url = $path_parts['dirname'] . '/' . - $img_root . '_' . $size . ".$ext"; - $filename = 'Twitter_' . $user->id . '_' . - $img_root . "_$size.$ext"; - - if (fetchAvatar($url, $filename)) { - newAvatar($id, $size, $mediatype, $filename); - } else { - common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Problem fetching Avatar: $url\n"; - } - } - } -} - -function updateAvatar($profile_id, $size, $mediatype, $filename) { - - common_debug("updating avatar: $size"); - - $profile = Profile::staticGet($profile_id); - - if (!$profile) { - common_debug("Couldn't get profile: $profile_id!"); - if (defined('SCRIPT_DEBUG')) { - print "Couldn't get profile: $profile_id!\n"; - } - return; - } - - $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); - $avatar = $profile->getAvatar($sizes[$size]); - - if ($avatar) { - common_debug("Deleting $size avatar for $profile->nickname."); - @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); - $avatar->delete(); - } - - newAvatar($profile->id, $size, $mediatype, $filename); -} - -function newAvatar($profile_id, $size, $mediatype, $filename) -{ - $avatar = new Avatar(); - $avatar->profile_id = $profile_id; - - switch($size) { - case 'mini': - $avatar->width = 24; - $avatar->height = 24; - break; - case 'normal': - $avatar->width = 48; - $avatar->height = 48; - break; - default: - - // Note: Twitter's big avatars are a different size than - // Laconica's (Laconica's = 96) - - $avatar->width = 73; - $avatar->height = 73; - } - - $avatar->original = 0; // we don't have the original - $avatar->mediatype = $mediatype; - $avatar->filename = $filename; - $avatar->url = Avatar::url($filename); - - common_debug("new filename: $avatar->url"); - - $avatar->created = common_sql_now(); - - $id = $avatar->insert(); - - if (!$id) { - common_log_db_error($avatar, 'INSERT', __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Could not insert avatar!\n"; - } - - return null; - } - - common_debug("Saved new $size avatar for $profile_id."); - - return $id; -} - -function fetchAvatar($url, $filename) -{ - $avatar_dir = INSTALLDIR . '/avatar/'; - - $avatarfile = $avatar_dir . $filename; - - $out = fopen($avatarfile, 'wb'); - if (!$out) { - common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Couldn't open file! $filename\n"; - } - return false; - } - - common_debug("Fetching avatar: $url", __FILE__); - if (defined('SCRIPT_DEBUG')) { - print "Fetching avatar from Twitter: $url\n"; - } - - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_FILE, $out); - curl_setopt($ch, CURLOPT_BINARYTRANSFER, true); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); - $result = curl_exec($ch); - curl_close($ch); - - fclose($out); - - return $result; -} diff --git a/scripts/stopdaemons.sh b/scripts/stopdaemons.sh index 2bb8f9ecb..764037e8f 100755 --- a/scripts/stopdaemons.sh +++ b/scripts/stopdaemons.sh @@ -24,7 +24,8 @@ SDIR=`dirname $0` DIR=`php $SDIR/getpiddir.php` for f in jabberhandler ombhandler publichandler smshandler pinghandler \ - xmppconfirmhandler xmppdaemon twitterhandler facebookhandler ; do + xmppconfirmhandler xmppdaemon twitterhandler facebookhandler \ + memcachehandler inboxhandler twitterstatusfetcher; do FILES="$DIR/$f.*.pid" for ff in "$FILES" ; do diff --git a/scripts/synctwitterfriends.php b/scripts/synctwitterfriends.php index 794301f0f..bd08ba58d 100755 --- a/scripts/synctwitterfriends.php +++ b/scripts/synctwitterfriends.php @@ -32,8 +32,25 @@ define('LACONICA', true); require_once(INSTALLDIR . '/lib/common.php'); +// Make a lockfile +$lockfilename = lockFilename(); +if (!($lockfile = @fopen($lockfilename, "w"))) { + print "Already running... exiting.\n"; + exit(1); +} + +// Obtain an exlcusive lock on file (will fail if script is already going) +if (!@flock( $lockfile, LOCK_EX | LOCK_NB, &$wouldblock) || $wouldblock) { + // Script already running - abort + @fclose($lockfile); + print "Already running... exiting.\n"; + exit(1); +} + $flink = new Foreign_link(); $flink->service = 1; // Twitter +$flink->orderBy('last_friendsync'); +$flink->limit(25); // sync this many users during this run $cnt = $flink->find(); print "Updating Twitter friends subscriptions for $cnt users.\n"; @@ -60,8 +77,11 @@ while ($flink->fetch()) { continue; } - $result = save_twitter_friends($user, $fuser->id, - $fuser->nickname, $flink->credentials); + save_twitter_friends($user, $fuser->id, $fuser->nickname, $flink->credentials); + + $flink->last_friendsync = common_sql_now(); + $flink->update(); + if (defined('SCRIPT_DEBUG')) { print "\nDONE\n"; } else { @@ -70,4 +90,18 @@ while ($flink->fetch()) { } } +function lockFilename() +{ + $piddir = common_config('daemon', 'piddir'); + if (!$piddir) { + $piddir = '/var/run'; + } + + return $piddir . '/synctwitterfriends.lock'; +} + +// Cleanup +fclose($lockfile); +unlink($lockfilename); + exit(0); diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php new file mode 100755 index 000000000..9dfadc760 --- /dev/null +++ b/scripts/twitterstatusfetcher.php @@ -0,0 +1,576 @@ +#!/usr/bin/env php +<?php +/* + * Laconica - a distributed open-source microblogging tool + * Copyright (C) 2008, Controlez-Vous, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +// Abort if called from a web server +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(); +} + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +// Tune number of processes and how often to poll Twitter +// XXX: Should these things be in config.php? +define('MAXCHILDREN', 2); +define('POLL_INTERVAL', 60); // in seconds + +// Uncomment this to get useful logging +define('SCRIPT_DEBUG', true); + +require_once(INSTALLDIR . '/lib/common.php'); +require_once(INSTALLDIR . '/lib/daemon.php'); + +class TwitterStatusFetcher extends Daemon +{ + + private $children = array(); + + function name() + { + return ('twitterstatusfetcher.generic'); + } + + function run() + { + do { + + $flinks = $this->refreshFlinks(); + + foreach ($flinks as $f){ + + // We have to disconnect from the DB before forking so + // each sub-process will open its own connection and + // avoid stomping on the others + + $conn = &$f->getDatabaseConnection(); + $conn->disconnect(); + + $pid = pcntl_fork(); + + if ($pid == -1) { + die ("Couldn't fork!"); + } + + if ($pid) { + + // Parent + if (defined('SCRIPT_DEBUG')) { + common_debug("Parent: forked new status fetcher process " . $pid); + } + + $this->children[] = $pid; + + } else { + + // Child + $this->getTimeline($f); + exit(); + } + + // Remove child from ps list as it finishes + while(($c = pcntl_wait($status, WNOHANG OR WUNTRACED)) > 0) { + + if (defined('SCRIPT_DEBUG')) { + common_debug("Child $c finished."); + } + + $this->remove_ps($this->children, $c); + } + + // Wait! We have too many damn kids. + if (sizeof($this->children) > MAXCHILDREN) { + + if (defined('SCRIPT_DEBUG')) { + common_debug('Too many children. Waiting...'); + } + + if (($c = pcntl_wait($status, WUNTRACED)) > 0){ + + if (defined('SCRIPT_DEBUG')) { + common_debug("Finished waiting for $c"); + } + + $this->remove_ps($this->children, $c); + } + } + } + + // Remove all children from the process list before restarting + while(($c = pcntl_wait($status, WUNTRACED)) > 0) { + + if (defined('SCRIPT_DEBUG')) { + common_debug("Child $c finished."); + } + + $this->remove_ps($this->children, $c); + } + + // Rest for a bit before we fetch more statuses + + if (defined('SCRIPT_DEBUG')) { + common_debug('Waiting ' . POLL_INTERVAL . + ' secs before hitting Twitter again.'); + } + + if (POLL_INTERVAL > 0) { + sleep(POLL_INTERVAL); + } + + } while (true); + } + + function refreshFlinks() { + + $flink = new Foreign_link(); + $flink->service = 1; // Twitter + $flink->orderBy('last_noticesync'); + + $cnt = $flink->find(); + + if (defined('SCRIPT_DEBUG')) { + common_debug('Updating Twitter friends subscriptions' . + " for $cnt users."); + } + + $flinks = array(); + + while ($flink->fetch()) { + + if (($flink->noticesync & FOREIGN_NOTICE_RECV) == + FOREIGN_NOTICE_RECV) { + $flinks[] = clone($flink); + } + } + + $flink->free(); + unset($flink); + + return $flinks; + } + + function remove_ps(&$plist, $ps){ + for ($i = 0; $i < sizeof($plist); $i++) { + if ($plist[$i] == $ps) { + unset($plist[$i]); + $plist = array_values($plist); + break; + } + } + } + + function getTimeline($flink) + { + + if (empty($flink)) { + common_log(LOG_WARNING, + "Can't retrieve Foreign_link for foreign ID $fid"); + return; + } + + $fuser = $flink->getForeignUser(); + + if (empty($fuser)) { + common_log(LOG_WARNING, "Unmatched user for ID " . + $flink->user_id); + return; + } + + if (defined('SCRIPT_DEBUG')) { + common_debug('Trying to get timeline for Twitter user ' . + "$fuser->nickname ($flink->foreign_id)."); + } + + // XXX: Biggest remaining issue - How do we know at which status + // to start importing? How many statuses? Right now I'm going + // with the default last 20. + + $url = 'http://twitter.com/statuses/friends_timeline.json'; + + $timeline_json = get_twitter_data($url, $fuser->nickname, + $flink->credentials); + + $timeline = json_decode($timeline_json); + + if (empty($timeline)) { + common_log(LOG_WARNING, "Empty timeline."); + return; + } + + foreach ($timeline as $status) { + + // Hacktastic: filter out stuff coming from this Laconica + $source = mb_strtolower(common_config('integration', 'source')); + + if (preg_match("/$source/", mb_strtolower($status->source))) { + if (defined('SCRIPT_DEBUG')) { + common_debug('Skipping import of status ' . $status->id . + ' with source ' . $source); + } + continue; + } + + $this->saveStatus($status, $flink); + } + + // Okay, record the time we synced with Twitter for posterity + $flink->last_noticesync = common_sql_now(); + $flink->update(); + } + + function saveStatus($status, $flink) + { + $id = $this->ensureProfile($status->user); + $profile = Profile::staticGet($id); + + if (!$profile) { + common_log(LOG_ERR, + 'Problem saving notice. No associated Profile.'); + return null; + } + + $uri = 'http://twitter.com/' . $status->user->screen_name . + '/status/' . $status->id; + + $notice = Notice::staticGet('uri', $uri); + + // check to see if we've already imported the status + if (!$notice) { + + $notice = new Notice(); + $notice->profile_id = $id; + + $notice->query('BEGIN'); + + // XXX: figure out reply_to + $notice->reply_to = null; + + // XXX: Should this be common_sql_now() instead of status create date? + + $notice->created = strftime('%Y-%m-%d %H:%M:%S', + strtotime($status->created_at)); + $notice->content = $status->text; + $notice->rendered = common_render_content($status->text, $notice); + $notice->source = 'twitter'; + $notice->is_local = 0; + $notice->uri = $uri; + + $notice_id = $notice->insert(); + + if (!$notice_id) { + common_log_db_error($notice, 'INSERT', __FILE__); + if (defined('SCRIPT_DEBUG')) { + common_debug('Could not save notice!'); + } + } + + // XXX: Figure out a better way to link Twitter replies? + $notice->saveReplies(); + + // XXX: Do we want to pollute our tag cloud with + // hashtags from Twitter? + $notice->saveTags(); + $notice->saveGroups(); + + $notice->query('COMMIT'); + + if (defined('SCRIPT_DEBUG')) { + common_debug("Saved status $status->id" . + " as notice $notice->id."); + } + } + + if (!Notice_inbox::staticGet('notice_id', $notice->id)) { + + // Add to inbox + $inbox = new Notice_inbox(); + $inbox->user_id = $flink->user_id; + $inbox->notice_id = $notice->id; + $inbox->created = common_sql_now(); + + $inbox->insert(); + } + } + + function ensureProfile($user) + { + // check to see if there's already a profile for this user + $profileurl = 'http://twitter.com/' . $user->screen_name; + $profile = Profile::staticGet('profileurl', $profileurl); + + if ($profile) { + if (defined('SCRIPT_DEBUG')) { + common_debug("Profile for $profile->nickname found."); + } + + // Check to see if the user's Avatar has changed + $this->checkAvatar($user, $profile); + + return $profile->id; + + } else { + if (defined('SCRIPT_DEBUG')) { + common_debug('Adding profile and remote profile ' . + "for Twitter user: $profileurl"); + } + + $profile = new Profile(); + $profile->query("BEGIN"); + + $profile->nickname = $user->screen_name; + $profile->fullname = $user->name; + $profile->homepage = $user->url; + $profile->bio = $user->description; + $profile->location = $user->location; + $profile->profileurl = $profileurl; + $profile->created = common_sql_now(); + + $id = $profile->insert(); + + if (empty($id)) { + common_log_db_error($profile, 'INSERT', __FILE__); + $profile->query("ROLLBACK"); + return false; + } + + // check for remote profile + $remote_pro = Remote_profile::staticGet('uri', $profileurl); + + if (!$remote_pro) { + + $remote_pro = new Remote_profile(); + + $remote_pro->id = $id; + $remote_pro->uri = $profileurl; + $remote_pro->created = common_sql_now(); + + $rid = $remote_pro->insert(); + + if (empty($rid)) { + common_log_db_error($profile, 'INSERT', __FILE__); + $profile->query("ROLLBACK"); + return false; + } + } + + $profile->query("COMMIT"); + + $this->saveAvatars($user, $id); + + return $id; + } + } + + function checkAvatar($user, $profile) + { + global $config; + + $path_parts = pathinfo($user->profile_image_url); + $newname = 'Twitter_' . $user->id . '_' . + $path_parts['basename']; + + $oldname = $profile->getAvatar(48)->filename; + + if ($newname != $oldname) { + + if (defined('SCRIPT_DEBUG')) { + common_debug('Avatar for Twitter user ' . + "$profile->nickname has changed."); + common_debug("old: $oldname new: $newname"); + } + + $img_root = substr($path_parts['basename'], 0, -11); + $ext = $path_parts['extension']; + $mediatype = $this->getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $user->id . '_' . + $img_root . "_$size.$ext"; + + if ($this->fetchAvatar($url, $filename)) { + $this->updateAvatar($profile->id, $size, $mediatype, $filename); + } + } + } + } + + function getMediatype($ext) + { + $mediatype = null; + + switch (strtolower($ext)) { + case 'jpg': + $mediatype = 'image/jpg'; + break; + case 'gif': + $mediatype = 'image/gif'; + break; + default: + $mediatype = 'image/png'; + } + + return $mediatype; + } + + function saveAvatars($user, $id) + { + global $config; + + $path_parts = pathinfo($user->profile_image_url); + $ext = $path_parts['extension']; + $end = strlen('_normal' . $ext); + $img_root = substr($path_parts['basename'], 0, -($end+1)); + $mediatype = $this->getMediatype($ext); + + foreach (array('mini', 'normal', 'bigger') as $size) { + $url = $path_parts['dirname'] . '/' . + $img_root . '_' . $size . ".$ext"; + $filename = 'Twitter_' . $user->id . '_' . + $img_root . "_$size.$ext"; + + if ($this->fetchAvatar($url, $filename)) { + $this->newAvatar($id, $size, $mediatype, $filename); + } else { + common_log(LOG_WARNING, "Problem fetching Avatar: $url", __FILE__); + } + } + } + + function updateAvatar($profile_id, $size, $mediatype, $filename) { + + if (defined('SCRIPT_DEBUG')) { + common_debug("Updating avatar: $size"); + } + + $profile = Profile::staticGet($profile_id); + + if (!$profile) { + if (defined('SCRIPT_DEBUG')) { + common_debug("Couldn't get profile: $profile_id!"); + } + return; + } + + $sizes = array('mini' => 24, 'normal' => 48, 'bigger' => 73); + $avatar = $profile->getAvatar($sizes[$size]); + + if ($avatar) { + if (defined('SCRIPT_DEBUG')) { + common_debug("Deleting $size avatar for $profile->nickname."); + } + @unlink(INSTALLDIR . '/avatar/' . $avatar->filename); + $avatar->delete(); + } + + $this->newAvatar($profile->id, $size, $mediatype, $filename); + } + + function newAvatar($profile_id, $size, $mediatype, $filename) + { + global $config; + + $avatar = new Avatar(); + $avatar->profile_id = $profile_id; + + switch($size) { + case 'mini': + $avatar->width = 24; + $avatar->height = 24; + break; + case 'normal': + $avatar->width = 48; + $avatar->height = 48; + break; + default: + + // Note: Twitter's big avatars are a different size than + // Laconica's (Laconica's = 96) + + $avatar->width = 73; + $avatar->height = 73; + } + + $avatar->original = 0; // we don't have the original + $avatar->mediatype = $mediatype; + $avatar->filename = $filename; + $avatar->url = Avatar::url($filename); + + if (defined('SCRIPT_DEBUG')) { + common_debug("new filename: $avatar->url"); + } + + $avatar->created = common_sql_now(); + + $id = $avatar->insert(); + + if (!$id) { + common_log_db_error($avatar, 'INSERT', __FILE__); + return null; + } + + if (defined('SCRIPT_DEBUG')) { + common_debug("Saved new $size avatar for $profile_id."); + } + + return $id; + } + + function fetchAvatar($url, $filename) + { + $avatar_dir = INSTALLDIR . '/avatar/'; + + $avatarfile = $avatar_dir . $filename; + + $out = fopen($avatarfile, 'wb'); + if (!$out) { + common_log(LOG_WARNING, "Couldn't open file $filename", __FILE__); + return false; + } + + if (defined('SCRIPT_DEBUG')) { + common_debug("Fetching avatar: $url"); + } + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_FILE, $out); + curl_setopt($ch, CURLOPT_BINARYTRANSFER, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0); + $result = curl_exec($ch); + curl_close($ch); + + fclose($out); + + return $result; + } +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); +declare(ticks = 1); + +$fetcher = new TwitterStatusFetcher(); +$fetcher->runOnce(); + diff --git a/scripts/xmppdaemon.php b/scripts/xmppdaemon.php index 0ce2f2a28..b79fa1b3b 100755 --- a/scripts/xmppdaemon.php +++ b/scripts/xmppdaemon.php @@ -152,11 +152,6 @@ class XMPPDaemon extends Daemon $body = preg_replace('/d[\ ]*('. $to .')[\ ]*/', '', $pl['body']); $this->add_direct($user, $body, $to, $from); } else { - $len = mb_strlen($pl['body']); - if($len > 140) { - $this->from_site($from, 'Message too long - maximum is 140 characters, you sent ' . $len); - return; - } $this->add_notice($user, $pl); } @@ -255,15 +250,13 @@ class XMPPDaemon extends Daemon function add_notice(&$user, &$pl) { $body = trim($pl['body']); - $content_shortened = common_shorten_link($body); + $content_shortened = common_shorten_links($body); if (mb_strlen($content_shortened) > 140) { - $content = trim(mb_substr($body, 0, 140)); - $content_shortened = common_shorten_link($content); - } - else { - $content = $body; + $from = jabber_normalize_jid($pl['from']); + $this->from_site($from, "Message too long - maximum is 140 characters, you sent ".mb_strlen($content_shortened)); + return; } - $notice = Notice::saveNew($user->id, $content, 'xmpp'); + $notice = Notice::saveNew($user->id, $content_shortened, 'xmpp'); if (is_string($notice)) { $this->log(LOG_ERR, $notice); return; |