Some fixes to make the twitterstatusfetcher behave better in a multi-site configuration

1 files changed, 94 insertions, 47 deletions

diff --git a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php
index bff657eb6..7c624fdb3 100755
--- a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php
+++ b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php
@@ -44,10 +44,17 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitterbasicauthclient.php';
 require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php';
 
 /**
- * Fetcher for statuses from Twitter
+ * Fetch statuses from Twitter
  *
- * Fetches statuses from Twitter and inserts them as notices in local
- * system.
+ * Fetches statuses from Twitter and inserts them as notices
+ *
+ * NOTE: an Avatar path MUST be set in config.php for this
+ * script to work, e.g.:
+ *     $config['avatar']['path'] = $config['site']['path'] . '/avatar/';
+ *
+ * @todo @fixme @gar Fix the above. For some reason $_path is always empty when
+ * this script is run, so the default avatar path is always set wrong in
+ * default.php. Therefore it must be set explicitly in config.php. --Z
  *
  * @category Twitter
  * @package  StatusNet
@@ -57,9 +64,6 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php';
  * @link     http://status.net/
  */
 
-// NOTE: an Avatar path MUST be set in config.php for this
-// script to work: e.g.: $config['avatar']['path'] = '/statusnet/avatar';
-
 class TwitterStatusFetcher extends ParallelizingDaemon
 {
     /**
@@ -195,6 +199,8 @@ class TwitterStatusFetcher extends ParallelizingDaemon
             return;
         }
 
+        common_debug(LOG_INFO, $this->name() . ' - Retrieved ' . sizeof($timeline) . ' statuses from Twitter.');
+
         // Reverse to preserve order
 
         foreach (array_reverse($timeline) as $status) {
@@ -209,13 +215,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon
                 continue;
             }
 
-            $notice = null;
-
-            $notice = $this->saveStatus($status, $flink);
-
-            if (!empty($notice)) {
-                common_broadcast_notice($notice);
-            }
+            $this->saveStatus($status, $flink);
         }
 
         // Okay, record the time we synced with Twitter for posterity
@@ -226,50 +226,77 @@ class TwitterStatusFetcher extends ParallelizingDaemon
 
     function saveStatus($status, $flink)
     {
-        $id = $this->ensureProfile($status->user);
-
-        $profile = Profile::staticGet($id);
+        $profile = $this->ensureProfile($status->user);
 
         if (empty($profile)) {
             common_log(LOG_ERR, $this->name() .
                 ' - Problem saving notice. No associated Profile.');
-            return null;
+            return;
         }
 
-        // XXX: change of screen name?
-
-        $uri = 'http://twitter.com/' . $status->user->screen_name .
-            '/status/' . $status->id;
+        $statusUri = 'http://twitter.com/'
+            . $status->user->screen_name
+            . '/status/'
+            . $status->id;
 
         // check to see if we've already imported the status
 
-        $notice = Notice::staticGet('uri', $uri);
+        $dupe = $this->checkDupe($profile, $statusUri);
+
+        if (!empty($dupe)) {
+            common_log(
+                LOG_INFO,
+                $this->name() .
+                " - Ignoring duplicate import: $statusUri"
+            );
+            return;
+        }
+
+        $notice = new Notice();
 
-        if (empty($notice)) {
+        $notice->profile_id = $profile->id;
+        $notice->uri        = $statusUri;
+        $notice->url        = $statusUri;
+        $notice->created    = strftime(
+            '%Y-%m-%d %H:%M:%S',
+            strtotime($status->created_at)
+        );
 
-            // XXX: transaction here?
+        $notice->source     = 'twitter';
+        $notice->reply_to   = null;
+        $notice->is_local   = Notice::GATEWAY;
 
-            $notice = new Notice();
+        $notice->content    = common_shorten_links($status->text);
+        $notice->rendered   = common_render_content(
+            $notice->content,
+            $notice
+        );
 
-            $notice->profile_id = $id;
-            $notice->uri        = $uri;
-            $notice->created    = strftime('%Y-%m-%d %H:%M:%S',
-                                           strtotime($status->created_at));
-            $notice->content    = common_shorten_links($status->text); // XXX
-            $notice->rendered   = common_render_content($notice->content, $notice);
-            $notice->source     = 'twitter';
-            $notice->reply_to   = null; // XXX: lookup reply
-            $notice->is_local   = Notice::GATEWAY;
+        if (Event::handle('StartNoticeSave', array(&$notice))) {
 
-            if (Event::handle('StartNoticeSave', array(&$notice))) {
-                $notice->insert();
-                Event::handle('EndNoticeSave', array($notice));
+            $id = $notice->insert();
+
+            if (!$id) {
+                common_log_db_error($notice, 'INSERT', __FILE__);
+                common_log(LOG_ERR, $this->name() .
+                    ' - Problem saving notice.');
             }
 
+            Event::handle('EndNoticeSave', array($notice));
         }
 
-        Inbox::insertNotice($flink->user_id, $notice->id);
+        $orig = clone($notice);
+        $conv = Conversation::create();
+
+        $notice->conversation = $conv->id;
+
+        if (!$notice->update($orig)) {
+            common_log_db_error($notice, 'UPDATE', __FILE__);
+            common_log(LOG_ERR, $this->name() .
+                ' - Problem saving notice.');
+        }
 
+        Inbox::insertNotice($flink->user_id, $notice->id);
         $notice->blowOnInsert();
 
         return $notice;
@@ -279,9 +306,10 @@ class TwitterStatusFetcher extends ParallelizingDaemon
      * Look up a Profile by profileurl field.  Profile::staticGet() was
      * not working consistently.
      *
-     * @param string $url the profile url
+     * @param string $nickname   local nickname of the Twitter user
+     * @param string $profileurl the profile url
      *
-     * @return mixed the first profile with that url, or null
+     * @return mixed value the first Profile with that url, or null
      */
 
     function getProfileByUrl($nickname, $profileurl)
@@ -299,6 +327,30 @@ class TwitterStatusFetcher extends ParallelizingDaemon
         return null;
     }
 
+    /**
+     * Check to see if this Twitter status has already been imported
+     *
+     * @param Profile $profile   Twitter user's local profile
+     * @param string  $statusUri URI of the status on Twitter
+     *
+     * @return mixed value a matching Notice or null
+     */
+
+    function checkDupe($profile, $statusUri)
+    {
+        $notice = new Notice();
+        $notice->uri = $statusUri;
+        $notice->profile_id = $profile->id;
+        $notice->limit(1);
+
+        if ($notice->find()) {
+            $notice->fetch();
+            return $notice;
+        }
+
+        return null;
+    }
+
     function ensureProfile($user)
     {
         // check to see if there's already a profile for this user
@@ -313,7 +365,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon
             // Check to see if the user's Avatar has changed
 
             $this->checkAvatar($user, $profile);
-            return $profile->id;
+            return $profile;
 
         } else {
 
@@ -372,7 +424,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon
 
             $this->saveAvatars($user, $id);
 
-            return $id;
+            return $profile;
         }
     }
 
@@ -403,7 +455,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon
 
             $this->updateAvatars($twitter_user, $profile);
         }
-
     }
 
     function updateAvatars($twitter_user, $profile) {
@@ -428,17 +479,13 @@ class TwitterStatusFetcher extends ParallelizingDaemon
     }
 
     function missingAvatarFile($profile) {
-
         foreach (array(24, 48, 73) as $size) {
-
             $filename = $profile->getAvatar($size)->filename;
             $avatarpath = Avatar::path($filename);
-
             if (file_exists($avatarpath) == FALSE) {
                 return true;
             }
         }
-
         return false;
     }