diff options
| -rw-r--r-- | classes/Memcached_DataObject.php | 68 | ||||
| -rw-r--r-- | lib/default.php | 1 | ||||
| -rw-r--r-- | lib/httpclient.php | 21 | ||||
| -rw-r--r-- | plugins/RSSCloud/RSSCloudPlugin.php | 2 | ||||
| -rwxr-xr-x | plugins/TwitterBridge/daemons/twitterstatusfetcher.php | 141 |
5 files changed, 176 insertions, 57 deletions
diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index bc4c3a000..8d54e1f0f 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -330,6 +330,10 @@ class Memcached_DataObject extends Safe_DataObject */ function _query($string) { + if (common_config('db', 'annotate_queries')) { + $string = $this->annotateQuery($string); + } + $start = microtime(true); $result = parent::_query($string); $delta = microtime(true) - $start; @@ -342,6 +346,70 @@ class Memcached_DataObject extends Safe_DataObject return $result; } + /** + * Find the first caller in the stack trace that's not a + * low-level database function and add a comment to the + * query string. This should then be visible in process lists + * and slow query logs, to help identify problem areas. + * + * Also marks whether this was a web GET/POST or which daemon + * was running it. + * + * @param string $string SQL query string + * @return string SQL query string, with a comment in it + */ + function annotateQuery($string) + { + $ignore = array('annotateQuery', + '_query', + 'query', + 'get', + 'insert', + 'delete', + 'update', + 'find'); + $ignoreStatic = array('staticGet', + 'pkeyGet', + 'cachedQuery'); + $here = get_class($this); // if we get confused + $bt = debug_backtrace(); + + // Find the first caller that's not us? + foreach ($bt as $frame) { + $func = $frame['function']; + if (isset($frame['type']) && $frame['type'] == '::') { + if (in_array($func, $ignoreStatic)) { + continue; + } + $here = $frame['class'] . '::' . $func; + break; + } else if (isset($frame['type']) && $frame['type'] == '->') { + if ($frame['object'] === $this && in_array($func, $ignore)) { + continue; + } + if (in_array($func, $ignoreStatic)) { + continue; // @fixme this shouldn't be needed? + } + $here = get_class($frame['object']) . '->' . $func; + break; + } + $here = $func; + break; + } + + if (php_sapi_name() == 'cli') { + $context = basename($_SERVER['PHP_SELF']); + } else { + $context = $_SERVER['REQUEST_METHOD']; + } + + // Slip the comment in after the first command, + // or DB_DataObject gets confused about handling inserts and such. + $parts = explode(' ', $string, 2); + $parts[0] .= " /* $context $here */"; + return implode(' ', $parts); + } + // Sanitize a query for logging // @fixme don't trim spaces in string literals function sanitizeQuery($string) diff --git a/lib/default.php b/lib/default.php index 10f3f1a97..7b0d08e4c 100644 --- a/lib/default.php +++ b/lib/default.php @@ -72,6 +72,7 @@ $default = 'quote_identifiers' => false, 'type' => 'mysql', 'schemacheck' => 'runtime', // 'runtime' or 'script' + 'annotate_queries' => false, // true to add caller comments to queries, eg /* POST Notice::saveNew */ 'log_queries' => false, // true to log all DB queries 'log_slow_queries' => 0), // if set, log queries taking over N seconds 'syslog' => diff --git a/lib/httpclient.php b/lib/httpclient.php index 64a51353c..384626ae0 100644 --- a/lib/httpclient.php +++ b/lib/httpclient.php @@ -43,6 +43,9 @@ require_once 'HTTP/Request2/Response.php'; * * This extends the HTTP_Request2_Response class with methods to get info * about any followed redirects. + * + * Originally used the name 'HTTPResponse' to match earlier code, but + * this conflicts with a class in in the PECL HTTP extension. * * @category HTTP * @package StatusNet @@ -51,7 +54,7 @@ require_once 'HTTP/Request2/Response.php'; * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @link http://status.net/ */ -class HTTPResponse extends HTTP_Request2_Response +class StatusNet_HTTPResponse extends HTTP_Request2_Response { function __construct(HTTP_Request2_Response $response, $url, $redirects=0) { @@ -146,7 +149,7 @@ class HTTPClient extends HTTP_Request2 /** * Convenience function to run a GET request. * - * @return HTTPResponse + * @return StatusNet_HTTPResponse * @throws HTTP_Request2_Exception */ public function get($url, $headers=array()) @@ -157,7 +160,7 @@ class HTTPClient extends HTTP_Request2 /** * Convenience function to run a HEAD request. * - * @return HTTPResponse + * @return StatusNet_HTTPResponse * @throws HTTP_Request2_Exception */ public function head($url, $headers=array()) @@ -171,7 +174,7 @@ class HTTPClient extends HTTP_Request2 * @param string $url * @param array $headers optional associative array of HTTP headers * @param array $data optional associative array or blob of form data to submit - * @return HTTPResponse + * @return StatusNet_HTTPResponse * @throws HTTP_Request2_Exception */ public function post($url, $headers=array(), $data=array()) @@ -183,7 +186,7 @@ class HTTPClient extends HTTP_Request2 } /** - * @return HTTPResponse + * @return StatusNet_HTTPResponse * @throws HTTP_Request2_Exception */ protected function doRequest($url, $method, $headers) @@ -217,12 +220,12 @@ class HTTPClient extends HTTP_Request2 } /** - * Actually performs the HTTP request and returns an HTTPResponse object - * with response body and header info. + * Actually performs the HTTP request and returns a + * StatusNet_HTTPResponse object with response body and header info. * * Wraps around parent send() to add logging and redirection processing. * - * @return HTTPResponse + * @return StatusNet_HTTPResponse * @throw HTTP_Request2_Exception */ public function send() @@ -265,6 +268,6 @@ class HTTPClient extends HTTP_Request2 } break; } while ($maxRedirs); - return new HTTPResponse($response, $this->getUrl(), $redirs); + return new StatusNet_HTTPResponse($response, $this->getUrl(), $redirs); } } diff --git a/plugins/RSSCloud/RSSCloudPlugin.php b/plugins/RSSCloud/RSSCloudPlugin.php index 9f444c8bb..001106ace 100644 --- a/plugins/RSSCloud/RSSCloudPlugin.php +++ b/plugins/RSSCloud/RSSCloudPlugin.php @@ -105,7 +105,7 @@ class RSSCloudPlugin extends Plugin * @return boolean hook return */ - function onRouterInitialized(&$m) + function onRouterInitialized($m) { $m->connect('/main/rsscloud/request_notify', array('action' => 'RSSCloudRequestNotify')); diff --git a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php index bff657eb6..7c624fdb3 100755 --- a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php +++ b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php @@ -44,10 +44,17 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitterbasicauthclient.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; /** - * Fetcher for statuses from Twitter + * Fetch statuses from Twitter * - * Fetches statuses from Twitter and inserts them as notices in local - * system. + * Fetches statuses from Twitter and inserts them as notices + * + * NOTE: an Avatar path MUST be set in config.php for this + * script to work, e.g.: + * $config['avatar']['path'] = $config['site']['path'] . '/avatar/'; + * + * @todo @fixme @gar Fix the above. For some reason $_path is always empty when + * this script is run, so the default avatar path is always set wrong in + * default.php. Therefore it must be set explicitly in config.php. --Z * * @category Twitter * @package StatusNet @@ -57,9 +64,6 @@ require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; * @link http://status.net/ */ -// NOTE: an Avatar path MUST be set in config.php for this -// script to work: e.g.: $config['avatar']['path'] = '/statusnet/avatar'; - class TwitterStatusFetcher extends ParallelizingDaemon { /** @@ -195,6 +199,8 @@ class TwitterStatusFetcher extends ParallelizingDaemon return; } + common_debug(LOG_INFO, $this->name() . ' - Retrieved ' . sizeof($timeline) . ' statuses from Twitter.'); + // Reverse to preserve order foreach (array_reverse($timeline) as $status) { @@ -209,13 +215,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon continue; } - $notice = null; - - $notice = $this->saveStatus($status, $flink); - - if (!empty($notice)) { - common_broadcast_notice($notice); - } + $this->saveStatus($status, $flink); } // Okay, record the time we synced with Twitter for posterity @@ -226,50 +226,77 @@ class TwitterStatusFetcher extends ParallelizingDaemon function saveStatus($status, $flink) { - $id = $this->ensureProfile($status->user); - - $profile = Profile::staticGet($id); + $profile = $this->ensureProfile($status->user); if (empty($profile)) { common_log(LOG_ERR, $this->name() . ' - Problem saving notice. No associated Profile.'); - return null; + return; } - // XXX: change of screen name? - - $uri = 'http://twitter.com/' . $status->user->screen_name . - '/status/' . $status->id; + $statusUri = 'http://twitter.com/' + . $status->user->screen_name + . '/status/' + . $status->id; // check to see if we've already imported the status - $notice = Notice::staticGet('uri', $uri); + $dupe = $this->checkDupe($profile, $statusUri); + + if (!empty($dupe)) { + common_log( + LOG_INFO, + $this->name() . + " - Ignoring duplicate import: $statusUri" + ); + return; + } + + $notice = new Notice(); - if (empty($notice)) { + $notice->profile_id = $profile->id; + $notice->uri = $statusUri; + $notice->url = $statusUri; + $notice->created = strftime( + '%Y-%m-%d %H:%M:%S', + strtotime($status->created_at) + ); - // XXX: transaction here? + $notice->source = 'twitter'; + $notice->reply_to = null; + $notice->is_local = Notice::GATEWAY; - $notice = new Notice(); + $notice->content = common_shorten_links($status->text); + $notice->rendered = common_render_content( + $notice->content, + $notice + ); - $notice->profile_id = $id; - $notice->uri = $uri; - $notice->created = strftime('%Y-%m-%d %H:%M:%S', - strtotime($status->created_at)); - $notice->content = common_shorten_links($status->text); // XXX - $notice->rendered = common_render_content($notice->content, $notice); - $notice->source = 'twitter'; - $notice->reply_to = null; // XXX: lookup reply - $notice->is_local = Notice::GATEWAY; + if (Event::handle('StartNoticeSave', array(&$notice))) { - if (Event::handle('StartNoticeSave', array(&$notice))) { - $notice->insert(); - Event::handle('EndNoticeSave', array($notice)); + $id = $notice->insert(); + + if (!$id) { + common_log_db_error($notice, 'INSERT', __FILE__); + common_log(LOG_ERR, $this->name() . + ' - Problem saving notice.'); } + Event::handle('EndNoticeSave', array($notice)); } - Inbox::insertNotice($flink->user_id, $notice->id); + $orig = clone($notice); + $conv = Conversation::create(); + + $notice->conversation = $conv->id; + + if (!$notice->update($orig)) { + common_log_db_error($notice, 'UPDATE', __FILE__); + common_log(LOG_ERR, $this->name() . + ' - Problem saving notice.'); + } + Inbox::insertNotice($flink->user_id, $notice->id); $notice->blowOnInsert(); return $notice; @@ -279,9 +306,10 @@ class TwitterStatusFetcher extends ParallelizingDaemon * Look up a Profile by profileurl field. Profile::staticGet() was * not working consistently. * - * @param string $url the profile url + * @param string $nickname local nickname of the Twitter user + * @param string $profileurl the profile url * - * @return mixed the first profile with that url, or null + * @return mixed value the first Profile with that url, or null */ function getProfileByUrl($nickname, $profileurl) @@ -299,6 +327,30 @@ class TwitterStatusFetcher extends ParallelizingDaemon return null; } + /** + * Check to see if this Twitter status has already been imported + * + * @param Profile $profile Twitter user's local profile + * @param string $statusUri URI of the status on Twitter + * + * @return mixed value a matching Notice or null + */ + + function checkDupe($profile, $statusUri) + { + $notice = new Notice(); + $notice->uri = $statusUri; + $notice->profile_id = $profile->id; + $notice->limit(1); + + if ($notice->find()) { + $notice->fetch(); + return $notice; + } + + return null; + } + function ensureProfile($user) { // check to see if there's already a profile for this user @@ -313,7 +365,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon // Check to see if the user's Avatar has changed $this->checkAvatar($user, $profile); - return $profile->id; + return $profile; } else { @@ -372,7 +424,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon $this->saveAvatars($user, $id); - return $id; + return $profile; } } @@ -403,7 +455,6 @@ class TwitterStatusFetcher extends ParallelizingDaemon $this->updateAvatars($twitter_user, $profile); } - } function updateAvatars($twitter_user, $profile) { @@ -428,17 +479,13 @@ class TwitterStatusFetcher extends ParallelizingDaemon } function missingAvatarFile($profile) { - foreach (array(24, 48, 73) as $size) { - $filename = $profile->getAvatar($size)->filename; $avatarpath = Avatar::path($filename); - if (file_exists($avatarpath) == FALSE) { return true; } } - return false; } |