From 4741683298ac02e14d7380ab20f20fd8a73775e2 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Wed, 10 Mar 2010 22:05:28 +0000 Subject: Allow site-specific doc files --- actions/doc.php | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/actions/doc.php b/actions/doc.php index 459f5f096..f876fb8be 100644 --- a/actions/doc.php +++ b/actions/doc.php @@ -13,7 +13,7 @@ * @link http://status.net/ * * StatusNet - the distributed open-source microblogging tool - * Copyright (C) 2008, 2009, StatusNet, Inc. + * Copyright (C) 2008-2010, StatusNet, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -168,14 +168,28 @@ class DocAction extends Action function getFilename() { - if (file_exists(INSTALLDIR.'/local/doc-src/'.$this->title)) { - $localDef = INSTALLDIR.'/local/doc-src/'.$this->title; - } + $localDef = null; + $local = null; + + $site = StatusNet::currentSite(); - $local = glob(INSTALLDIR.'/local/doc-src/'.$this->title.'.*'); - if ($local === false) { - // Some systems return false, others array(), if dir didn't exist. - $local = array(); + if (!empty($site) && file_exists(INSTALLDIR.'/local/doc-src/'.$site.'/'.$this->title)) { + $localDef = INSTALLDIR.'/local/doc-src/'.$site.'/'.$this->title; + + $local = glob(INSTALLDIR.'/local/doc-src/'.$site.'/'.$this->title.'.*'); + if ($local === false) { + // Some systems return false, others array(), if dir didn't exist. + $local = array(); + } + } else { + if (file_exists(INSTALLDIR.'/local/doc-src/'.$this->title)) { + $localDef = INSTALLDIR.'/local/doc-src/'.$this->title; + } + + $local = glob(INSTALLDIR.'/local/doc-src/'.$this->title.'.*'); + if ($local === false) { + $local = array(); + } } if (count($local) || isset($localDef)) { -- cgit v1.2.3-54-g00ecf From f02cb7c71800e6a4426b92ce04c4b8f89006b10a Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 10 Mar 2010 13:39:42 -0800 Subject: Fix for attachment "h bug": posting a shortened link to an oembed-able resource that has been previously used in the system would incorrectly save "h" as the item's type and title. --- classes/File.php | 9 +++++++- classes/File_oembed.php | 8 ++++++- classes/File_redirection.php | 51 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/classes/File.php b/classes/File.php index 1b8ef1b3e..a83ecac4c 100644 --- a/classes/File.php +++ b/classes/File.php @@ -67,7 +67,14 @@ class File extends Memcached_DataObject return $att; } - function saveNew($redir_data, $given_url) { + /** + * Save a new file record. + * + * @param array $redir_data lookup data eg from File_redirection::where() + * @param string $given_url + * @return File + */ + function saveNew(array $redir_data, $given_url) { $x = new File; $x->url = $given_url; if (!empty($redir_data['protected'])) $x->protected = $redir_data['protected']; diff --git a/classes/File_oembed.php b/classes/File_oembed.php index 11f160718..f59eaf24c 100644 --- a/classes/File_oembed.php +++ b/classes/File_oembed.php @@ -81,7 +81,13 @@ class File_oembed extends Memcached_DataObject } } - function saveNew($data, $file_id) { + /** + * Save embedding info for a new file. + * + * @param array $data lookup data as from File_redirection::where + * @param int $file_id + */ + function saveNew(array $data, $file_id) { $file_oembed = new File_oembed; $file_oembed->file_id = $file_id; $file_oembed->version = $data->version; diff --git a/classes/File_redirection.php b/classes/File_redirection.php index 08a6e8d8b..d96979158 100644 --- a/classes/File_redirection.php +++ b/classes/File_redirection.php @@ -115,11 +115,45 @@ class File_redirection extends Memcached_DataObject return $ret; } + /** + * Check if this URL is a redirect and return redir info. + * If a File record is present for this URL, it is not considered a redirect. + * If a File_redirection record is present for this URL, the recorded target is returned. + * + * If no File or File_redirect record is present, the URL is hit and any + * redirects are followed, up to 10 levels or until a protected URL is + * reached. + * + * @param string $in_url + * @return mixed one of: + * string - target URL, if this is a direct link or a known redirect + * array - redirect info if this is an *unknown* redirect: + * associative array with the following elements: + * code: HTTP status code + * redirects: count of redirects followed + * url: URL string of final target + * type (optional): MIME type from Content-Type header + * size (optional): byte size from Content-Length header + * time (optional): timestamp from Last-Modified header + */ function where($in_url) { $ret = File_redirection::_redirectWhere_imp($in_url); return $ret; } + /** + * Shorten a URL with the current user's configured shortening + * options, if applicable. + * + * If it cannot be shortened or the "short" URL is longer than the + * original, the original is returned. + * + * If the referenced item has not been seen before, embedding data + * may be saved. + * + * @param string $long_url + * @return string + */ function makeShort($long_url) { $canon = File_redirection::_canonUrl($long_url); @@ -141,11 +175,20 @@ class File_redirection extends Memcached_DataObject // store it $file = File::staticGet('url', $long_url); if (empty($file)) { + // Check if the target URL is itself a redirect... $redir_data = File_redirection::where($long_url); - $file = File::saveNew($redir_data, $long_url); - $file_id = $file->id; - if (!empty($redir_data['oembed']['json'])) { - File_oembed::saveNew($redir_data['oembed']['json'], $file_id); + if (is_array($redir_data)) { + // We haven't seen the target URL before. + // Save file and embedding data about it! + $file = File::saveNew($redir_data, $long_url); + $file_id = $file->id; + if (!empty($redir_data['oembed']['json'])) { + File_oembed::saveNew($redir_data['oembed']['json'], $file_id); + } + } else if (is_string($redir_data)) { + // The file is a known redirect target. + $file = File::staticGet('url', $redir_data); + $file_id = $file->id; } } else { $file_id = $file->id; -- cgit v1.2.3-54-g00ecf From 294b290dd95a2e4a09026932a2b066ccee587681 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 10 Mar 2010 14:31:29 -0800 Subject: Fixup script for files w/ bogus data saved into file record ('h bug') --- classes/File.php | 24 +++++++++++++++--- classes/File_oembed.php | 4 +-- classes/File_redirection.php | 60 ++++++++++++++++++++++++++++++-------------- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/classes/File.php b/classes/File.php index a83ecac4c..ba8332841 100644 --- a/classes/File.php +++ b/classes/File.php @@ -84,19 +84,36 @@ class File extends Memcached_DataObject if (isset($redir_data['time']) && $redir_data['time'] > 0) $x->date = intval($redir_data['time']); $file_id = $x->insert(); + $x->saveOembed($redir_data, $given_url); + return $x; + } + + /** + * Save embedding information for this file, if applicable. + * + * Normally this won't need to be called manually, as File::saveNew() + * takes care of it. + * + * @param array $redir_data lookup data eg from File_redirection::where() + * @param string $given_url + * @return boolean success + */ + public function saveOembed($redir_data, $given_url) + { if (isset($redir_data['type']) && (('text/html' === substr($redir_data['type'], 0, 9) || 'application/xhtml+xml' === substr($redir_data['type'], 0, 21))) && ($oembed_data = File_oembed::_getOembed($given_url))) { - $fo = File_oembed::staticGet('file_id', $file_id); + $fo = File_oembed::staticGet('file_id', $this->id); if (empty($fo)) { - File_oembed::saveNew($oembed_data, $file_id); + File_oembed::saveNew($oembed_data, $this->id); + return true; } else { common_log(LOG_WARNING, "Strangely, a File_oembed object exists for new file $file_id", __FILE__); } } - return $x; + return false; } function processNew($given_url, $notice_id=null) { @@ -112,6 +129,7 @@ class File extends Memcached_DataObject $redir_url = $redir_data['url']; } elseif (is_string($redir_data)) { $redir_url = $redir_data; + $redir_data = array(); } else { throw new ServerException("Can't process url '$given_url'"); } diff --git a/classes/File_oembed.php b/classes/File_oembed.php index f59eaf24c..041b44740 100644 --- a/classes/File_oembed.php +++ b/classes/File_oembed.php @@ -84,10 +84,10 @@ class File_oembed extends Memcached_DataObject /** * Save embedding info for a new file. * - * @param array $data lookup data as from File_redirection::where + * @param object $data Services_oEmbed_Object_* * @param int $file_id */ - function saveNew(array $data, $file_id) { + function saveNew($data, $file_id) { $file_oembed = new File_oembed; $file_oembed->file_id = $file_id; $file_oembed->version = $data->version; diff --git a/classes/File_redirection.php b/classes/File_redirection.php index d96979158..f128b3e07 100644 --- a/classes/File_redirection.php +++ b/classes/File_redirection.php @@ -58,24 +58,30 @@ class File_redirection extends Memcached_DataObject return $request; } - function _redirectWhere_imp($short_url, $redirs = 10, $protected = false) { + /** + * Check if this URL is a redirect and return redir info. + * + * Most code should call File_redirection::where instead, to check if we + * already know that redirection and avoid extra hits to the web. + * + * The URL is hit and any redirects are followed, up to 10 levels or until + * a protected URL is reached. + * + * @param string $in_url + * @return mixed one of: + * string - target URL, if this is a direct link or can't be followed + * array - redirect info if this is an *unknown* redirect: + * associative array with the following elements: + * code: HTTP status code + * redirects: count of redirects followed + * url: URL string of final target + * type (optional): MIME type from Content-Type header + * size (optional): byte size from Content-Length header + * time (optional): timestamp from Last-Modified header + */ + public function lookupWhere($short_url, $redirs = 10, $protected = false) { if ($redirs < 0) return false; - // let's see if we know this... - $a = File::staticGet('url', $short_url); - - if (!empty($a)) { - // this is a direct link to $a->url - return $a->url; - } else { - $b = File_redirection::staticGet('url', $short_url); - if (!empty($b)) { - // this is a redirect to $b->file_id - $a = File::staticGet('id', $b->file_id); - return $a->url; - } - } - if(strpos($short_url,'://') === false){ return $short_url; } @@ -93,12 +99,13 @@ class File_redirection extends Memcached_DataObject } } catch (Exception $e) { // Invalid URL or failure to reach server + common_log(LOG_ERR, "Error while following redirects for $short_url: " . $e->getMessage()); return $short_url; } if ($response->getRedirectCount() && File::isProtected($response->getUrl())) { // Bump back up the redirect chain until we find a non-protected URL - return self::_redirectWhere_imp($short_url, $response->getRedirectCount() - 1, true); + return self::lookupWhere($short_url, $response->getRedirectCount() - 1, true); } $ret = array('code' => $response->getStatus() @@ -136,8 +143,23 @@ class File_redirection extends Memcached_DataObject * size (optional): byte size from Content-Length header * time (optional): timestamp from Last-Modified header */ - function where($in_url) { - $ret = File_redirection::_redirectWhere_imp($in_url); + public function where($in_url) { + // let's see if we know this... + $a = File::staticGet('url', $in_url); + + if (!empty($a)) { + // this is a direct link to $a->url + return $a->url; + } else { + $b = File_redirection::staticGet('url', $in_url); + if (!empty($b)) { + // this is a redirect to $b->file_id + $a = File::staticGet('id', $b->file_id); + return $a->url; + } + } + + $ret = File_redirection::lookupWhere($in_url); return $ret; } -- cgit v1.2.3-54-g00ecf From 5cd020bf299619ca2844f4d14418891a59a0dd22 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 10 Mar 2010 15:08:40 -0800 Subject: Workaround intermittent bugs with HEAD requests by disabling keepalive in HTTPClient. I think this is a bug in Youtube's web server (sending chunked encoding of an empty body with a HEAD response, leaving the connection out of sync when it doesn't attempt to read a body) but the HTTP_Request2 library may need to be adjusted to watch out for that. --- lib/httpclient.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/httpclient.php b/lib/httpclient.php index 4c3af8d7d..64a51353c 100644 --- a/lib/httpclient.php +++ b/lib/httpclient.php @@ -120,6 +120,16 @@ class HTTPClient extends HTTP_Request2 { $this->config['max_redirs'] = 10; $this->config['follow_redirects'] = true; + + // We've had some issues with keepalive breaking with + // HEAD requests, such as to youtube which seems to be + // emitting chunked encoding info for an empty body + // instead of not emitting anything. This may be a + // bug on YouTube's end, but the upstream libray + // ought to be investigated to see if we can handle + // it gracefully in that case as well. + $this->config['protocol_version'] = '1.0'; + parent::__construct($url, $method, $config); $this->setHeader('User-Agent', $this->userAgent()); } -- cgit v1.2.3-54-g00ecf From 66518df4356ea878bfd8693191f0354caebfb549 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 10 Mar 2010 17:00:05 -0800 Subject: OStatus: reject attempts to create a remote profile for a local user or group. Some stray shadow entries were ending up getting created, which would steal group posts from remote users. Run plugins/OStatus/scripts/fixup-shadow.php for each site to remove any existing ones. --- plugins/OStatus/OStatusPlugin.php | 37 ++++++++++++++++ plugins/OStatus/classes/Ostatus_profile.php | 19 +++++--- plugins/OStatus/scripts/fixup-shadow.php | 69 +++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 plugins/OStatus/scripts/fixup-shadow.php diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index a97f3475b..ef28ab22e 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -929,4 +929,41 @@ class OStatusPlugin extends Plugin return true; } + + /** + * Utility function to check if the given URL is a canonical group profile + * page, and if so return the ID number. + * + * @param string $url + * @return mixed int or false + */ + public static function localGroupFromUrl($url) + { + $template = common_local_url('groupbyid', array('id' => '31337')); + $template = preg_quote($template, '/'); + $template = str_replace('31337', '(\d+)', $template); + if (preg_match("/$template/", $url, $matches)) { + return intval($matches[1]); + } + return false; + } + + /** + * Utility function to check if the given URL is a canonical user profile + * page, and if so return the ID number. + * + * @param string $url + * @return mixed int or false + */ + public static function localProfileFromUrl($url) + { + $template = common_local_url('userbyid', array('id' => '31337')); + $template = preg_quote($template, '/'); + $template = str_replace('31337', '(\d+)', $template); + if (preg_match("/$template/", $url, $matches)) { + return intval($matches[1]); + } + return false; + } + } diff --git a/plugins/OStatus/classes/Ostatus_profile.php b/plugins/OStatus/classes/Ostatus_profile.php index abc8100ce..6ae8e4fd5 100644 --- a/plugins/OStatus/classes/Ostatus_profile.php +++ b/plugins/OStatus/classes/Ostatus_profile.php @@ -675,13 +675,10 @@ class Ostatus_profile extends Memcached_DataObject } // Is the recipient a local group? - // @fixme we need a uri on user_group + // @fixme uri on user_group isn't reliable yet // $group = User_group::staticGet('uri', $recipient); - $template = common_local_url('groupbyid', array('id' => '31337')); - $template = preg_quote($template, '/'); - $template = str_replace('31337', '(\d+)', $template); - if (preg_match("/$template/", $recipient, $matches)) { - $id = $matches[1]; + $id = OStatusPlugin::localGroupFromUrl($recipient); + if ($id) { $group = User_group::staticGet('id', $id); if ($group) { // Deliver to all members of this local group if allowed. @@ -992,7 +989,15 @@ class Ostatus_profile extends Memcached_DataObject if (!$homeuri) { common_log(LOG_DEBUG, __METHOD__ . " empty actor profile URI: " . var_export($activity, true)); - throw new ServerException("No profile URI"); + throw new Exception("No profile URI"); + } + + if (OStatusPlugin::localProfileFromUrl($homeuri)) { + throw new Exception("Local user can't be referenced as remote."); + } + + if (OStatusPlugin::localGroupFromUrl($homeuri)) { + throw new Exception("Local group can't be referenced as remote."); } if (array_key_exists('feedurl', $hints)) { diff --git a/plugins/OStatus/scripts/fixup-shadow.php b/plugins/OStatus/scripts/fixup-shadow.php new file mode 100644 index 000000000..0171b77bc --- /dev/null +++ b/plugins/OStatus/scripts/fixup-shadow.php @@ -0,0 +1,69 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..')); + +$longoptions = array('dry-run'); + +$helptext = << $marker)); +$encProfile = $oprofile->escape($profileTemplate, true); +$encProfile = str_replace($marker, '%', $encProfile); + +$groupTemplate = common_local_url('groupbyid', array('id' => $marker)); +$encGroup = $oprofile->escape($groupTemplate, true); +$encGroup = str_replace($marker, '%', $encGroup); + +$sql = "SELECT * FROM ostatus_profile WHERE uri LIKE '%s' OR uri LIKE '%s'"; +$oprofile->query(sprintf($sql, $encProfile, $encGroup)); + +echo "Found $oprofile->N bogus ostatus_profile entries:\n"; + +while ($oprofile->fetch()) { + echo "$oprofile->uri"; + + if ($dry) { + echo " (unchanged)\n"; + } else { + echo " deleting..."; + $evil = clone($oprofile); + $evil->delete(); + echo " ok\n"; + } +} + +echo "done.\n"; + -- cgit v1.2.3-54-g00ecf