Rebuilt HTTPClient class as an extension of PEAR HTTP_Request2 package, adding redirect handling and convenience functions.

Caching support will be added in future work after unit tests have been added.

* extlib: add PEAR HTTP_Request2 0.4.1 alpha
* extlib: update PEAR Net_URL2 to 0.3.0 beta for HTTP_Request2 compatibility
* moved direct usage of CURL and file_get_contents to HTTPClient class, excluding external-sourced libraries
* adapted GeonamesPlugin for new HTTPResponse interface

Note some plugins haven't been fully tested yet.

7 files changed, 222 insertions, 294 deletions

diff --git a/lib/Shorturl_api.php b/lib/Shorturl_api.php
index 18ae7719b..de4d55012 100644
--- a/lib/Shorturl_api.php
+++ b/lib/Shorturl_api.php
@@ -41,22 +41,18 @@ abstract class ShortUrlApi
         return strlen($url) >= common_config('site', 'shorturllength');
     }
 
-    protected function http_post($data) {
-        $ch = curl_init();
-        curl_setopt($ch, CURLOPT_URL, $this->service_url);
-        curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
-        curl_setopt($ch, CURLOPT_POST, 1);
-        curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
-        $response = curl_exec($ch);
-        $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-        curl_close($ch);
-        if (($code < 200) || ($code >= 400)) return false;
-        return $response;
+    protected function http_post($data)
+    {
+        $request = HTTPClient::start();
+        $response = $request->post($this->service_url, null, $data);
+        return $response->getBody();
     }
 
-    protected function http_get($url) {
-        $encoded_url = urlencode($url);
-        return file_get_contents("{$this->service_url}$encoded_url");
+    protected function http_get($url)
+    {
+        $request = HTTPClient::start();
+        $response = $request->get($this->service_url . urlencode($url));
+        return $response->getBody();
     }
 
     protected function tidy($response) {
diff --git a/lib/curlclient.php b/lib/curlclient.php
deleted file mode 100644
index c307c2984..000000000
--- a/lib/curlclient.php
+++ /dev/null
@@ -1,179 +0,0 @@
-<?php
-/**
- * StatusNet, the distributed open-source microblogging tool
- *
- * Utility class for wrapping Curl
- *
- * PHP version 5
- *
- * LICENCE: This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * @category  HTTP
- * @package   StatusNet
- * @author    Evan Prodromou <evan@status.net>
- * @copyright 2009 StatusNet, Inc.
- * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
- * @link      http://status.net/
- */
-
-if (!defined('STATUSNET')) {
-    exit(1);
-}
-
-define(CURLCLIENT_VERSION, "0.1");
-
-/**
- * Wrapper for Curl
- *
- * Makes Curl HTTP client calls within our HTTPClient framework
- *
- * @category HTTP
- * @package  StatusNet
- * @author   Evan Prodromou <evan@status.net>
- * @license  http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
- * @link     http://status.net/
- */
-
-class CurlClient extends HTTPClient
-{
-    function __construct()
-    {
-    }
-
-    function head($url, $headers=null)
-    {
-        $ch = curl_init($url);
-
-        $this->setup($ch);
-
-        curl_setopt_array($ch,
-                          array(CURLOPT_NOBODY => true));
-
-        if (!is_null($headers)) {
-            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
-        }
-
-        $result = curl_exec($ch);
-
-        curl_close($ch);
-
-        return $this->parseResults($result);
-    }
-
-    function get($url, $headers=null)
-    {
-        $ch = curl_init($url);
-
-        $this->setup($ch);
-
-        if (!is_null($headers)) {
-            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
-        }
-
-        $result = curl_exec($ch);
-
-        curl_close($ch);
-
-        return $this->parseResults($result);
-    }
-
-    function post($url, $headers=null, $body=null)
-    {
-        $ch = curl_init($url);
-
-        $this->setup($ch);
-
-        curl_setopt($ch, CURLOPT_POST, true);
-
-        if (!is_null($body)) {
-            curl_setopt($ch, CURLOPT_POSTFIELDS, $body);
-        }
-
-        if (!is_null($headers)) {
-            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
-        }
-
-        $result = curl_exec($ch);
-
-        curl_close($ch);
-
-        return $this->parseResults($result);
-    }
-
-    function setup($ch)
-    {
-        curl_setopt_array($ch,
-                          array(CURLOPT_USERAGENT => $this->userAgent(),
-                                CURLOPT_HEADER => true,
-                                CURLOPT_RETURNTRANSFER => true));
-    }
-
-    function userAgent()
-    {
-        $version = curl_version();
-        return parent::userAgent() . " CurlClient/".CURLCLIENT_VERSION . " cURL/" . $version['version'];
-    }
-
-    function parseResults($results)
-    {
-        $resp = new HTTPResponse();
-
-        $lines = explode("\r\n", $results);
-
-        if (preg_match("#^HTTP/1.[01] (\d\d\d) .+$#", $lines[0], $match)) {
-            $resp->code = $match[1];
-        } else {
-            throw Exception("Bad format: initial line is not HTTP status line");
-        }
-
-        $lastk = null;
-
-        for ($i = 1; $i < count($lines); $i++) {
-            $l =& $lines[$i];
-            if (mb_strlen($l) == 0) {
-                $resp->body = implode("\r\n", array_slice($lines, $i + 1));
-                break;
-            }
-            if (preg_match("#^(\S+):\s+(.*)$#", $l, $match)) {
-                $k = $match[1];
-                $v = $match[2];
-
-                if (array_key_exists($k, $resp->headers)) {
-                    if (is_array($resp->headers[$k])) {
-                        $resp->headers[$k][] = $v;
-                    } else {
-                        $resp->headers[$k] = array($resp->headers[$k], $v);
-                    }
-                } else {
-                    $resp->headers[$k] = $v;
-                }
-                $lastk = $k;
-            } else if (preg_match("#^\s+(.*)$#", $l, $match)) {
-                // continuation line
-                if (is_null($lastk)) {
-                    throw Exception("Bad format: initial whitespace in headers");
-                }
-                $h =& $resp->headers[$lastk];
-                if (is_array($h)) {
-                    $n = count($h);
-                    $h[$n-1] .= $match[1];
-                } else {
-                    $h .= $match[1];
-                }
-            }
-        }
-
-        return $resp;
-    }
-}
diff --git a/lib/default.php b/lib/default.php
index 7ec8558b0..f6cc4b725 100644
--- a/lib/default.php
+++ b/lib/default.php
@@ -228,8 +228,6 @@ $default =
         array('contentlimit' => null),
         'message' =>
         array('contentlimit' => null),
-        'http' =>
-        array('client' => 'curl'), // XXX: should this be the default?
         'location' =>
         array('namespace' => 1), // 1 = geonames, 2 = Yahoo Where on Earth
         );
diff --git a/lib/httpclient.php b/lib/httpclient.php
index f16e31e10..3f8262076 100644
--- a/lib/httpclient.php
+++ b/lib/httpclient.php
@@ -31,6 +31,9 @@ if (!defined('STATUSNET')) {
     exit(1);
 }
 
+require_once 'HTTP/Request2.php';
+require_once 'HTTP/Request2/Response.php';
+
 /**
  * Useful structure for HTTP responses
  *
@@ -38,18 +41,53 @@ if (!defined('STATUSNET')) {
  * ways of doing them. This class hides the specifics of what underlying
  * library (curl or PHP-HTTP or whatever) that's used.
  *
+ * This extends the HTTP_Request2_Response class with methods to get info
+ * about any followed redirects.
+ *
  * @category HTTP
- * @package  StatusNet
- * @author   Evan Prodromou <evan@status.net>
- * @license  http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
- * @link     http://status.net/
+ * @package StatusNet
+ * @author Evan Prodromou <evan@status.net>
+ * @author Brion Vibber <brion@status.net>
+ * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
+ * @link http://status.net/
  */
-
-class HTTPResponse
+class HTTPResponse extends HTTP_Request2_Response
 {
-    public $code = null;
-    public $headers = array();
-    public $body = null;
+    function __construct(HTTP_Request2_Response $response, $url, $redirects=0)
+    {
+        foreach (get_object_vars($response) as $key => $val) {
+            $this->$key = $val;
+        }
+        $this->url = strval($url);
+        $this->redirectCount = intval($redirects);
+    }
+
+    /**
+     * Get the count of redirects that have been followed, if any.
+     * @return int
+     */
+    function getRedirectCount()
+    {
+        return $this->redirectCount;
+    }
+
+    /**
+     * Gets the final target URL, after any redirects have been followed.
+     * @return string URL
+     */
+    function getUrl()
+    {
+        return $this->url;
+    }
+
+    /**
+     * Check if the response is OK, generally a 200 status code.
+     * @return bool
+     */
+    function isOk()
+    {
+        return ($this->getStatus() == 200);
+    }
 }
 
 /**
@@ -59,64 +97,163 @@ class HTTPResponse
  * ways of doing them. This class hides the specifics of what underlying
  * library (curl or PHP-HTTP or whatever) that's used.
  *
+ * This extends the PEAR HTTP_Request2 package:
+ * - sends StatusNet-specific User-Agent header
+ * - 'follow_redirects' config option, defaulting off
+ * - 'max_redirs' config option, defaulting to 10
+ * - extended response class adds getRedirectCount() and getUrl() methods
+ * - get() and post() convenience functions return body content directly
+ *
  * @category HTTP
  * @package  StatusNet
  * @author   Evan Prodromou <evan@status.net>
+ * @author   Brion Vibber <brion@status.net>
  * @license  http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
  * @link     http://status.net/
  */
 
-class HTTPClient
+class HTTPClient extends HTTP_Request2
 {
-    static $_client = null;
 
-    static function start()
+    function __construct($url=null, $method=self::METHOD_GET, $config=array())
     {
-        if (!is_null(self::$_client)) {
-            return self::$_client;
-        }
-
-        $type = common_config('http', 'client');
-
-        switch ($type) {
-         case 'curl':
-            self::$_client = new CurlClient();
-            break;
-         default:
-            throw new Exception("Unknown HTTP client type '$type'");
-            break;
-        }
-
-        return self::$_client;
+        $this->config['max_redirs'] = 10;
+        $this->config['follow_redirects'] = true;
+        parent::__construct($url, $method, $config);
+        $this->setHeader('User-Agent', $this->userAgent());
     }
 
-    function head($url, $headers)
+    /**
+     * Convenience/back-compat instantiator
+     * @return HTTPClient
+     */
+    public static function start()
     {
-        throw new Exception("HEAD method unimplemented");
+        return new HTTPClient();
     }
 
-    function get($url, $headers)
+    /**
+     * Convenience function to run a GET request.
+     *
+     * @return HTTPResponse
+     * @throws HTTP_Request2_Exception
+     */
+    public function get($url, $headers=array())
     {
-        throw new Exception("GET method unimplemented");
+        return $this->doRequest($url, self::METHOD_GET, $headers);
     }
 
-    function post($url, $headers, $body)
+    /**
+     * Convenience function to run a HEAD request.
+     *
+     * @return HTTPResponse
+     * @throws HTTP_Request2_Exception
+     */
+    public function head($url, $headers=array())
     {
-        throw new Exception("POST method unimplemented");
+        return $this->doRequest($url, self::METHOD_HEAD, $headers);
     }
 
-    function put($url, $headers, $body)
+    /**
+     * Convenience function to POST form data.
+     *
+     * @param string $url
+     * @param array $headers optional associative array of HTTP headers
+     * @param array $data optional associative array or blob of form data to submit
+     * @return HTTPResponse
+     * @throws HTTP_Request2_Exception
+     */
+    public function post($url, $headers=array(), $data=array())
     {
-        throw new Exception("PUT method unimplemented");
+        if ($data) {
+            $this->addPostParameter($data);
+        }
+        return $this->doRequest($url, self::METHOD_POST, $headers);
     }
 
-    function delete($url, $headers)
+    /**
+     * @return HTTPResponse
+     * @throws HTTP_Request2_Exception
+     */
+    protected function doRequest($url, $method, $headers)
     {
-        throw new Exception("DELETE method unimplemented");
+        $this->setUrl($url);
+        $this->setMethod($method);
+        if ($headers) {
+            foreach ($headers as $header) {
+                $this->setHeader($header);
+            }
+        }
+        $response = $this->send();
+        return $response;
+    }
+    
+    protected function log($level, $detail) {
+        $method = $this->getMethod();
+        $url = $this->getUrl();
+        common_log($level, __CLASS__ . ": HTTP $method $url - $detail");
     }
 
+    /**
+     * Pulls up StatusNet's customized user-agent string, so services
+     * we hit can track down the responsible software.
+     *
+     * @return string
+     */
     function userAgent()
     {
         return "StatusNet/".STATUSNET_VERSION." (".STATUSNET_CODENAME.")";
     }
+
+    /**
+     * Actually performs the HTTP request and returns an HTTPResponse object
+     * with response body and header info.
+     *
+     * Wraps around parent send() to add logging and redirection processing.
+     *
+     * @return HTTPResponse
+     * @throw HTTP_Request2_Exception
+     */
+    public function send()
+    {
+        $maxRedirs = intval($this->config['max_redirs']);
+        if (empty($this->config['follow_redirects'])) {
+            $maxRedirs = 0;
+        }
+        $redirs = 0;
+        do {
+            try {
+                $response = parent::send();
+            } catch (HTTP_Request2_Exception $e) {
+                $this->log(LOG_ERR, $e->getMessage());
+                throw $e;
+            }
+            $code = $response->getStatus();
+            if ($code >= 200 && $code < 300) {
+                $reason = $response->getReasonPhrase();
+                $this->log(LOG_INFO, "$code $reason");
+            } elseif ($code >= 300 && $code < 400) {
+                $url = $this->getUrl();
+                $target = $response->getHeader('Location');
+                
+                if (++$redirs >= $maxRedirs) {
+                    common_log(LOG_ERR, __CLASS__ . ": Too many redirects: skipping $code redirect from $url to $target");
+                    break;
+                }
+                try {
+                    $this->setUrl($target);
+                    $this->setHeader('Referer', $url);
+                    common_log(LOG_INFO, __CLASS__ . ": Following $code redirect from $url to $target");
+                    continue;
+                } catch (HTTP_Request2_Exception $e) {
+                    common_log(LOG_ERR, __CLASS__ . ": Invalid $code redirect from $url to $target");
+                }
+            } else {
+                $reason = $response->getReasonPhrase();
+                $this->log(LOG_ERR, "$code $reason");
+            }
+            break;
+        } while ($maxRedirs);
+        return new HTTPResponse($response, $this->getUrl(), $redirs);
+    }
 }
diff --git a/lib/oauthclient.php b/lib/oauthclient.php
index f1827726e..1a86e2460 100644
--- a/lib/oauthclient.php
+++ b/lib/oauthclient.php
@@ -43,7 +43,7 @@ require_once 'OAuth.php';
  * @link     http://status.net/
  *
  */
-class OAuthClientCurlException extends Exception
+class OAuthClientException extends Exception
 {
 }
 
@@ -97,9 +97,14 @@ class OAuthClient
     function getRequestToken($url)
     {
         $response = $this->oAuthGet($url);
-        parse_str($response);
-        $token = new OAuthToken($oauth_token, $oauth_token_secret);
-        return $token;
+        $arr = array();
+        parse_str($response, $arr);
+        if (isset($arr['oauth_token']) && isset($arr['oauth_token_secret'])) {
+            $token = new OAuthToken($arr['oauth_token'], @$arr['oauth_token_secret']);
+            return $token;
+        } else {
+            throw new OAuthClientException();
+        }
     }
 
     /**
@@ -177,7 +182,7 @@ class OAuthClient
     }
 
     /**
-     * Make a HTTP request using cURL.
+     * Make a HTTP request.
      *
      * @param string $url    Where to make the
      * @param array  $params post parameters
@@ -186,40 +191,32 @@ class OAuthClient
      */
     function httpRequest($url, $params = null)
     {
-        $options = array(
-            CURLOPT_RETURNTRANSFER => true,
-            CURLOPT_FAILONERROR    => true,
-            CURLOPT_HEADER         => false,
-            CURLOPT_FOLLOWLOCATION => true,
-            CURLOPT_USERAGENT      => 'StatusNet',
-            CURLOPT_CONNECTTIMEOUT => 120,
-            CURLOPT_TIMEOUT        => 120,
-            CURLOPT_HTTPAUTH       => CURLAUTH_ANY,
-            CURLOPT_SSL_VERIFYPEER => false,
-
-            // Twitter is strict about accepting invalid "Expect" headers
-
-            CURLOPT_HTTPHEADER => array('Expect:')
-        );
+        $request = new HTTPClient($url);
+        $request->setConfig(array(
+            'connect_timeout' => 120,
+            'timeout' => 120,
+            'follow_redirects' => true,
+            'ssl_verify_peer' => false,
+        ));
+
+        // Twitter is strict about accepting invalid "Expect" headers
+        $request->setHeader('Expect', '');
 
         if (isset($params)) {
-            $options[CURLOPT_POST]       = true;
-            $options[CURLOPT_POSTFIELDS] = $params;
+            $request->setMethod(HTTP_Request2::METHOD_POST);
+            $request->setBody($params);
         }
 
-        $ch = curl_init($url);
-        curl_setopt_array($ch, $options);
-        $response = curl_exec($ch);
-
-        if ($response === false) {
-            $msg  = curl_error($ch);
-            $code = curl_errno($ch);
-            throw new OAuthClientCurlException($msg, $code);
+        try {
+            $response = $request->send();
+            $code = $response->getStatus();
+            if ($code < 200 || $code >= 400) {
+                throw new OAuthClientException($response->getBody(), $code);
+            }
+            return $response->getBody();
+        } catch (Exception $e) {
+            throw new OAuthClientException($e->getMessage(), $e->getCode());
         }
-
-        curl_close($ch);
-
-        return $response;
     }
 
 }
diff --git a/lib/ping.php b/lib/ping.php
index 175bf8440..5698c4038 100644
--- a/lib/ping.php
+++ b/lib/ping.php
@@ -44,20 +44,16 @@ function ping_broadcast_notice($notice) {
 																array('nickname' => $profile->nickname)),
 											   $tags));
 
-            $context = stream_context_create(array('http' => array('method' => "POST",
-                                                                   'header' =>
-                                                                   "Content-Type: text/xml\r\n".
-                                                                   "User-Agent: StatusNet/".STATUSNET_VERSION."\r\n",
-                                                                   'content' => $req)));
-            $file = file_get_contents($notify_url, false, $context);
+            $request = HTTPClient::start();
+            $httpResponse = $request->post($notify_url, array('Content-Type: text/xml'), $req);
 
-            if ($file === false || mb_strlen($file) == 0) {
+            if (!$httpResponse || mb_strlen($httpResponse->getBody()) == 0) {
                 common_log(LOG_WARNING,
                            "XML-RPC empty results for ping ($notify_url, $notice->id) ");
                 continue;
             }
 
-            $response = xmlrpc_decode($file);
+            $response = xmlrpc_decode($httpResponse->getBody());
 
             if (is_array($response) && xmlrpc_is_fault($response)) {
                 common_log(LOG_WARNING,
diff --git a/lib/snapshot.php b/lib/snapshot.php
index ede846e5b..2a10c6b93 100644
--- a/lib/snapshot.php
+++ b/lib/snapshot.php
@@ -172,26 +172,9 @@ class Snapshot
     {
         // XXX: Use OICU2 and OAuth to make authorized requests
 
-        $postdata = http_build_query($this->stats);
-
-        $opts =
-          array('http' =>
-                array(
-                      'method'  => 'POST',
-                      'header'  => 'Content-type: '.
-                                   'application/x-www-form-urlencoded',
-                      'content' => $postdata,
-                      'user_agent' => 'StatusNet/'.STATUSNET_VERSION
-                      )
-                );
-
-        $context = stream_context_create($opts);
-
         $reporturl = common_config('snapshot', 'reporturl');
-
-        $result = @file_get_contents($reporturl, false, $context);
-
-        return $result;
+        $request = HTTPClient::start();
+        $request->post($reporturl, null, $this->stats);
     }
 
     /**