diff options
82 files changed, 8058 insertions, 1142 deletions
diff --git a/.gitignore b/.gitignore index f4c2bba5f..5394f5eac 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,4 @@ config-*.php good-config.php lac08.log php.log -config.php.* + @@ -134,7 +134,7 @@ Prerequisites The following software packages are *required* for this software to run correctly. -- PHP 5.2.x. It may be possible to run this software on earlier +- PHP 5.2.3+. It may be possible to run this software on earlier versions of PHP, but many of the functions used are only available in PHP 5.2 or above. - MySQL 5.x. The Laconica database is stored, by default, in a MySQL @@ -262,13 +262,16 @@ especially if you've previously installed PHP/MySQL packages. that user's default group instead. As a last resort, you can create a new group like "mublog" and add the Web server's user to the group. -4. You should also take this moment to make your avatar subdirectory - writeable by the Web server. An insecure way to do this is: +4. You should also take this moment to make your avatar, background, and + file subdirectories writeable by the Web server. An insecure way to do + this is: chmod a+w /var/www/mublog/avatar + chmod a+w /var/www/mublog/background + chmod a+w /var/www/mublog/file - You can also make the avatar directory writeable by the Web server - group, as noted above. + You can also make the avatar, background, and file directories + writeable by the Web server group, as noted above. 5. Create a database to hold your microblog data. Something like this should work: diff --git a/actions/api.php b/actions/api.php index 452ed8e82..99ab262ad 100644 --- a/actions/api.php +++ b/actions/api.php @@ -129,6 +129,8 @@ class ApiAction extends Action 'laconica/config', 'laconica/wadl', 'tags/timeline', + 'oembed/oembed', + 'groups/show', 'groups/timeline'); static $bareauth = array('statuses/user_timeline', diff --git a/actions/attachment.php b/actions/attachment.php index ee4cd9640..c6a5d0d52 100644 --- a/actions/attachment.php +++ b/actions/attachment.php @@ -98,6 +98,28 @@ class AttachmentAction extends Action return $a->title(); } + function extraHead() + { + $this->element('link',array('rel'=>'alternate', + 'type'=>'application/json+oembed', + 'href'=>common_local_url( + 'api', + array('apiaction'=>'oembed','method'=>'oembed.json'), + array('url'=> + common_local_url('attachment', + array('attachment' => $this->attachment->id)))), + 'title'=>'oEmbed'),null); + $this->element('link',array('rel'=>'alternate', + 'type'=>'text/xml+oembed', + 'href'=>common_local_url( + 'api', + array('apiaction'=>'oembed','method'=>'oembed.xml'), + array('url'=> + common_local_url('attachment', + array('attachment' => $this->attachment->id)))), + 'title'=>'oEmbed'),null); + } + /** * Handle input * diff --git a/actions/conversation.php b/actions/conversation.php index c8755ba6e..6b5d8d54d 100644 --- a/actions/conversation.php +++ b/actions/conversation.php @@ -167,6 +167,8 @@ class ConversationTree extends NoticeList function _buildTree() { + $cnt = 0; + $this->tree = array(); $this->table = array(); diff --git a/actions/finishopenidlogin.php b/actions/finishopenidlogin.php index e9f7c746b..ff0b35218 100644 --- a/actions/finishopenidlogin.php +++ b/actions/finishopenidlogin.php @@ -83,7 +83,7 @@ class FinishopenidloginAction extends Action function showContent() { if (!empty($this->message_text)) { - $this->element('p', null, $this->message); + $this->element('div', array('class' => 'error'), $this->message_text); return; } diff --git a/actions/invite.php b/actions/invite.php index bdea4807d..26c951ed2 100644 --- a/actions/invite.php +++ b/actions/invite.php @@ -216,7 +216,7 @@ class InviteAction extends CurrentUserDesignAction $recipients = array($email); $headers['From'] = mail_notify_from(); - $headers['To'] = $email; + $headers['To'] = trim($email); $headers['Subject'] = sprintf(_('%1$s has invited you to join them on %2$s'), $bestname, $sitename); $body = sprintf(_("%1\$s has invited you to join them on %2\$s (%3\$s).\n\n". diff --git a/actions/recoverpassword.php b/actions/recoverpassword.php index 2afd052a7..721edea7f 100644 --- a/actions/recoverpassword.php +++ b/actions/recoverpassword.php @@ -194,6 +194,9 @@ class RecoverpasswordAction extends Action 'or your registered email address.')); $this->elementEnd('li'); $this->elementEnd('ul'); + $this->element('input', array('name' => 'recover', + 'type' => 'hidden', + 'value' => _('Recover'))); $this->submit('recover', _('Recover')); $this->elementEnd('fieldset'); $this->elementEnd('form'); diff --git a/actions/shownotice.php b/actions/shownotice.php index 1ec38a76b..8f73dc824 100644 --- a/actions/shownotice.php +++ b/actions/shownotice.php @@ -275,6 +275,20 @@ class ShownoticeAction extends OwnerDesignAction $this->element('meta', array('name' => 'microid', 'content' => $id->toString())); } + $this->element('link',array('rel'=>'alternate', + 'type'=>'application/json+oembed', + 'href'=>common_local_url( + 'api', + array('apiaction'=>'oembed','method'=>'oembed.json'), + array('url'=>$this->notice->uri)), + 'title'=>'oEmbed'),null); + $this->element('link',array('rel'=>'alternate', + 'type'=>'text/xml+oembed', + 'href'=>common_local_url( + 'api', + array('apiaction'=>'oembed','method'=>'oembed.xml'), + array('url'=>$this->notice->uri)), + 'title'=>'oEmbed'),null); } } diff --git a/actions/twitapifavorites.php b/actions/twitapifavorites.php index 8256668f3..6f9361065 100644 --- a/actions/twitapifavorites.php +++ b/actions/twitapifavorites.php @@ -207,32 +207,10 @@ class TwitapifavoritesAction extends TwitterapiAction $other = User::staticGet('id', $notice->profile_id); if ($other && $other->id != $user->id) { if ($other->email && $other->emailnotifyfav) { - $this->notify_mail($other, $user, $notice); + mail_notify_fave($other, $user, $notice); } # XXX: notify by IM # XXX: notify by SMS } } - - function notify_mail($other, $user, $notice) - { - $profile = $user->getProfile(); - $bestname = $profile->getBestName(); - $subject = sprintf(_('%s added your notice as a favorite'), $bestname); - $body = sprintf(_("%1\$s just added your notice from %2\$s as one of their favorites.\n\n" . - "In case you forgot, you can see the text of your notice here:\n\n" . - "%3\$s\n\n" . - "You can see the list of %1\$s's favorites here:\n\n" . - "%4\$s\n\n" . - "Faithfully yours,\n" . - "%5\$s\n"), - $bestname, - common_exact_date($notice->created), - common_local_url('shownotice', array('notice' => $notice->id)), - common_local_url('showfavorites', array('nickname' => $user->nickname)), - common_config('site', 'name')); - - mail_to_user($other, $subject, $body); - } - -}
\ No newline at end of file +} diff --git a/actions/twitapigroups.php b/actions/twitapigroups.php index 71a0776f4..82604ebff 100644 --- a/actions/twitapigroups.php +++ b/actions/twitapigroups.php @@ -51,6 +51,32 @@ require_once INSTALLDIR.'/lib/twitterapi.php'; class TwitapigroupsAction extends TwitterapiAction { + function show($args, $apidata) + { + parent::handle($args); + + common_debug("in groups api action"); + + $this->auth_user = $apidata['user']; + $group = $this->get_group($apidata['api_arg'], $apidata); + + if (empty($group)) { + $this->clientError('Not Found', 404, $apidata['content-type']); + return; + } + + switch($apidata['content-type']) { + case 'xml': + $this->show_single_xml_group($group); + break; + case 'json': + $this->show_single_json_group($group); + break; + default: + $this->clientError(_('API method not found!'), $code = 404); + } + } + function timeline($args, $apidata) { parent::handle($args); @@ -88,8 +114,7 @@ require_once INSTALLDIR.'/lib/twitterapi.php'; $this->show_xml_timeline($notice); break; case 'rss': - $this->show_rss_timeline($notice, $title, $link, - $subtitle, $suplink); + $this->show_rss_timeline($notice, $title, $link, $subtitle); break; case 'atom': if (isset($apidata['api_arg'])) { @@ -101,7 +126,7 @@ require_once INSTALLDIR.'/lib/twitterapi.php'; 'api/laconica/groups/timeline.atom'; } $this->show_atom_timeline($notice, $title, $id, $link, - $subtitle, $suplink, $selfuri); + $subtitle, null, $selfuri); break; case 'json': $this->show_json_timeline($notice); diff --git a/actions/twitapilaconica.php b/actions/twitapilaconica.php index 8cd7a64b9..442fdbcef 100644 --- a/actions/twitapilaconica.php +++ b/actions/twitapilaconica.php @@ -171,4 +171,5 @@ class TwitapilaconicaAction extends TwitterapiAction parent::handle($args); $this->serverError(_('API method under construction.'), 501); } + } diff --git a/actions/twitapioembed.php b/actions/twitapioembed.php new file mode 100644 index 000000000..3019e5878 --- /dev/null +++ b/actions/twitapioembed.php @@ -0,0 +1,173 @@ +<?php +/** + * Laconica, the distributed open-source microblogging tool + * + * Laconica-only extensions to the Twitter-like API + * + * PHP version 5 + * + * LICENCE: This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @category Twitter + * @package Laconica + * @author Evan Prodromou <evan@controlyourself.ca> + * @copyright 2008 Control Yourself, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +if (!defined('LACONICA')) { + exit(1); +} + +require_once INSTALLDIR.'/lib/twitterapi.php'; + +/** + * Oembed provider implementation + * + * This class handles all /main/oembed(.xml|.json)/ requests. + * + * @category oEmbed + * @package Laconica + * @author Craig Andrews <candrews@integralblue.com> + * @copyright 2008 Control Yourself, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +class TwitapioembedAction extends TwitterapiAction +{ + + function oembed($args, $apidata) + { + parent::handle($args); + + common_debug("in oembed api action"); + + $this->auth_user = $apidata['user']; + + $url = $args['url']; + if( substr(strtolower($url),0,strlen(common_root_url())) == strtolower(common_root_url()) ){ + $path = substr($url,strlen(common_root_url())); + + $r = Router::get(); + + $proxy_args = $r->map($path); + + if (!$proxy_args) { + $this->serverError(_("$path not found"), 404); + } + $oembed=array(); + $oembed['version']='1.0'; + $oembed['provider_name']=common_config('site', 'name'); + $oembed['provider_url']=common_root_url(); + switch($proxy_args['action']){ + case 'shownotice': + $oembed['type']='link'; + $id = $proxy_args['notice']; + $notice = Notice::staticGet($id); + if(empty($notice)){ + $this->serverError(_("notice $id not found"), 404); + } + $profile = $notice->getProfile(); + if (empty($profile)) { + $this->serverError(_('Notice has no profile'), 500); + } + if (!empty($profile->fullname)) { + $authorname = $profile->fullname . ' (' . $profile->nickname . ')'; + } else { + $authorname = $profile->nickname; + } + $oembed['title'] = sprintf(_('%1$s\'s status on %2$s'), + $authorname, + common_exact_date($notice->created)); + $oembed['author_name']=$authorname; + $oembed['author_url']=$profile->profileurl; + $oembed['url']=($notice->url?$notice->url:$notice->uri); + $oembed['html']=$notice->rendered; + break; + case 'attachment': + $id = $proxy_args['attachment']; + $attachment = File::staticGet($id); + if(empty($attachment)){ + $this->serverError(_("attachment $id not found"), 404); + } + if(empty($attachment->filename) && $file_oembed = File_oembed::staticGet('file_id', $attachment->id)){ + // Proxy the existing oembed information + $oembed['type']=$file_oembed->type; + $oembed['provider']=$file_oembed->provider; + $oembed['provider_url']=$file_oembed->provider_url; + $oembed['width']=$file_oembed->width; + $oembed['height']=$file_oembed->height; + $oembed['html']=$file_oembed->html; + $oembed['title']=$file_oembed->title; + $oembed['author_name']=$file_oembed->author_name; + $oembed['author_url']=$file_oembed->author_url; + $oembed['url']=$file_oembed->url; + }else if(substr($attachment->mimetype,0,strlen('image/'))=='image/'){ + $oembed['type']='photo'; + //TODO set width and height + //$oembed['width']= + //$oembed['height']= + $oembed['url']=$attachment->url; + }else{ + $oembed['type']='link'; + $oembed['url']=common_local_url('attachment', + array('attachment' => $attachment->id)); + } + if($attachment->title) $oembed['title']=$attachment->title; + break; + default: + $this->serverError(_("$path not supported for oembed requests"), 501); + } + + switch($apidata['content-type']){ + case 'xml': + $this->init_document('xml'); + $this->elementStart('oembed'); + $this->element('version',null,$oembed['version']); + $this->element('type',null,$oembed['type']); + if($oembed['provider_name']) $this->element('provider_name',null,$oembed['provider_name']); + if($oembed['provider_url']) $this->element('provider_url',null,$oembed['provider_url']); + if($oembed['title']) $this->element('title',null,$oembed['title']); + if($oembed['author_name']) $this->element('author_name',null,$oembed['author_name']); + if($oembed['author_url']) $this->element('author_url',null,$oembed['author_url']); + if($oembed['url']) $this->element('url',null,$oembed['url']); + if($oembed['html']) $this->element('html',null,$oembed['html']); + if($oembed['width']) $this->element('width',null,$oembed['width']); + if($oembed['height']) $this->element('height',null,$oembed['height']); + if($oembed['cache_age']) $this->element('cache_age',null,$oembed['cache_age']); + if($oembed['thumbnail_url']) $this->element('thumbnail_url',null,$oembed['thumbnail_url']); + if($oembed['thumbnail_width']) $this->element('thumbnail_width',null,$oembed['thumbnail_width']); + if($oembed['thumbnail_height']) $this->element('thumbnail_height',null,$oembed['thumbnail_height']); + + + $this->elementEnd('oembed'); + $this->end_document('xml'); + break; + case 'json': + $this->init_document('json'); + print(json_encode($oembed)); + $this->end_document('json'); + break; + default: + $this->serverError(_('content type ' . $apidata['content-type'] . ' not supported'), 501); + } + + }else{ + $this->serverError(_('Only ' . common_root_url() . ' urls over plain http please'), 404); + } + } +} + diff --git a/actions/twitapitags.php b/actions/twitapitags.php index 5c8527530..e19e1b1ed 100644 --- a/actions/twitapitags.php +++ b/actions/twitapitags.php @@ -88,8 +88,7 @@ require_once INSTALLDIR.'/lib/twitterapi.php'; $this->show_xml_timeline($notice); break; case 'rss': - $this->show_rss_timeline($notice, $title, $link, - $subtitle, $suplink); + $this->show_rss_timeline($notice, $title, $link, $subtitle); break; case 'atom': if (isset($apidata['api_arg'])) { @@ -101,7 +100,7 @@ require_once INSTALLDIR.'/lib/twitterapi.php'; 'api/laconica/tags/timeline.atom'; } $this->show_atom_timeline($notice, $title, $id, $link, - $subtitle, $suplink, $selfuri); + $subtitle, null, $selfuri); break; case 'json': $this->show_json_timeline($notice); diff --git a/classes/Design.php b/classes/Design.php index dc1712aff..9354bfcda 100644 --- a/classes/Design.php +++ b/classes/Design.php @@ -107,7 +107,7 @@ class Design extends Memcached_DataObject static function toWebColor($color) { - if (is_null($color)) { + if ($color == null) { return null; } diff --git a/classes/Fave.php b/classes/Fave.php index d1e3b01b3..11e876ff1 100644 --- a/classes/Fave.php +++ b/classes/Fave.php @@ -79,7 +79,7 @@ class Fave extends Memcached_DataObject $qry .= 'ORDER BY modified DESC '; if (!is_null($offset)) { - $qry .= "LIMIT $offset, $limit"; + $qry .= "LIMIT $limit OFFSET $offset"; } $fav->query($qry); diff --git a/classes/File.php b/classes/File.php index 56d9f9827..959301eda 100644 --- a/classes/File.php +++ b/classes/File.php @@ -79,9 +79,8 @@ class File extends Memcached_DataObject if (isset($redir_data['type']) && ('text/html' === substr($redir_data['type'], 0, 9)) - && ($oembed_data = File_oembed::_getOembed($given_url)) - && isset($oembed_data['json'])) { - File_oembed::saveNew($oembed_data['json'], $file_id); + && ($oembed_data = File_oembed::_getOembed($given_url))) { + File_oembed::saveNew($oembed_data, $file_id); } return $x; } @@ -94,7 +93,6 @@ class File extends Memcached_DataObject if (empty($file)) { $file_redir = File_redirection::staticGet('url', $given_url); if (empty($file_redir)) { - common_debug("processNew() '$given_url' not a known redirect.\n"); $redir_data = File_redirection::where($given_url); $redir_url = $redir_data['url']; if ($redir_url === $given_url) { @@ -115,7 +113,9 @@ class File extends Memcached_DataObject if (empty($x)) { $x = File::staticGet($file_id); - if (empty($x)) die('Impossible!'); + if (empty($x)) { + throw new ServerException("Robin thinks something is impossible."); + } } File_to_post::processNew($file_id, $notice_id); @@ -123,6 +123,7 @@ class File extends Memcached_DataObject } function isRespectsQuota($user,$fileSize) { + if ($fileSize > common_config('attachments', 'file_quota')) { return sprintf(_('No file may be larger than %d bytes ' . 'and the file you sent was %d bytes. Try to upload a smaller version.'), @@ -136,8 +137,7 @@ class File extends Memcached_DataObject if ($total > common_config('attachments', 'user_quota')) { return sprintf(_('A file this large would exceed your user quota of %d bytes.'), common_config('attachments', 'user_quota')); } - - $query .= ' month(modified) = month(now()) and year(modified) = year(now())'; + $query .= ' AND EXTRACT(month FROM file.modified) = EXTRACT(month FROM now()) and EXTRACT(year FROM file.modified) = EXTRACT(year FROM now())'; $this->query($query); $this->fetch(); $total = $this->total + $fileSize; diff --git a/classes/File_oembed.php b/classes/File_oembed.php index 69230e4a4..bbf112729 100644 --- a/classes/File_oembed.php +++ b/classes/File_oembed.php @@ -56,33 +56,46 @@ class File_oembed extends Memcached_DataObject return array(false, false, false); } - function _getOembed($url, $maxwidth = 500, $maxheight = 400, $format = 'json') { - $cmd = common_config('oohembed', 'endpoint') . '?url=' . urlencode($url); - if (is_int($maxwidth)) $cmd .= "&maxwidth=$maxwidth"; - if (is_int($maxheight)) $cmd .= "&maxheight=$maxheight"; - if (is_string($format)) $cmd .= "&format=$format"; - $oe = @file_get_contents($cmd); - if (false === $oe) return false; - return array($format => (('json' === $format) ? json_decode($oe, true) : $oe)); + function _getOembed($url, $maxwidth = 500, $maxheight = 400) { + require_once INSTALLDIR.'/extlib/Services/oEmbed.php'; + $parameters = array( + 'maxwidth'=>$maxwidth, + 'maxheight'=>$maxheight, + ); + try{ + $oEmbed = new Services_oEmbed($url); + $object = $oEmbed->getObject($parameters); + return $object; + }catch(Exception $e){ + try{ + $oEmbed = new Services_oEmbed($url, array( + Services_oEmbed::OPTION_API => common_config('oohembed', 'endpoint') + )); + $object = $oEmbed->getObject($parameters); + return $object; + }catch(Exception $ex){ + return false; + } + } } function saveNew($data, $file_id) { $file_oembed = new File_oembed; $file_oembed->file_id = $file_id; - $file_oembed->version = $data['version']; - $file_oembed->type = $data['type']; - if (!empty($data['provider_name'])) $file_oembed->provider = $data['provider_name']; - if (!isset($file_oembed->provider) && !empty($data['provide'])) $file_oembed->provider = $data['provider']; - if (!empty($data['provide_url'])) $file_oembed->provider_url = $data['provider_url']; - if (!empty($data['width'])) $file_oembed->width = intval($data['width']); - if (!empty($data['height'])) $file_oembed->height = intval($data['height']); - if (!empty($data['html'])) $file_oembed->html = $data['html']; - if (!empty($data['title'])) $file_oembed->title = $data['title']; - if (!empty($data['author_name'])) $file_oembed->author_name = $data['author_name']; - if (!empty($data['author_url'])) $file_oembed->author_url = $data['author_url']; - if (!empty($data['url'])) $file_oembed->url = $data['url']; + $file_oembed->version = $data->version; + $file_oembed->type = $data->type; + if (!empty($data->provider_name)) $file_oembed->provider = $data->provider_name; + if (!empty($data->provider)) $file_oembed->provider = $data->provider; + if (!empty($data->provide_url)) $file_oembed->provider_url = $data->provider_url; + if (!empty($data->width)) $file_oembed->width = intval($data->width); + if (!empty($data->height)) $file_oembed->height = intval($data->height); + if (!empty($data->html)) $file_oembed->html = $data->html; + if (!empty($data->title)) $file_oembed->title = $data->title; + if (!empty($data->author_name)) $file_oembed->author_name = $data->author_name; + if (!empty($data->author_url)) $file_oembed->author_url = $data->author_url; + if (!empty($data->url)) $file_oembed->url = $data->url; $file_oembed->insert(); - if (!empty($data['thumbnail_url'])) { + if (!empty($data->thumbnail_url)) { File_thumbnail::saveNew($data, $file_id); } } diff --git a/classes/File_thumbnail.php b/classes/File_thumbnail.php index 44b92a2fa..0b09c6af8 100644 --- a/classes/File_thumbnail.php +++ b/classes/File_thumbnail.php @@ -51,9 +51,9 @@ class File_thumbnail extends Memcached_DataObject function saveNew($data, $file_id) { $tn = new File_thumbnail; $tn->file_id = $file_id; - $tn->url = $data['thumbnail_url']; - $tn->width = intval($data['thumbnail_width']); - $tn->height = intval($data['thumbnail_height']); + $tn->url = $data->thumbnail_url; + $tn->width = intval($data->thumbnail_width); + $tn->height = intval($data->thumbnail_height); $tn->insert(); } } diff --git a/classes/Notice.php b/classes/Notice.php index 4b9a866b0..ebd5e1efd 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -98,13 +98,20 @@ class Notice extends Memcached_DataObject function saveTags() { /* extract all #hastags */ - $count = preg_match_all('/(?:^|\s)#([A-Za-z0-9_\-\.]{1,64})/', strtolower($this->content), $match); + $count = preg_match_all('/(?:^|\s)#([\pL\pN_\-\.]{1,64})/', strtolower($this->content), $match); if (!$count) { return true; } + //turn each into their canonical tag + //this is needed to remove dupes before saving e.g. #hash.tag = #hashtag + $hashtags = array(); + for($i=0; $i<count($match[1]); $i++) { + $hashtags[] = common_canonical_tag($match[1][$i]); + } + /* Add them to the database */ - foreach(array_unique($match[1]) as $hashtag) { + foreach(array_unique($hashtags) as $hashtag) { /* elide characters we don't want in the tag */ $this->saveTag($hashtag); } @@ -113,8 +120,6 @@ class Notice extends Memcached_DataObject function saveTag($hashtag) { - $hashtag = common_canonical_tag($hashtag); - $tag = new Notice_tag(); $tag->notice_id = $this->id; $tag->tag = $hashtag; @@ -177,29 +182,30 @@ class Notice extends Memcached_DataObject $notice->is_local = $is_local; } - $notice->query('BEGIN'); - - $notice->reply_to = $reply_to; if (!empty($created)) { $notice->created = $created; } else { $notice->created = common_sql_now(); } + $notice->content = $final; $notice->rendered = common_render_content($final, $notice); $notice->source = $source; $notice->uri = $uri; - if (!empty($reply_to)) { - $reply_notice = Notice::staticGet('id', $reply_to); - if (!empty($reply_notice)) { - $notice->reply_to = $reply_to; - $notice->conversation = $reply_notice->conversation; - } + $notice->reply_to = self::getReplyTo($reply_to, $profile_id, $source, $final); + + if (!empty($notice->reply_to)) { + $reply = Notice::staticGet('id', $notice->reply_to); + $notice->conversation = $reply->conversation; } if (Event::handle('StartNoticeSave', array(&$notice))) { + // XXX: some of these functions write to the DB + + $notice->query('BEGIN'); + $id = $notice->insert(); if (!$id) { @@ -207,18 +213,33 @@ class Notice extends Memcached_DataObject return _('Problem saving notice.'); } - # Update the URI after the notice is in the database - if (!$uri) { - $orig = clone($notice); + // Update ID-dependent columns: URI, conversation + + $orig = clone($notice); + + $changed = false; + + if (empty($uri)) { $notice->uri = common_notice_uri($notice); + $changed = true; + } + // If it's not part of a conversation, it's + // the beginning of a new conversation. + + if (empty($notice->conversation)) { + $notice->conversation = $notice->id; + $changed = true; + } + + if ($changed) { if (!$notice->update($orig)) { common_log_db_error($notice, 'UPDATE', __FILE__); return _('Problem saving notice.'); } } - # XXX: do we need to change this for remote users? + // XXX: do we need to change this for remote users? $notice->saveReplies(); $notice->saveTags(); @@ -226,8 +247,13 @@ class Notice extends Memcached_DataObject $notice->addToInboxes(); $notice->saveUrls(); + + // FIXME: why do we have to re-render the content? + // Remove this if it's not necessary. + $orig2 = clone($notice); - $notice->rendered = common_render_content($final, $notice); + + $notice->rendered = common_render_content($final, $notice); if (!$notice->update($orig2)) { common_log_db_error($notice, 'UPDATE', __FILE__); return _('Problem saving notice.'); @@ -284,9 +310,9 @@ class Notice extends Memcached_DataObject $notice->profile_id = $profile_id; $notice->content = $content; if (common_config('db','type') == 'pgsql') - $notice->whereAdd('extract(epoch from now() - created) < ' . common_config('site', 'dupelimit')); + $notice->whereAdd('extract(epoch from now() - created) < ' . common_config('site', 'dupelimit')); else - $notice->whereAdd('now() - created < ' . common_config('site', 'dupelimit')); + $notice->whereAdd('now() - created < ' . common_config('site', 'dupelimit')); $cnt = $notice->count(); return ($cnt == 0); @@ -874,8 +900,11 @@ class Notice extends Memcached_DataObject if ($cnt > 0) { $qry .= ', '; } - $qry .= '('.$id.', '.$this->id.', '.$source.', "'.$this->created.'") '; + $qry .= '('.$id.', '.$this->id.', '.$source.", '".$this->created. "') "; $cnt++; + if (rand() % NOTICE_INBOX_SOFT_LIMIT == 0) { + Notice_inbox::gc($id); + } if ($cnt >= MAX_BOXCARS) { $inbox = new Notice_inbox(); $inbox->query($qry); @@ -897,10 +926,14 @@ class Notice extends Memcached_DataObject { $user = new User(); + if(common_config('db','quote_identifiers')) + $user_table = '"user"'; + else $user_table = 'user'; + $qry = 'SELECT id ' . - 'FROM user JOIN subscription '. - 'ON user.id = subscription.subscriber ' . + 'FROM '. $user_table .' JOIN subscription '. + 'ON '. $user_table .'.id = subscription.subscriber ' . 'WHERE subscription.subscribed = %d '; $user->query(sprintf($qry, $this->profile_id)); @@ -1018,16 +1051,6 @@ class Notice extends Memcached_DataObject if (!$recipient) { continue; } - if ($i == 0 && ($recipient->id != $sender->id) && !$this->reply_to) { // Don't save reply to self - $reply_for = $recipient; - $recipient_notice = $reply_for->getCurrentNotice(); - if ($recipient_notice) { - $orig = clone($this); - $this->reply_to = $recipient_notice->id; - $this->conversation = $recipient_notice->conversation; - $this->update($orig); - } - } // Don't save replies from blocked profile to local user $recipient_user = User::staticGet('id', $recipient->id); if ($recipient_user && $recipient_user->hasBlocked($sender)) { @@ -1074,14 +1097,6 @@ class Notice extends Memcached_DataObject } } - // If it's not a reply, make it the root of a new conversation - - if (empty($this->conversation)) { - $orig = clone($this); - $this->conversation = $this->id; - $this->update($orig); - } - foreach (array_keys($replied) as $recipient) { $user = User::staticGet('id', $recipient); if ($user) { @@ -1253,4 +1268,76 @@ class Notice extends Memcached_DataObject return $ids; } + + /** + * Determine which notice, if any, a new notice is in reply to. + * + * For conversation tracking, we try to see where this notice fits + * in the tree. Rough algorithm is: + * + * if (reply_to is set and valid) { + * return reply_to; + * } else if ((source not API or Web) and (content starts with "T NAME" or "@name ")) { + * return ID of last notice by initial @name in content; + * } + * + * Note that all @nickname instances will still be used to save "reply" records, + * so the notice shows up in the mentioned users' "replies" tab. + * + * @param integer $reply_to ID passed in by Web or API + * @param integer $profile_id ID of author + * @param string $source Source tag, like 'web' or 'gwibber' + * @param string $content Final notice content + * + * @return integer ID of replied-to notice, or null for not a reply. + */ + + static function getReplyTo($reply_to, $profile_id, $source, $content) + { + static $lb = array('xmpp', 'mail', 'sms', 'omb'); + + // If $reply_to is specified, we check that it exists, and then + // return it if it does + + if (!empty($reply_to)) { + $reply_notice = Notice::staticGet('id', $reply_to); + if (!empty($reply_notice)) { + return $reply_to; + } + } + + // If it's not a "low bandwidth" source (one where you can't set + // a reply_to argument), we return. This is mostly web and API + // clients. + + if (!in_array($source, $lb)) { + return null; + } + + // Is there an initial @ or T? + + if (preg_match('/^T ([A-Z0-9]{1,64}) /', $content, $match) || + preg_match('/^@([a-z0-9]{1,64})\s+/', $content, $match)) { + $nickname = common_canonical_nickname($match[1]); + } else { + return null; + } + + // Figure out who that is. + + $sender = Profile::staticGet('id', $profile_id); + $recipient = common_relative_profile($sender, $nickname, common_sql_now()); + + if (empty($recipient)) { + return null; + } + + // Get their last notice + + $last = $recipient->getCurrentNotice(); + + if (!empty($last)) { + return $last->id; + } + } } diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php index 940381f84..2af34b1a4 100644 --- a/classes/Notice_inbox.php +++ b/classes/Notice_inbox.php @@ -24,6 +24,10 @@ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; // We keep 5 pages of inbox notices in memcache, +1 for pagination check define('INBOX_CACHE_WINDOW', 101); +define('NOTICE_INBOX_GC_BOXCAR', 128); +define('NOTICE_INBOX_GC_MAX', 12800); +define('NOTICE_INBOX_LIMIT', 1000); +define('NOTICE_INBOX_SOFT_LIMIT', 1000); define('NOTICE_INBOX_SOURCE_SUB', 1); define('NOTICE_INBOX_SOURCE_GROUP', 2); @@ -100,4 +104,41 @@ class Notice_inbox extends Memcached_DataObject { return Memcached_DataObject::pkeyGet('Notice_inbox', $kv); } + + static function gc($user_id) + { + $entry = new Notice_inbox(); + $entry->user_id = $user_id; + $entry->orderBy('created DESC'); + $entry->limit(NOTICE_INBOX_LIMIT - 1, NOTICE_INBOX_GC_MAX); + + $total = $entry->find(); + + if ($total > 0) { + $notices = array(); + $cnt = 0; + while ($entry->fetch()) { + $notices[] = $entry->notice_id; + $cnt++; + if ($cnt >= NOTICE_INBOX_GC_BOXCAR) { + self::deleteMatching($user_id, $notices); + $notices = array(); + $cnt = 0; + } + } + + if ($cnt > 0) { + self::deleteMatching($user_id, $notices); + $notices = array(); + } + } + } + + static function deleteMatching($user_id, $notices) + { + $entry = new Notice_inbox(); + return $entry->query('DELETE FROM notice_inbox '. + 'WHERE user_id = ' . $user_id . ' ' . + 'AND notice_id in ('.implode(',', $notices).')'); + } } diff --git a/classes/Profile.php b/classes/Profile.php index 224b61bd2..f926b2cef 100644 --- a/classes/Profile.php +++ b/classes/Profile.php @@ -199,7 +199,7 @@ class Profile extends Memcached_DataObject $query .= ' order by id DESC'; if (!is_null($offset)) { - $query .= " limit $offset, $limit"; + $query .= " LIMIT $limit OFFSET $offset"; } $notice->query($query); @@ -360,7 +360,6 @@ class Profile extends Memcached_DataObject $c->set(common_cache_key('profile:subscription_count:'.$this->id), $cnt); } - common_debug("subscriptionCount == $cnt"); return $cnt; } @@ -385,7 +384,6 @@ class Profile extends Memcached_DataObject $c->set(common_cache_key('profile:subscriber_count:'.$this->id), $cnt); } - common_debug("subscriberCount == $cnt"); return $cnt; } @@ -407,7 +405,6 @@ class Profile extends Memcached_DataObject $c->set(common_cache_key('profile:fave_count:'.$this->id), $cnt); } - common_debug("faveCount == $cnt"); return $cnt; } @@ -430,7 +427,6 @@ class Profile extends Memcached_DataObject $c->set(common_cache_key('profile:notice_count:'.$this->id), $cnt); } - common_debug("noticeCount == $cnt"); return $cnt; } diff --git a/classes/User_group.php b/classes/User_group.php index 27b444705..b1ab1c2d3 100644 --- a/classes/User_group.php +++ b/classes/User_group.php @@ -275,11 +275,14 @@ class User_group extends Memcached_DataObject // XXX: cache this $user = new User(); + if(common_config('db','quote_identifiers')) + $user_table = '"user"'; + else $user_table = 'user'; $qry = 'SELECT id ' . - 'FROM user JOIN group_member '. - 'ON user.id = group_member.profile_id ' . + 'FROM '. $user_table .' JOIN group_member '. + 'ON '. $user_table .'.id = group_member.profile_id ' . 'WHERE group_member.group_id = %d '; $user->query(sprintf($qry, $this->id)); diff --git a/db/074to080_pg.sql b/db/074to080_pg.sql new file mode 100644 index 000000000..0a7171ae5 --- /dev/null +++ b/db/074to080_pg.sql @@ -0,0 +1,108 @@ +BEGIN; +create sequence design_seq; +create table design ( + id bigint default nextval('design_seq') /* comment 'design ID'*/, + backgroundcolor integer /* comment 'main background color'*/ , + contentcolor integer /*comment 'content area background color'*/ , + sidebarcolor integer /*comment 'sidebar background color'*/ , + textcolor integer /*comment 'text color'*/ , + linkcolor integer /*comment 'link color'*/, + backgroundimage varchar(255) /*comment 'background image, if any'*/, + disposition int default 1 /*comment 'bit 1 = hide background image, bit 2 = display background image, bit 4 = tile background image'*/, + primary key (id) +); +alter table "user" + add column design_id integer references design(id); +alter table "user" + add column viewdesigns integer default 1; + +alter table notice add column + conversation integer references notice (id); + +create index notice_conversation_idx on notice(conversation); + +alter table foreign_user + alter column id TYPE bigint; + +alter table foreign_user alter column id set not null; + +alter table foreign_link + alter column foreign_id TYPE bigint; + +alter table user_group + add column design_id integer; + +/*attachments and URLs stuff */ +create sequence file_seq; +create table file ( + id bigint default nextval('file_seq') primary key /* comment 'unique identifier' */, + url varchar(255) unique, + mimetype varchar(50), + size integer, + title varchar(255), + date integer, + protected integer, + filename text /* comment 'if a local file, name of the file' */, + modified timestamp default CURRENT_TIMESTAMP /* comment 'date this record was modified'*/ +); + +create sequence file_oembed_seq; +create table file_oembed ( + file_id bigint default nextval('file_oembed_seq') primary key /* comment 'unique identifier' */, + version varchar(20), + type varchar(20), + provider varchar(50), + provider_url varchar(255), + width integer, + height integer, + html text, + title varchar(255), + author_name varchar(50), + author_url varchar(255), + url varchar(255) +); + +create sequence file_redirection_seq; +create table file_redirection ( + url varchar(255) primary key, + file_id bigint, + redirections integer, + httpcode integer +); + +create sequence file_thumbnail_seq; +create table file_thumbnail ( + file_id bigint primary key, + url varchar(255) unique, + width integer, + height integer +); +create sequence file_to_post_seq; +create table file_to_post ( + file_id bigint, + post_id bigint, + + primary key (file_id, post_id) +); + + +create table group_block ( + group_id integer not null /* comment 'group profile is blocked from' */ references user_group (id), + blocked integer not null /* comment 'profile that is blocked' */references profile (id), + blocker integer not null /* comment 'user making the block'*/ references "user" (id), + modified timestamp /* comment 'date of blocking'*/ , + + primary key (group_id, blocked) +); + +create table group_alias ( + + alias varchar(64) /* comment 'additional nickname for the group'*/ , + group_id integer not null /* comment 'group profile is blocked from'*/ references user_group (id), + modified timestamp /* comment 'date alias was created'*/, + primary key (alias) + +); +create index group_alias_group_id_idx on group_alias (group_id); + +COMMIT;
\ No newline at end of file diff --git a/db/laconica_pg.sql b/db/laconica_pg.sql index dae8b8faf..ad34720a2 100644 --- a/db/laconica_pg.sql +++ b/db/laconica_pg.sql @@ -1,508 +1,539 @@ -/* local and remote users have profiles */
-
-create sequence profile_seq;
-create table profile (
- id bigint default nextval('profile_seq') primary key /* comment 'unique identifier' */,
- nickname varchar(64) not null /* comment 'nickname or username' */,
- fullname varchar(255) /* comment 'display name' */,
- profileurl varchar(255) /* comment 'URL, cached so we dont regenerate' */,
- homepage varchar(255) /* comment 'identifying URL' */,
- bio varchar(140) /* comment 'descriptive biography' */,
- location varchar(255) /* comment 'physical location' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- textsearch tsvector
-);
-create index profile_nickname_idx on profile using btree(nickname);
-
-create table avatar (
- profile_id integer not null /* comment 'foreign key to profile table' */ references profile (id) ,
- original integer default 0 /* comment 'uploaded by user or generated?' */,
- width integer not null /* comment 'image width' */,
- height integer not null /* comment 'image height' */,
- mediatype varchar(32) not null /* comment 'file type' */,
- filename varchar(255) null /* comment 'local filename, if local' */,
- url varchar(255) unique /* comment 'avatar location' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- primary key(profile_id, width, height)
-);
-create index avatar_profile_id_idx on avatar using btree(profile_id);
-
-create sequence sms_carrier_seq;
-create table sms_carrier (
- id bigint default nextval('sms_carrier_seq') primary key /* comment 'primary key for SMS carrier' */,
- name varchar(64) unique /* comment 'name of the carrier' */,
- email_pattern varchar(255) not null /* comment 'sprintf pattern for making an email address from a phone number' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified ' */
-);
-
-/* local users */
-
-create table "user" (
- id integer primary key /* comment 'foreign key to profile table' */ references profile (id) ,
- nickname varchar(64) unique /* comment 'nickname or username, duped in profile' */,
- password varchar(255) /* comment 'salted password, can be null for OpenID users' */,
- email varchar(255) unique /* comment 'email address for password recovery etc.' */,
- incomingemail varchar(255) unique /* comment 'email address for post-by-email' */,
- emailnotifysub integer default 1 /* comment 'Notify by email of subscriptions' */,
- emailnotifyfav integer default 1 /* comment 'Notify by email of favorites' */,
- emailnotifynudge integer default 1 /* comment 'Notify by email of nudges' */,
- emailnotifymsg integer default 1 /* comment 'Notify by email of direct messages' */,
- emailnotifyattn integer default 1 /* command 'Notify by email of @-replies' */,
- emailmicroid integer default 1 /* comment 'whether to publish email microid' */,
- language varchar(50) /* comment 'preferred language' */,
- timezone varchar(50) /* comment 'timezone' */,
- emailpost integer default 1 /* comment 'Post by email' */,
- jabber varchar(255) unique /* comment 'jabber ID for notices' */,
- jabbernotify integer default 0 /* comment 'whether to send notices to jabber' */,
- jabberreplies integer default 0 /* comment 'whether to send notices to jabber on replies' */,
- jabbermicroid integer default 1 /* comment 'whether to publish xmpp microid' */,
- updatefrompresence integer default 0 /* comment 'whether to record updates from Jabber presence notices' */,
- sms varchar(64) unique /* comment 'sms phone number' */,
- carrier integer /* comment 'foreign key to sms_carrier' */ references sms_carrier (id) ,
- smsnotify integer default 0 /* comment 'whether to send notices to SMS' */,
- smsreplies integer default 0 /* comment 'whether to send notices to SMS on replies' */,
- smsemail varchar(255) /* comment 'built from sms and carrier' */,
- uri varchar(255) unique /* comment 'universally unique identifier, usually a tag URI' */,
- autosubscribe integer default 0 /* comment 'automatically subscribe to users who subscribe to us' */,
- urlshorteningservice varchar(50) default 'ur1.ca' /* comment 'service to use for auto-shortening URLs' */,
- inboxed integer default 0 /* comment 'has an inbox been created for this user?' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */
-
-);
-create index user_smsemail_idx on "user" using btree(smsemail);
-
-/* remote people */
-
-create table remote_profile (
- id integer primary key /* comment 'foreign key to profile table' */ references profile (id) ,
- uri varchar(255) unique /* comment 'universally unique identifier, usually a tag URI' */,
- postnoticeurl varchar(255) /* comment 'URL we use for posting notices' */,
- updateprofileurl varchar(255) /* comment 'URL we use for updates to this profile' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */
-);
-
-create table subscription (
- subscriber integer not null /* comment 'profile listening' */,
- subscribed integer not null /* comment 'profile being listened to' */,
- jabber integer default 1 /* comment 'deliver jabber messages' */,
- sms integer default 1 /* comment 'deliver sms messages' */,
- token varchar(255) /* comment 'authorization token' */,
- secret varchar(255) /* comment 'token secret' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- primary key (subscriber, subscribed)
-);
-create index subscription_subscriber_idx on subscription using btree(subscriber);
-create index subscription_subscribed_idx on subscription using btree(subscribed);
-
-create sequence notice_seq;
-create table notice (
-
- id bigint default nextval('notice_seq') primary key /* comment 'unique identifier' */,
- profile_id integer not null /* comment 'who made the update' */ references profile (id) ,
- uri varchar(255) unique /* comment 'universally unique identifier, usually a tag URI' */,
- content varchar(140) /* comment 'update content' */,
- rendered text /* comment 'HTML version of the content' */,
- url varchar(255) /* comment 'URL of any attachment (image, video, bookmark, whatever)' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
- reply_to integer /* comment 'notice replied to (usually a guess)' */ references notice (id) ,
- is_local integer default 0 /* comment 'notice was generated by a user' */,
- source varchar(32) /* comment 'source of comment, like "web", "im", or "clientname"' */,
- conversation integer /*id of root notice in this conversation' */ references notice (id)
-
-
-/* FULLTEXT(content) */
-);
-create index notice_profile_id_idx on notice using btree(profile_id);
-create index notice_created_idx on notice using btree(created);
-
-create table notice_source (
- code varchar(32) primary key not null /* comment 'source code' */,
- name varchar(255) not null /* comment 'name of the source' */,
- url varchar(255) not null /* comment 'url to link to' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */
-);
-
-create table reply (
-
- notice_id integer not null /* comment 'notice that is the reply' */ references notice (id) ,
- profile_id integer not null /* comment 'profile replied to' */ references profile (id) ,
- modified timestamp /* comment 'date this record was modified' */,
- replied_id integer /* comment 'notice replied to (not used, see notice.reply_to)' */,
-
- primary key (notice_id, profile_id)
-
-);
-create index reply_notice_id_idx on reply using btree(notice_id);
-create index reply_profile_id_idx on reply using btree(profile_id);
-create index reply_replied_id_idx on reply using btree(replied_id);
-
-create table fave (
-
- notice_id integer not null /* comment 'notice that is the favorite' */ references notice (id),
- user_id integer not null /* comment 'user who likes this notice' */ references "user" (id) ,
- modified timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was modified' */,
- primary key (notice_id, user_id)
-
-);
-create index fave_notice_id_idx on fave using btree(notice_id);
-create index fave_user_id_idx on fave using btree(user_id);
-create index fave_modified_idx on fave using btree(modified);
-
-/* tables for OAuth */
-
-create table consumer (
- consumer_key varchar(255) primary key /* comment 'unique identifier, root URL' */,
- seed char(32) not null /* comment 'seed for new tokens by this consumer' */,
-
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */
-);
-
-create table token (
- consumer_key varchar(255) not null /* comment 'unique identifier, root URL' */ references consumer (consumer_key),
- tok char(32) not null /* comment 'identifying value' */,
- secret char(32) not null /* comment 'secret value' */,
- type integer not null default 0 /* comment 'request or access' */,
- state integer default 0 /* comment 'for requests 0 = initial, 1 = authorized, 2 = used' */,
-
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- primary key (consumer_key, tok)
-);
-
-create table nonce (
- consumer_key varchar(255) not null /* comment 'unique identifier, root URL' */,
- tok char(32) not null /* comment 'identifying value' */,
- nonce char(32) null /* comment 'buggy old value, ignored */,
- ts integer not null /* comment 'timestamp sent' values are epoch, and only used internally */,
-
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- primary key (consumer_key, ts, nonce)
-);
-
-/* One-to-many relationship of user to openid_url */
-
-create table user_openid (
- canonical varchar(255) primary key /* comment 'Canonical true URL' */,
- display varchar(255) not null unique /* comment 'URL for viewing, may be different from canonical' */,
- user_id integer not null /* comment 'user owning this URL' */ references "user" (id) ,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */
-
-);
-create index user_openid_user_id_idx on user_openid using btree(user_id);
-
-/* These are used by JanRain OpenID library */
-
-create table oid_associations (
- server_url varchar(2047),
- handle varchar(255),
- secret bytea,
- issued integer,
- lifetime integer,
- assoc_type varchar(64),
- primary key (server_url, handle)
-);
-
-create table oid_nonces (
- server_url varchar(2047),
- "timestamp" integer,
- salt character(40),
- unique (server_url, "timestamp", salt)
-);
-
-create table confirm_address (
- code varchar(32) not null primary key /* comment 'good random code' */,
- user_id integer not null /* comment 'user who requested confirmation' */ references "user" (id),
- address varchar(255) not null /* comment 'address (email, Jabber, SMS, etc.)' */,
- address_extra varchar(255) not null default '' /* comment 'carrier ID, for SMS' */,
- address_type varchar(8) not null /* comment 'address type ("email", "jabber", "sms")' */,
- claimed timestamp /* comment 'date this was claimed for queueing' */,
- sent timestamp /* comment 'date this was sent for queueing' */,
- modified timestamp /* comment 'date this record was modified' */
-);
-
-create table remember_me (
- code varchar(32) not null primary key /* comment 'good random code' */,
- user_id integer not null /* comment 'user who is logged in' */ references "user" (id),
- modified timestamp /* comment 'date this record was modified' */
-);
-
-create table queue_item (
-
- notice_id integer not null /* comment 'notice queued' */ references notice (id) ,
- transport varchar(8) not null /* comment 'queue for what? "email", "jabber", "sms", "irc", ...' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- claimed timestamp /* comment 'date this item was claimed' */,
-
- primary key (notice_id, transport)
-
-);
-create index queue_item_created_idx on queue_item using btree(created);
-
-/* Hash tags */
-create table notice_tag (
- tag varchar( 64 ) not null /* comment 'hash tag associated with this notice' */,
- notice_id integer not null /* comment 'notice tagged' */ references notice (id) ,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
-
- primary key (tag, notice_id)
-);
-create index notice_tag_created_idx on notice_tag using btree(created);
-
-/* Synching with foreign services */
-
-create table foreign_service (
- id int not null primary key /* comment 'numeric key for service' */,
- name varchar(32) not null unique /* comment 'name of the service' */,
- description varchar(255) /* comment 'description' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */
-);
-
-create table foreign_user (
- id int not null unique /* comment 'unique numeric key on foreign service' */,
- service int not null /* comment 'foreign key to service' */ references foreign_service(id) ,
- uri varchar(255) not null unique /* comment 'identifying URI' */,
- nickname varchar(255) /* comment 'nickname on foreign service' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- primary key (id, service)
-);
-
-create table foreign_link (
- user_id int /* comment 'link to user on this system, if exists' */ references "user" (id),
- foreign_id int /* comment 'link' */ references foreign_user (id),
- service int not null /* comment 'foreign key to service' */ references foreign_service (id),
- credentials varchar(255) /* comment 'authc credentials, typically a password' */,
- noticesync int not null default 1 /* comment 'notice synchronisation, bit 1 = sync outgoing, bit 2 = sync incoming, bit 3 = filter local replies' */,
- friendsync int not null default 2 /* comment 'friend synchronisation, bit 1 = sync outgoing, bit 2 = sync incoming */,
- profilesync int not null default 1 /* comment 'profile synchronization, bit 1 = sync outgoing, bit 2 = sync incoming' */,
- last_noticesync timestamp default null /* comment 'last time notices were imported' */,
- last_friendsync timestamp default null /* comment 'last time friends were imported' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- primary key (user_id,foreign_id,service)
-);
-create index foreign_user_user_id_idx on foreign_link using btree(user_id);
-
-create table foreign_subscription (
- service int not null /* comment 'service where relationship happens' */ references foreign_service(id) ,
- subscriber int not null /* comment 'subscriber on foreign service' */ ,
- subscribed int not null /* comment 'subscribed user' */ ,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
-
- primary key (service, subscriber, subscribed)
-);
-create index foreign_subscription_subscriber_idx on foreign_subscription using btree(subscriber);
-create index foreign_subscription_subscribed_idx on foreign_subscription using btree(subscribed);
-
-create table invitation (
- code varchar(32) not null primary key /* comment 'random code for an invitation' */,
- user_id int not null /* comment 'who sent the invitation' */ references "user" (id),
- address varchar(255) not null /* comment 'invitation sent to' */,
- address_type varchar(8) not null /* comment 'address type ("email", "jabber", "sms") '*/,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */
-
-);
-create index invitation_address_idx on invitation using btree(address,address_type);
-create index invitation_user_id_idx on invitation using btree(user_id);
-
-create sequence message_seq;
-create table message (
-
- id bigint default nextval('message_seq') primary key /* comment 'unique identifier' */,
- uri varchar(255) unique /* comment 'universally unique identifier' */,
- from_profile integer not null /* comment 'who the message is from' */ references profile (id),
- to_profile integer not null /* comment 'who the message is to' */ references profile (id),
- content varchar(140) /* comment 'message content' */,
- rendered text /* comment 'HTML version of the content' */,
- url varchar(255) /* comment 'URL of any attachment (image, video, bookmark, whatever)' */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
- source varchar(32) /* comment 'source of comment, like "web", "im", or "clientname"' */
-
-);
-create index message_from_idx on message using btree(from_profile);
-create index message_to_idx on message using btree(to_profile);
-create index message_created_idx on message using btree(created);
-
-create table notice_inbox (
-
- user_id integer not null /* comment 'user receiving the message' */ references "user" (id),
- notice_id integer not null /* comment 'notice received' */ references notice (id),
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date the notice was created' */,
- source integer default 1 /* comment 'reason it is in the inbox: 1=subscription' */,
-
- primary key (user_id, notice_id)
-);
-create index notice_inbox_notice_id_idx on notice_inbox using btree(notice_id);
-
-create table profile_tag (
- tagger integer not null /* comment 'user making the tag' */ references "user" (id),
- tagged integer not null /* comment 'profile tagged' */ references profile (id),
- tag varchar(64) not null /* comment 'hash tag associated with this notice' */,
- modified timestamp /* comment 'date the tag was added' */,
-
- primary key (tagger, tagged, tag)
-);
-create index profile_tag_modified_idx on profile_tag using btree(modified);
-create index profile_tag_tagger_tag_idx on profile_tag using btree(tagger,tag);
-
-create table profile_block (
-
- blocker integer not null /* comment 'user making the block' */ references "user" (id),
- blocked integer not null /* comment 'profile that is blocked' */ references profile (id),
- modified timestamp /* comment 'date of blocking' */,
-
- primary key (blocker, blocked)
-
-);
-
-create sequence user_group_seq;
-create table user_group (
-
- id bigint default nextval('user_group_seq') primary key /* comment 'unique identifier' */,
-
- nickname varchar(64) unique /* comment 'nickname for addressing' */,
- fullname varchar(255) /* comment 'display name' */,
- homepage varchar(255) /* comment 'URL, cached so we dont regenerate' */,
- description varchar(140) /* comment 'descriptive biography' */,
- location varchar(255) /* comment 'related physical location, if any' */,
-
- original_logo varchar(255) /* comment 'original size logo' */,
- homepage_logo varchar(255) /* comment 'homepage (profile) size logo' */,
- stream_logo varchar(255) /* comment 'stream-sized logo' */,
- mini_logo varchar(255) /* comment 'mini logo' */,
-
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */
-
-);
-create index user_group_nickname_idx on user_group using btree(nickname);
-
-create table group_member (
-
- group_id integer not null /* comment 'foreign key to user_group' */ references user_group (id),
- profile_id integer not null /* comment 'foreign key to profile table' */ references profile (id),
- is_admin integer default 0 /* comment 'is this user an admin?' */,
-
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
- modified timestamp /* comment 'date this record was modified' */,
-
- primary key (group_id, profile_id)
-);
-
-create table related_group (
-
- group_id integer not null /* comment 'foreign key to user_group' */ references user_group (id) ,
- related_group_id integer not null /* comment 'foreign key to user_group' */ references user_group (id),
-
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */,
-
- primary key (group_id, related_group_id)
-
-);
-
-create table group_inbox (
- group_id integer not null /* comment 'group receiving the message' references user_group (id) */,
- notice_id integer not null /* comment 'notice received' references notice (id) */,
- created timestamp not null default CURRENT_TIMESTAMP /* comment 'date the notice was created' */,
-
- primary key (group_id, notice_id)
-);
-create index group_inbox_created_idx on group_inbox using btree(created);
-
-
-/*attachments and URLs stuff */
-create sequence file_seq;
-create table file (
- id bigint default nextval('file_seq') primary key /* comment 'unique identifier' */,
- url varchar(255) unique,
- mimetype varchar(50),
- size integer,
- title varchar(255),
- date integer,
- protected integer
-);
-
-create sequence file_oembed_seq;
-create table file_oembed (
- id bigint default nextval('file_oembed_seq') primary key /* comment 'unique identifier' */,
- file_id bigint unique,
- version varchar(20),
- type varchar(20),
- provider varchar(50),
- provider_url varchar(255),
- width integer,
- height integer,
- html text,
- title varchar(255),
- author_name varchar(50),
- author_url varchar(255),
- url varchar(255)
-);
-
-create sequence file_redirection_seq;
-create table file_redirection (
- id bigint default nextval('file_redirection_seq') primary key /* comment 'unique identifier' */,
- url varchar(255) unique,
- file_id bigint,
- redirections integer,
- httpcode integer
-);
-
-create sequence file_thumbnail_seq;
-create table file_thumbnail (
- id bigint default nextval('file_thumbnail_seq') primary key /* comment 'unique identifier' */,
- file_id bigint unique,
- url varchar(255) unique,
- width integer,
- height integer
-);
-
-create sequence file_to_post_seq;
-create table file_to_post (
- id bigint default nextval('file_to_post_seq') primary key /* comment 'unique identifier' */,
- file_id bigint,
- post_id bigint,
-
- unique(file_id, post_id)
-);
-
-create sequence design_seq;
-create table design (
- id bigint default nextval('design_seq') /* comment 'design ID'*/,
- backgroundcolor integer /* comment 'main background color'*/ ,
- contentcolor integer /*comment 'content area background color'*/ ,
- sidebarcolor integer /*comment 'sidebar background color'*/ ,
- textcolor integer /*comment 'text color'*/ ,
- linkcolor integer /*comment 'link color'*/,
- backgroundimage varchar(255) /*comment 'background image, if any'*/,
- disposition int default 1 /*comment 'bit 1 = hide background image, bit 2 = display background image, bit 4 = tile background image'*/,
- primary key (id)
-);
-
-/* Textsearch stuff */
-
-create index textsearch_idx on profile using gist(textsearch);
-create index noticecontent_idx on notice using gist(to_tsvector('english',content));
-create trigger textsearchupdate before insert or update on profile for each row
-execute procedure tsvector_update_trigger(textsearch, 'pg_catalog.english', nickname, fullname, location, bio, homepage);
-
+/* local and remote users have profiles */ + +create sequence profile_seq; +create table profile ( + id bigint default nextval('profile_seq') primary key /* comment 'unique identifier' */, + nickname varchar(64) not null /* comment 'nickname or username' */, + fullname varchar(255) /* comment 'display name' */, + profileurl varchar(255) /* comment 'URL, cached so we dont regenerate' */, + homepage varchar(255) /* comment 'identifying URL' */, + bio varchar(140) /* comment 'descriptive biography' */, + location varchar(255) /* comment 'physical location' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + textsearch tsvector +); +create index profile_nickname_idx on profile using btree(nickname); + +create table avatar ( + profile_id integer not null /* comment 'foreign key to profile table' */ references profile (id) , + original integer default 0 /* comment 'uploaded by user or generated?' */, + width integer not null /* comment 'image width' */, + height integer not null /* comment 'image height' */, + mediatype varchar(32) not null /* comment 'file type' */, + filename varchar(255) null /* comment 'local filename, if local' */, + url varchar(255) unique /* comment 'avatar location' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + primary key(profile_id, width, height) +); +create index avatar_profile_id_idx on avatar using btree(profile_id); + +create sequence sms_carrier_seq; +create table sms_carrier ( + id bigint default nextval('sms_carrier_seq') primary key /* comment 'primary key for SMS carrier' */, + name varchar(64) unique /* comment 'name of the carrier' */, + email_pattern varchar(255) not null /* comment 'sprintf pattern for making an email address from a phone number' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified ' */ +); + +create sequence design_seq; +create table design ( + id bigint default nextval('design_seq') /* comment 'design ID'*/, + backgroundcolor integer /* comment 'main background color'*/ , + contentcolor integer /*comment 'content area background color'*/ , + sidebarcolor integer /*comment 'sidebar background color'*/ , + textcolor integer /*comment 'text color'*/ , + linkcolor integer /*comment 'link color'*/, + backgroundimage varchar(255) /*comment 'background image, if any'*/, + disposition int default 1 /*comment 'bit 1 = hide background image, bit 2 = display background image, bit 4 = tile background image'*/, + primary key (id) +); + +/* local users */ + +create table "user" ( + id integer primary key /* comment 'foreign key to profile table' */ references profile (id) , + nickname varchar(64) unique /* comment 'nickname or username, duped in profile' */, + password varchar(255) /* comment 'salted password, can be null for OpenID users' */, + email varchar(255) unique /* comment 'email address for password recovery etc.' */, + incomingemail varchar(255) unique /* comment 'email address for post-by-email' */, + emailnotifysub integer default 1 /* comment 'Notify by email of subscriptions' */, + emailnotifyfav integer default 1 /* comment 'Notify by email of favorites' */, + emailnotifynudge integer default 1 /* comment 'Notify by email of nudges' */, + emailnotifymsg integer default 1 /* comment 'Notify by email of direct messages' */, + emailnotifyattn integer default 1 /* command 'Notify by email of @-replies' */, + emailmicroid integer default 1 /* comment 'whether to publish email microid' */, + language varchar(50) /* comment 'preferred language' */, + timezone varchar(50) /* comment 'timezone' */, + emailpost integer default 1 /* comment 'Post by email' */, + jabber varchar(255) unique /* comment 'jabber ID for notices' */, + jabbernotify integer default 0 /* comment 'whether to send notices to jabber' */, + jabberreplies integer default 0 /* comment 'whether to send notices to jabber on replies' */, + jabbermicroid integer default 1 /* comment 'whether to publish xmpp microid' */, + updatefrompresence integer default 0 /* comment 'whether to record updates from Jabber presence notices' */, + sms varchar(64) unique /* comment 'sms phone number' */, + carrier integer /* comment 'foreign key to sms_carrier' */ references sms_carrier (id) , + smsnotify integer default 0 /* comment 'whether to send notices to SMS' */, + smsreplies integer default 0 /* comment 'whether to send notices to SMS on replies' */, + smsemail varchar(255) /* comment 'built from sms and carrier' */, + uri varchar(255) unique /* comment 'universally unique identifier, usually a tag URI' */, + autosubscribe integer default 0 /* comment 'automatically subscribe to users who subscribe to us' */, + urlshorteningservice varchar(50) default 'ur1.ca' /* comment 'service to use for auto-shortening URLs' */, + inboxed integer default 0 /* comment 'has an inbox been created for this user?' */, + design_id integer /* comment 'id of a design' */references design(id), + viewdesigns integer default 1 /* comment 'whether to view user-provided designs'*/, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */ + +); +create index user_smsemail_idx on "user" using btree(smsemail); + +/* remote people */ + +create table remote_profile ( + id integer primary key /* comment 'foreign key to profile table' */ references profile (id) , + uri varchar(255) unique /* comment 'universally unique identifier, usually a tag URI' */, + postnoticeurl varchar(255) /* comment 'URL we use for posting notices' */, + updateprofileurl varchar(255) /* comment 'URL we use for updates to this profile' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */ +); + +create table subscription ( + subscriber integer not null /* comment 'profile listening' */, + subscribed integer not null /* comment 'profile being listened to' */, + jabber integer default 1 /* comment 'deliver jabber messages' */, + sms integer default 1 /* comment 'deliver sms messages' */, + token varchar(255) /* comment 'authorization token' */, + secret varchar(255) /* comment 'token secret' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + primary key (subscriber, subscribed) +); +create index subscription_subscriber_idx on subscription using btree(subscriber); +create index subscription_subscribed_idx on subscription using btree(subscribed); + +create sequence notice_seq; +create table notice ( + + id bigint default nextval('notice_seq') primary key /* comment 'unique identifier' */, + profile_id integer not null /* comment 'who made the update' */ references profile (id) , + uri varchar(255) unique /* comment 'universally unique identifier, usually a tag URI' */, + content varchar(140) /* comment 'update content' */, + rendered text /* comment 'HTML version of the content' */, + url varchar(255) /* comment 'URL of any attachment (image, video, bookmark, whatever)' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + reply_to integer /* comment 'notice replied to (usually a guess)' */ references notice (id) , + is_local integer default 0 /* comment 'notice was generated by a user' */, + source varchar(32) /* comment 'source of comment, like "web", "im", or "clientname"' */, + conversation integer /*id of root notice in this conversation' */ references notice (id) + + +/* FULLTEXT(content) */ +); +create index notice_profile_id_idx on notice using btree(profile_id); +create index notice_created_idx on notice using btree(created); + +create table notice_source ( + code varchar(32) primary key not null /* comment 'source code' */, + name varchar(255) not null /* comment 'name of the source' */, + url varchar(255) not null /* comment 'url to link to' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */ +); + +create table reply ( + + notice_id integer not null /* comment 'notice that is the reply' */ references notice (id) , + profile_id integer not null /* comment 'profile replied to' */ references profile (id) , + modified timestamp /* comment 'date this record was modified' */, + replied_id integer /* comment 'notice replied to (not used, see notice.reply_to)' */, + + primary key (notice_id, profile_id) + +); +create index reply_notice_id_idx on reply using btree(notice_id); +create index reply_profile_id_idx on reply using btree(profile_id); +create index reply_replied_id_idx on reply using btree(replied_id); + +create table fave ( + + notice_id integer not null /* comment 'notice that is the favorite' */ references notice (id), + user_id integer not null /* comment 'user who likes this notice' */ references "user" (id) , + modified timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was modified' */, + primary key (notice_id, user_id) + +); +create index fave_notice_id_idx on fave using btree(notice_id); +create index fave_user_id_idx on fave using btree(user_id); +create index fave_modified_idx on fave using btree(modified); + +/* tables for OAuth */ + +create table consumer ( + consumer_key varchar(255) primary key /* comment 'unique identifier, root URL' */, + seed char(32) not null /* comment 'seed for new tokens by this consumer' */, + + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */ +); + +create table token ( + consumer_key varchar(255) not null /* comment 'unique identifier, root URL' */ references consumer (consumer_key), + tok char(32) not null /* comment 'identifying value' */, + secret char(32) not null /* comment 'secret value' */, + type integer not null default 0 /* comment 'request or access' */, + state integer default 0 /* comment 'for requests 0 = initial, 1 = authorized, 2 = used' */, + + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + primary key (consumer_key, tok) +); + +create table nonce ( + consumer_key varchar(255) not null /* comment 'unique identifier, root URL' */, + tok char(32) /* comment 'buggy old value, ignored' */, + nonce char(32) null /* comment 'buggy old value, ignored */, + ts integer not null /* comment 'timestamp sent' values are epoch, and only used internally */, + + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + primary key (consumer_key, ts, nonce) +); + +/* One-to-many relationship of user to openid_url */ + +create table user_openid ( + canonical varchar(255) primary key /* comment 'Canonical true URL' */, + display varchar(255) not null unique /* comment 'URL for viewing, may be different from canonical' */, + user_id integer not null /* comment 'user owning this URL' */ references "user" (id) , + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */ + +); +create index user_openid_user_id_idx on user_openid using btree(user_id); + +/* These are used by JanRain OpenID library */ + +create table oid_associations ( + server_url varchar(2047), + handle varchar(255), + secret bytea, + issued integer, + lifetime integer, + assoc_type varchar(64), + primary key (server_url, handle) +); + +create table oid_nonces ( + server_url varchar(2047), + "timestamp" integer, + salt character(40), + unique (server_url, "timestamp", salt) +); + +create table confirm_address ( + code varchar(32) not null primary key /* comment 'good random code' */, + user_id integer not null /* comment 'user who requested confirmation' */ references "user" (id), + address varchar(255) not null /* comment 'address (email, Jabber, SMS, etc.)' */, + address_extra varchar(255) not null default '' /* comment 'carrier ID, for SMS' */, + address_type varchar(8) not null /* comment 'address type ("email", "jabber", "sms")' */, + claimed timestamp /* comment 'date this was claimed for queueing' */, + sent timestamp /* comment 'date this was sent for queueing' */, + modified timestamp /* comment 'date this record was modified' */ +); + +create table remember_me ( + code varchar(32) not null primary key /* comment 'good random code' */, + user_id integer not null /* comment 'user who is logged in' */ references "user" (id), + modified timestamp /* comment 'date this record was modified' */ +); + +create table queue_item ( + + notice_id integer not null /* comment 'notice queued' */ references notice (id) , + transport varchar(8) not null /* comment 'queue for what? "email", "jabber", "sms", "irc", ...' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + claimed timestamp /* comment 'date this item was claimed' */, + + primary key (notice_id, transport) + +); +create index queue_item_created_idx on queue_item using btree(created); + +/* Hash tags */ +create table notice_tag ( + tag varchar( 64 ) not null /* comment 'hash tag associated with this notice' */, + notice_id integer not null /* comment 'notice tagged' */ references notice (id) , + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + + primary key (tag, notice_id) +); +create index notice_tag_created_idx on notice_tag using btree(created); + +/* Synching with foreign services */ + +create table foreign_service ( + id int not null primary key /* comment 'numeric key for service' */, + name varchar(32) not null unique /* comment 'name of the service' */, + description varchar(255) /* comment 'description' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */ +); + +create table foreign_user ( + id int not null unique /* comment 'unique numeric key on foreign service' */, + service int not null /* comment 'foreign key to service' */ references foreign_service(id) , + uri varchar(255) not null unique /* comment 'identifying URI' */, + nickname varchar(255) /* comment 'nickname on foreign service' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + primary key (id, service) +); + +create table foreign_link ( + user_id int /* comment 'link to user on this system, if exists' */ references "user" (id), + foreign_id int /* comment 'link' */ references foreign_user (id), + service int not null /* comment 'foreign key to service' */ references foreign_service (id), + credentials varchar(255) /* comment 'authc credentials, typically a password' */, + noticesync int not null default 1 /* comment 'notice synchronisation, bit 1 = sync outgoing, bit 2 = sync incoming, bit 3 = filter local replies' */, + friendsync int not null default 2 /* comment 'friend synchronisation, bit 1 = sync outgoing, bit 2 = sync incoming */, + profilesync int not null default 1 /* comment 'profile synchronization, bit 1 = sync outgoing, bit 2 = sync incoming' */, + last_noticesync timestamp default null /* comment 'last time notices were imported' */, + last_friendsync timestamp default null /* comment 'last time friends were imported' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + primary key (user_id,foreign_id,service) +); +create index foreign_user_user_id_idx on foreign_link using btree(user_id); + +create table foreign_subscription ( + service int not null /* comment 'service where relationship happens' */ references foreign_service(id) , + subscriber int not null /* comment 'subscriber on foreign service' */ , + subscribed int not null /* comment 'subscribed user' */ , + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + + primary key (service, subscriber, subscribed) +); +create index foreign_subscription_subscriber_idx on foreign_subscription using btree(subscriber); +create index foreign_subscription_subscribed_idx on foreign_subscription using btree(subscribed); + +create table invitation ( + code varchar(32) not null primary key /* comment 'random code for an invitation' */, + user_id int not null /* comment 'who sent the invitation' */ references "user" (id), + address varchar(255) not null /* comment 'invitation sent to' */, + address_type varchar(8) not null /* comment 'address type ("email", "jabber", "sms") '*/, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */ + +); +create index invitation_address_idx on invitation using btree(address,address_type); +create index invitation_user_id_idx on invitation using btree(user_id); + +create sequence message_seq; +create table message ( + + id bigint default nextval('message_seq') primary key /* comment 'unique identifier' */, + uri varchar(255) unique /* comment 'universally unique identifier' */, + from_profile integer not null /* comment 'who the message is from' */ references profile (id), + to_profile integer not null /* comment 'who the message is to' */ references profile (id), + content varchar(140) /* comment 'message content' */, + rendered text /* comment 'HTML version of the content' */, + url varchar(255) /* comment 'URL of any attachment (image, video, bookmark, whatever)' */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + source varchar(32) /* comment 'source of comment, like "web", "im", or "clientname"' */ + +); +create index message_from_idx on message using btree(from_profile); +create index message_to_idx on message using btree(to_profile); +create index message_created_idx on message using btree(created); + +create table notice_inbox ( + + user_id integer not null /* comment 'user receiving the message' */ references "user" (id), + notice_id integer not null /* comment 'notice received' */ references notice (id), + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date the notice was created' */, + source integer default 1 /* comment 'reason it is in the inbox: 1=subscription' */, + + primary key (user_id, notice_id) +); +create index notice_inbox_notice_id_idx on notice_inbox using btree(notice_id); + +create table profile_tag ( + tagger integer not null /* comment 'user making the tag' */ references "user" (id), + tagged integer not null /* comment 'profile tagged' */ references profile (id), + tag varchar(64) not null /* comment 'hash tag associated with this notice' */, + modified timestamp /* comment 'date the tag was added' */, + + primary key (tagger, tagged, tag) +); +create index profile_tag_modified_idx on profile_tag using btree(modified); +create index profile_tag_tagger_tag_idx on profile_tag using btree(tagger,tag); + +create table profile_block ( + + blocker integer not null /* comment 'user making the block' */ references "user" (id), + blocked integer not null /* comment 'profile that is blocked' */ references profile (id), + modified timestamp /* comment 'date of blocking' */, + + primary key (blocker, blocked) + +); + +create sequence user_group_seq; +create table user_group ( + + id bigint default nextval('user_group_seq') primary key /* comment 'unique identifier' */, + + nickname varchar(64) unique /* comment 'nickname for addressing' */, + fullname varchar(255) /* comment 'display name' */, + homepage varchar(255) /* comment 'URL, cached so we dont regenerate' */, + description varchar(140) /* comment 'descriptive biography' */, + location varchar(255) /* comment 'related physical location, if any' */, + + original_logo varchar(255) /* comment 'original size logo' */, + homepage_logo varchar(255) /* comment 'homepage (profile) size logo' */, + stream_logo varchar(255) /* comment 'stream-sized logo' */, + mini_logo varchar(255) /* comment 'mini logo' */, + design_id integer /*comment 'id of a design' */ references design(id), + + + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */ + +); +create index user_group_nickname_idx on user_group using btree(nickname); + +create table group_member ( + + group_id integer not null /* comment 'foreign key to user_group' */ references user_group (id), + profile_id integer not null /* comment 'foreign key to profile table' */ references profile (id), + is_admin integer default 0 /* comment 'is this user an admin?' */, + + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + modified timestamp /* comment 'date this record was modified' */, + + primary key (group_id, profile_id) +); + +create table related_group ( + + group_id integer not null /* comment 'foreign key to user_group' */ references user_group (id) , + related_group_id integer not null /* comment 'foreign key to user_group' */ references user_group (id), + + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, + + primary key (group_id, related_group_id) + +); + +create table group_inbox ( + group_id integer not null /* comment 'group receiving the message' references user_group (id) */, + notice_id integer not null /* comment 'notice received' references notice (id) */, + created timestamp not null default CURRENT_TIMESTAMP /* comment 'date the notice was created' */, + primary key (group_id, notice_id) +); +create index group_inbox_created_idx on group_inbox using btree(created); + + +/*attachments and URLs stuff */ +create sequence file_seq; +create table file ( + id bigint default nextval('file_seq') primary key /* comment 'unique identifier' */, + url varchar(255) unique, + mimetype varchar(50), + size integer, + title varchar(255), + date integer, + protected integer, + filename text /* comment 'if a local file, name of the file' */, + modified timestamp default CURRENT_TIMESTAMP /* comment 'date this record was modified'*/ +); + +create sequence file_oembed_seq; +create table file_oembed ( + file_id bigint default nextval('file_oembed_seq') primary key /* comment 'unique identifier' */, + version varchar(20), + type varchar(20), + provider varchar(50), + provider_url varchar(255), + width integer, + height integer, + html text, + title varchar(255), + author_name varchar(50), + author_url varchar(255), + url varchar(255) +); + +create sequence file_redirection_seq; +create table file_redirection ( + url varchar(255) primary key, + file_id bigint, + redirections integer, + httpcode integer +); + +create sequence file_thumbnail_seq; +create table file_thumbnail ( + file_id bigint primary key, + url varchar(255) unique, + width integer, + height integer +); + +create sequence file_to_post_seq; +create table file_to_post ( + file_id bigint, + post_id bigint, + + primary key (file_id, post_id) +); + +create table group_block ( + group_id integer not null /* comment 'group profile is blocked from' */ references user_group (id), + blocked integer not null /* comment 'profile that is blocked' */references profile (id), + blocker integer not null /* comment 'user making the block'*/ references "user" (id), + modified timestamp /* comment 'date of blocking'*/ , + + primary key (group_id, blocked) +); + +create table group_alias ( + + alias varchar(64) /* comment 'additional nickname for the group'*/ , + group_id integer not null /* comment 'group profile is blocked from'*/ references user_group (id), + modified timestamp /* comment 'date alias was created'*/, + primary key (alias) + +); +create index group_alias_group_id_idx on group_alias (group_id); + +create table session ( + + id varchar(32) primary key /* comment 'session ID'*/, + session_data text /* comment 'session data'*/, + created timestamp not null DEFAULT CURRENT_TIMESTAMP /* comment 'date this record was created'*/, + modified integer DEFAULT extract(epoch from CURRENT_TIMESTAMP) /* comment 'date this record was modified'*/ +); + +create index session_modified_idx on session (modified); + + +/* Textsearch stuff */ + +create index textsearch_idx on profile using gist(textsearch); +create index noticecontent_idx on notice using gist(to_tsvector('english',content)); +create trigger textsearchupdate before insert or update on profile for each row +execute procedure tsvector_update_trigger(textsearch, 'pg_catalog.english', nickname, fullname, location, bio, homepage); + diff --git a/db/notice_source.sql b/db/notice_source.sql index 983ea9150..71fa89344 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -22,6 +22,7 @@ VALUES ('IdentiFox','IdentiFox','http://www.bitbucket.org/uncryptic/identifox/', now()), ('identitwitch','IdentiTwitch','http://richfish.org/identitwitch/', now()), ('LaTwit','LaTwit','http://latwit.mac65.com/', now()), + ('livetweeter', 'livetweeter', 'http://addons.songbirdnest.com/addon/1204', now()), ('maisha', 'Maisha', 'http://maisha.grango.org/', now()), ('mbpidgin','mbpidgin','http://code.google.com/p/microblog-purple/', now()), ('Mobidentica', 'Mobidentica', 'http://www.substanceofcode.com/software/mobidentica/', now()), diff --git a/db/sms_carrier.sql b/db/sms_carrier.sql index 6879f2089..055606f58 100644 --- a/db/sms_carrier.sql +++ b/db/sms_carrier.sql @@ -60,4 +60,5 @@ VALUES (100112, 'Cincinnati Bell Wireless', '%s@gocbw.com', now()), (100113, 'T-Mobile Germany', '%s@t-mobile-sms.de', now()), (100114, 'Vodafone Germany', '%s@vodafone-sms.de', now()), - (100115, 'E-Plus', '%s@smsmail.eplus.de', now()); + (100115, 'E-Plus', '%s@smsmail.eplus.de', now()), + (100116, 'Cellular South', '%s@csouth1.com', now()); diff --git a/doc-src/im b/doc-src/im index da07f9fe7..c722a4e2c 100644 --- a/doc-src/im +++ b/doc-src/im @@ -32,4 +32,15 @@ currently-implemented commands: you subscribe to. * **off**: Turn off notifications. You'll no longer receive Jabber notifications. - +* **stop**: Same as 'off' +* **quit**: Same as 'off' +* **help**: Show this help. List available Jabber/XMPP commands +* **follow <nickname>**: Subscribe to <nickname> +* **sub <nickname>**: Same as follow +* **leave <nickname>**: Subscribe to <nickname> +* **unsub <nickname>**: Same as leave +* **d <nickname> <text>**: Send direct message to <nickname> with message body <text> +* **get <nickname>**: Get last notice from <nickname> +* **last <nickname>**: Same as 'get' +* **whois <nickname>**: Get Profile info on <nickname> +* **fav <nickname>**: Add user's last notice as a favorite
\ No newline at end of file diff --git a/extlib/DB/DataObject.php b/extlib/DB/DataObject.php index 0c6a13dc2..8e226b8fa 100644 --- a/extlib/DB/DataObject.php +++ b/extlib/DB/DataObject.php @@ -2,11 +2,11 @@ /** * Object Based Database Query Builder and data store * - * PHP versions 4 and 5 + * For PHP versions 4,5 and 6 * - * LICENSE: This source file is subject to version 3.0 of the PHP license + * LICENSE: This source file is subject to version 3.01 of the PHP license * that is available through the world-wide-web at the following URI: - * http://www.php.net/license/3_0.txt. If you did not receive a copy of + * http://www.php.net/license/3_01.txt. If you did not receive a copy of * the PHP License and are unable to obtain it through the web, please * send a note to license@php.net so we can mail you a copy immediately. * @@ -14,8 +14,8 @@ * @package DB_DataObject * @author Alan Knowles <alan@akbkhome.com> * @copyright 1997-2006 The PHP Group - * @license http://www.php.net/license/3_0.txt PHP License 3.0 - * @version CVS: $Id: DataObject.php,v 1.439 2008/01/30 02:14:06 alan_k Exp $ + * @license http://www.php.net/license/3_01.txt PHP License 3.01 + * @version CVS: $Id: DataObject.php 284150 2009-07-15 23:27:59Z alan_k $ * @link http://pear.php.net/package/DB_DataObject */ @@ -235,7 +235,7 @@ class DB_DataObject extends DB_DataObject_Overload * @access private * @var string */ - var $_DB_DataObject_version = "1.8.8"; + var $_DB_DataObject_version = "1.8.11"; /** * The Database table (used by table extends) @@ -1027,7 +1027,13 @@ class DB_DataObject extends DB_DataObject_Overload if ($leftq || $useNative) { $table = ($quoteIdentifiers ? $DB->quoteIdentifier($this->__table) : $this->__table); - $r = $this->_query("INSERT INTO {$table} ($leftq) VALUES ($rightq) "); + + if (($dbtype == 'pgsql') && empty($leftq)) { + $r = $this->_query("INSERT INTO {$table} DEFAULT VALUES"); + } else { + $r = $this->_query("INSERT INTO {$table} ($leftq) VALUES ($rightq) "); + } + @@ -1339,7 +1345,7 @@ class DB_DataObject extends DB_DataObject_Overload * build the condition only using the object parameters. * * @access public - * @return mixed True on success, false on failure, 0 on no data affected + * @return mixed Int (No. of rows affected) on success, false on failure, 0 on no data affected */ function delete($useWhere = false) { @@ -1369,7 +1375,13 @@ class DB_DataObject extends DB_DataObject_Overload if (($this->_query !== false) && $this->_query['condition']) { $table = ($quoteIdentifiers ? $DB->quoteIdentifier($this->__table) : $this->__table); - $sql = "DELETE FROM {$table} {$this->_query['condition']}{$extra_cond}"; + $sql = "DELETE "; + // using a joined delete. - with useWhere.. + $sql .= (!empty($this->_join) && $useWhere) ? + "{$table} FROM {$table} {$this->_join} " : + "FROM {$table} "; + + $sql .= $this->_query['condition']. $extra_cond; // add limit.. @@ -1521,15 +1533,15 @@ class DB_DataObject extends DB_DataObject_Overload } $keys = $this->keys(); - if (!$keys[0] && !is_string($countWhat)) { + if (empty($keys[0]) && (!is_string($countWhat) || (strtoupper($countWhat) == 'DISTINCT'))) { $this->raiseError( - "You cannot do run count without keys - use \$do->keys('id');", + "You cannot do run count without keys - use \$do->count('id'), or use \$do->count('distinct id')';", DB_DATAOBJECT_ERROR_INVALIDARGS,PEAR_ERROR_DIE); return false; } $table = ($quoteIdentifiers ? $DB->quoteIdentifier($this->__table) : $this->__table); - $key_col = ($quoteIdentifiers ? $DB->quoteIdentifier($keys[0]) : $keys[0]); + $key_col = empty($keys[0]) ? '' : (($quoteIdentifiers ? $DB->quoteIdentifier($keys[0]) : $keys[0])); $as = ($quoteIdentifiers ? $DB->quoteIdentifier('DATAOBJECT_NUM') : 'DATAOBJECT_NUM'); // support distinct on default keys. @@ -2044,7 +2056,7 @@ class DB_DataObject extends DB_DataObject_Overload // technically postgres native here... // we need to get the new improved tabledata sorted out first. - if ( in_array($dbtype , array('psql', 'mysql', 'mysqli', 'mssql', 'ifx')) && + if ( in_array($dbtype , array('pgsql', 'mysql', 'mysqli', 'mssql', 'ifx')) && ($table[$usekey] & DB_DATAOBJECT_INT) && isset($realkeys[$usekey]) && ($realkeys[$usekey] == 'N') ) { @@ -2125,10 +2137,13 @@ class DB_DataObject extends DB_DataObject_Overload $this->_loadConfig(); } // Set database driver for reference - $db_driver = empty($_DB_DATAOBJECT['CONFIG']['db_driver']) ? 'DB' : $_DB_DATAOBJECT['CONFIG']['db_driver']; - // is it already connected ? - + $db_driver = empty($_DB_DATAOBJECT['CONFIG']['db_driver']) ? + 'DB' : $_DB_DATAOBJECT['CONFIG']['db_driver']; + + // is it already connected ? if ($this->_database_dsn_md5 && !empty($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5])) { + + // connection is an error... if (PEAR::isError($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5])) { return $this->raiseError( $_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]->message, @@ -2137,7 +2152,7 @@ class DB_DataObject extends DB_DataObject_Overload } - if (!$this->_database) { + if (empty($this->_database)) { $this->_database = $_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]->dsn['database']; $hasGetDatabase = method_exists($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5], 'getDatabase'); @@ -2166,6 +2181,7 @@ class DB_DataObject extends DB_DataObject_Overload // try and work out what to use for the dsn ! $options= &$_DB_DATAOBJECT['CONFIG']; + // if the databse dsn dis defined in the object.. $dsn = isset($this->_database_dsn) ? $this->_database_dsn : null; if (!$dsn) { @@ -2173,14 +2189,14 @@ class DB_DataObject extends DB_DataObject_Overload $this->_database = isset($options["table_{$this->__table}"]) ? $options["table_{$this->__table}"] : null; } if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { - $this->debug("Checking for database database_{$this->_database} in options","CONNECT"); + $this->debug("Checking for database specific ini ('{$this->_database}') : database_{$this->_database} in options","CONNECT"); } if ($this->_database && !empty($options["database_{$this->_database}"])) { - $dsn = $options["database_{$this->_database}"]; } else if (!empty($options['database'])) { $dsn = $options['database']; + } } @@ -2205,6 +2221,9 @@ class DB_DataObject extends DB_DataObject_Overload if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { $this->debug("USING CACHED CONNECTION", "CONNECT",3); } + + + if (!$this->_database) { $hasGetDatabase = method_exists($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5], 'getDatabase'); @@ -2221,7 +2240,7 @@ class DB_DataObject extends DB_DataObject_Overload return true; } if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { - $this->debug("NEW CONNECTION", "CONNECT",3); + $this->debug("NEW CONNECTION TP DATABASE :" .$this->_database , "CONNECT",3); /* actualy make a connection */ $this->debug(print_r($dsn,true) ." {$this->_database_dsn_md5}", "CONNECT",3); } @@ -2265,8 +2284,8 @@ class DB_DataObject extends DB_DataObject_Overload ); } - - if (!$this->_database) { + + if (empty($this->_database)) { $hasGetDatabase = method_exists($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5], 'getDatabase'); $this->_database = ($db_driver != 'DB' && $hasGetDatabase) @@ -2357,38 +2376,38 @@ class DB_DataObject extends DB_DataObject_Overload $t= explode(' ',microtime()); $_DB_DATAOBJECT['QUERYENDTIME'] = $time = $t[0]+$t[1]; - - do { - if ($_DB_driver == 'DB') { - $result = $DB->query($string); - } else { - switch (strtolower(substr(trim($string),0,6))) { + for ($tries = 0;$tries < 3;$tries++) { - case 'insert': - case 'update': - case 'delete': - $result = $DB->exec($string); - break; - - default: - $result = $DB->query($string); - break; + if ($_DB_driver == 'DB') { + + $result = $DB->query($string); + } else { + switch (strtolower(substr(trim($string),0,6))) { + + case 'insert': + case 'update': + case 'delete': + $result = $DB->exec($string); + break; + + default: + $result = $DB->query($string); + break; + } } + + // see if we got a failure.. - try again a few times.. + if (!is_a($result,'PEAR_Error')) { + break; + } + if ($result->getCode() != -14) { // *DB_ERROR_NODBSELECTED + break; // not a connection error.. + } + sleep(1); // wait before retyring.. + $DB->connect($DB->dsn); } - - // try to reconnect, at most 3 times - $again = false; - if (is_a($result, 'PEAR_Error') - AND $result->getCode() == DB_ERROR_NODBSELECTED - AND $cpt++<3) { - $DB->disconnect(); - sleep(1); - $DB->connect($DB->dsn); - $again = true; - } - - } while ($again); + if (is_a($result,'PEAR_Error')) { if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { @@ -2556,11 +2575,13 @@ class DB_DataObject extends DB_DataObject_Overload * use @ to silence it (if you are sure it is acceptable) * eg. $do = @DB_DataObject::factory('person') * - * table name will eventually be databasename/table + * table name can bedatabasename/table * - and allow modular dataobjects to be written.. * (this also helps proxy creation) * - * + * Experimental Support for Multi-Database factory eg. mydatabase.mytable + * + * * @param string $table tablename (use blank to create a new instance of the same class.) * @access private * @return DataObject|PEAR_Error @@ -2570,9 +2591,27 @@ class DB_DataObject extends DB_DataObject_Overload function factory($table = '') { global $_DB_DATAOBJECT; + + + // multi-database support.. - experimental. + $database = ''; + + if (strpos( $table,'/') !== false ) { + list($database,$table) = explode('.',$table, 2); + + } + if (empty($_DB_DATAOBJECT['CONFIG'])) { DB_DataObject::_loadConfig(); } + // no configuration available for database + if (!empty($database) && empty($_DB_DATAOBJECT['CONFIG']['database_'.$database])) { + return DB_DataObject::raiseError( + "unable to find database_{$database} in Configuration, It is required for factory with database" + , 0, PEAR_ERROR_DIE ); + } + + if ($table === '') { if (is_a($this,'DB_DataObject') && strlen($this->__table)) { @@ -2584,17 +2623,22 @@ class DB_DataObject extends DB_DataObject_Overload } } - + // does this need multi db support?? $p = isset($_DB_DATAOBJECT['CONFIG']['class_prefix']) ? $_DB_DATAOBJECT['CONFIG']['class_prefix'] : ''; $class = $p . preg_replace('/[^A-Z0-9]/i','_',ucfirst($table)); - $ce = substr(phpversion(),0,1) > 4 ? class_exists($class,false) : class_exists($class); + $class = $ce ? $class : DB_DataObject::_autoloadClass($class); // proxy = full|light if (!$class && isset($_DB_DATAOBJECT['CONFIG']['proxy'])) { + + DB_DataObject::debug("FAILED TO Autoload $database.$table - using proxy.","FACTORY",1); + + $proxyMethod = 'getProxy'.$_DB_DATAOBJECT['CONFIG']['proxy']; + // if you have loaded (some other way) - dont try and load it again.. class_exists('DB_DataObject_Generator') ? '' : require_once 'DB/DataObject/Generator.php'; @@ -2614,8 +2658,12 @@ class DB_DataObject extends DB_DataObject_Overload "factory could not find class $class from $table", DB_DATAOBJECT_ERROR_INVALIDCONFIG); } - - return new $class; + $ret = new $class; + if (!empty($database)) { + DB_DataObject::debug("Setting database to $database","FACTORY",1); + $ret->database($database); + } + return $ret; } /** * autoload Class @@ -3079,7 +3127,7 @@ class DB_DataObject extends DB_DataObject_Overload return; } - + //echo '<PRE>'; print_r(func_get_args()); $useWhereAsOn = false; // support for 2nd argument as an array of options if (is_array($joinType)) { @@ -3119,8 +3167,39 @@ class DB_DataObject extends DB_DataObject_Overload $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; + /// CHANGED 26 JUN 2009 - we prefer links from our local table over the remote one. - + /* otherwise see if there are any links from this table to the obj. */ + //print_r($this->links()); + if (($ofield === false) && ($links = $this->links())) { + foreach ($links as $k => $v) { + /* link contains {this column} = {linked table}:{linked column} */ + $ar = explode(':', $v); + // Feature Request #4266 - Allow joins with multiple keys + if (strpos($k, ',') !== false) { + $k = explode(',', $k); + } + if (strpos($ar[1], ',') !== false) { + $ar[1] = explode(',', $ar[1]); + } + + if ($ar[0] == $obj->__table) { + if ($joinCol !== false) { + if ($k == $joinCol) { + $tfield = $k; + $ofield = $ar[1]; + break; + } else { + continue; + } + } else { + $tfield = $k; + $ofield = $ar[1]; + break; + } + } + } + } /* look up the links for obj table */ //print_r($obj->links()); if (!$ofield && ($olinks = $obj->links())) { @@ -3164,37 +3243,6 @@ class DB_DataObject extends DB_DataObject_Overload } } - /* otherwise see if there are any links from this table to the obj. */ - //print_r($this->links()); - if (($ofield === false) && ($links = $this->links())) { - foreach ($links as $k => $v) { - /* link contains {this column} = {linked table}:{linked column} */ - $ar = explode(':', $v); - // Feature Request #4266 - Allow joins with multiple keys - if (strpos($k, ',') !== false) { - $k = explode(',', $k); - } - if (strpos($ar[1], ',') !== false) { - $ar[1] = explode(',', $ar[1]); - } - - if ($ar[0] == $obj->__table) { - if ($joinCol !== false) { - if ($k == $joinCol) { - $tfield = $k; - $ofield = $ar[1]; - break; - } else { - continue; - } - } else { - $tfield = $k; - $ofield = $ar[1]; - break; - } - } - } - } // finally if these two table have column names that match do a join by default on them if (($ofield === false) && $joinCol) { @@ -3383,22 +3431,25 @@ class DB_DataObject extends DB_DataObject_Overload case 'RIGHT': // others??? .. cross, left outer, right outer, natural..? // Feature Request #4266 - Allow joins with multiple keys - $this->_join .= "\n {$joinType} JOIN {$objTable} {$fullJoinAs}"; + $jadd = "\n {$joinType} JOIN {$objTable} {$fullJoinAs}"; + //$this->_join .= "\n {$joinType} JOIN {$objTable} {$fullJoinAs}"; if (is_array($ofield)) { $key_count = count($ofield); for($i = 0; $i < $key_count; $i++) { if ($i == 0) { - $this->_join .= " ON ({$joinAs}.{$ofield[$i]}={$table}.{$tfield[$i]}) "; + $jadd .= " ON ({$joinAs}.{$ofield[$i]}={$table}.{$tfield[$i]}) "; } else { - $this->_join .= " AND {$joinAs}.{$ofield[$i]}={$table}.{$tfield[$i]} "; + $jadd .= " AND {$joinAs}.{$ofield[$i]}={$table}.{$tfield[$i]} "; } } - $this->_join .= ' ' . $appendJoin . ' '; + $jadd .= ' ' . $appendJoin . ' '; } else { - $this->_join .= " ON ({$joinAs}.{$ofield}={$table}.{$tfield}) {$appendJoin} "; + $jadd .= " ON ({$joinAs}.{$ofield}={$table}.{$tfield}) {$appendJoin} "; } - + // jadd avaliable for debugging join build. + //echo $jadd ."\n"; + $this->_join .= $jadd; break; case '': // this is just a standard multitable select.. @@ -3459,7 +3510,7 @@ class DB_DataObject extends DB_DataObject_Overload continue; } - if (empty($from[$k]) && $skipEmpty) { + if (empty($from[sprintf($format,$k)]) && $skipEmpty) { continue; } diff --git a/extlib/DB/DataObject/Cast.php b/extlib/DB/DataObject/Cast.php index 616abb55e..095d2a4d2 100644 --- a/extlib/DB/DataObject/Cast.php +++ b/extlib/DB/DataObject/Cast.php @@ -15,9 +15,9 @@ * @category Database * @package DB_DataObject * @author Alan Knowles <alan@akbkhome.com> - * @copyright 1997-2006 The PHP Group + * @copyright 1997-2008 The PHP Group * @license http://www.php.net/license/3_0.txt PHP License 3.0 - * @version CVS: $Id: Cast.php,v 1.15 2005/07/07 05:30:53 alan_k Exp $ + * @version CVS: $Id: Cast.php 264148 2008-08-04 03:44:59Z alan_k $ * @link http://pear.php.net/package/DB_DataObject */ @@ -391,7 +391,10 @@ class DB_DataObject_Cast { // this is funny - the parameter order is reversed ;) return "'".mysqli_real_escape_string($db->connection, $this->value)."'"; - + case 'sqlite': + // this is funny - the parameter order is reversed ;) + return "'".sqlite_escape_string($this->value)."'"; + default: return PEAR::raiseError("DB_DataObject_Cast cant handle blobs for Database:{$db->dsn['phptype']} Yet"); diff --git a/extlib/DB/DataObject/Error.php b/extlib/DB/DataObject/Error.php index 05a741408..382115453 100644 --- a/extlib/DB/DataObject/Error.php +++ b/extlib/DB/DataObject/Error.php @@ -18,7 +18,7 @@ * @author Alan Knowles <alan@akbkhome.com> * @copyright 1997-2006 The PHP Group * @license http://www.php.net/license/3_0.txt PHP License 3.0 - * @version CVS: $Id: Error.php,v 1.3 2005/03/23 02:35:35 alan_k Exp $ + * @version CVS: $Id: Error.php 277015 2009-03-12 05:51:03Z alan_k $ * @link http://pear.php.net/package/DB_DataObject */ diff --git a/extlib/DB/DataObject/Generator.php b/extlib/DB/DataObject/Generator.php index de16af692..ff6e42c7d 100644 --- a/extlib/DB/DataObject/Generator.php +++ b/extlib/DB/DataObject/Generator.php @@ -4,9 +4,9 @@ * * PHP versions 4 and 5 * - * LICENSE: This source file is subject to version 3.0 of the PHP license + * LICENSE: This source file is subject to version 3.01 of the PHP license * that is available through the world-wide-web at the following URI: - * http://www.php.net/license/3_0.txt. If you did not receive a copy of + * http://www.php.net/license/3_01.txt. If you did not receive a copy of * the PHP License and are unable to obtain it through the web, please * send a note to license@php.net so we can mail you a copy immediately. * @@ -14,8 +14,8 @@ * @package DB_DataObject * @author Alan Knowles <alan@akbkhome.com> * @copyright 1997-2006 The PHP Group - * @license http://www.php.net/license/3_0.txt PHP License 3.0 - * @version CVS: $Id: Generator.php,v 1.141 2008/01/30 02:29:39 alan_k Exp $ + * @license http://www.php.net/license/3_01.txt PHP License 3.01 + * @version CVS: $Id: Generator.php 284150 2009-07-15 23:27:59Z alan_k $ * @link http://pear.php.net/package/DB_DataObject */ @@ -193,7 +193,11 @@ class DB_DataObject_Generator extends DB_DataObject /** * set portability and some modules to fetch the informations */ - $__DB->setOption('portability', MDB2_PORTABILITY_ALL ^ MDB2_PORTABILITY_FIX_CASE); + $db_options = PEAR::getStaticProperty('MDB2','options'); + if (empty($db_options)) { + $__DB->setOption('portability', MDB2_PORTABILITY_ALL ^ MDB2_PORTABILITY_FIX_CASE); + } + $__DB->loadModule('Manager'); $__DB->loadModule('Reverse'); } @@ -265,12 +269,7 @@ class DB_DataObject_Generator extends DB_DataObject } else { $defs = $__DB->reverse->tableInfo($quotedTable); // rename the length value, so it matches db's return. - foreach ($defs as $k => $v) { - if (!isset($defs[$k]['length'])) { - continue; - } - $defs[$k]['len'] = $defs[$k]['length']; - } + } if (is_a($defs,'PEAR_Error')) { @@ -286,7 +285,10 @@ class DB_DataObject_Generator extends DB_DataObject if (!is_array($def)) { continue; } - + // rename the length value, so it matches db's return. + if (isset($def['length']) && !isset($def['len'])) { + $def['len'] = $def['length']; + } $this->_definitions[$table][] = (object) $def; } @@ -391,7 +393,10 @@ class DB_DataObject_Generator extends DB_DataObject $fk = array(); foreach($this->tables as $this->table) { - $res =& $DB->query('SHOW CREATE TABLE ' . $this->table); + $quotedTable = !empty($options['quote_identifiers_tableinfo']) ? $DB->quoteIdentifier($table) : $this->table; + + $res =& $DB->query('SHOW CREATE TABLE ' . $quotedTable ); + if (PEAR::isError($res)) { die($res->getMessage()); } @@ -467,7 +472,7 @@ class DB_DataObject_Generator extends DB_DataObject function _generateDefinitionsTable() { global $_DB_DATAOBJECT; - + $options = PEAR::getStaticProperty('DB_DataObject','options'); $defs = $this->_definitions[$this->table]; $this->_newConfig .= "\n[{$this->table}]\n"; $keys_out = "\n[{$this->table}__keys]\n"; @@ -551,6 +556,9 @@ class DB_DataObject_Generator extends DB_DataObject case 'ENUM': case 'SET': // not really but oh well + + case 'POINT': // mysql geometry stuff - not really string - but will do.. + case 'TIMESTAMPTZ': // postgres case 'BPCHAR': // postgres case 'INTERVAL': // postgres (eg. '12 days') @@ -594,14 +602,18 @@ class DB_DataObject_Generator extends DB_DataObject DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME; break; - - case 'TINYBLOB': + case 'BLOB': /// these should really be ignored!!!??? + case 'TINYBLOB': case 'MEDIUMBLOB': case 'LONGBLOB': + + case 'CLOB': // oracle character lob support + case 'BYTEA': // postgres blob support.. $type = DB_DATAOBJECT_STR + DB_DATAOBJECT_BLOB; break; + default: echo "*****************************************************************\n". "** WARNING UNKNOWN TYPE **\n". @@ -653,7 +665,9 @@ class DB_DataObject_Generator extends DB_DataObject // only use primary key or nextval(), cause the setFrom blocks you setting all key items... // if no keys exist fall back to using unique //echo "\n{$t->name} => {$t->flags}\n"; - if (preg_match("/(auto_increment|nextval\()/i",rawurldecode($t->flags)) + $secondary_key_match = isset($options['generator_secondary_key_match']) ? $options['generator_secondary_key_match'] : 'primary|unique'; + + if (preg_match('/(auto_increment|nextval\()/i',rawurldecode($t->flags)) || (isset($t->autoincrement) && ($t->autoincrement === true))) { // native sequences = 2 @@ -662,7 +676,7 @@ class DB_DataObject_Generator extends DB_DataObject } $ret_keys_primary[$t->name] = 'N'; - } else if (preg_match("/(primary|unique)/i",$t->flags)) { + } else if ($secondary_key_match && preg_match('/('.$secondary_key_match.')/i',$t->flags)) { // keys.. = 1 $key_type = 'K'; if (!preg_match("/(primary)/i",$t->flags)) { @@ -868,10 +882,13 @@ class DB_DataObject_Generator extends DB_DataObject // then we should add var $_database = here // as database names may not always match.. + if (empty($GLOBALS['_DB_DATAOBJECT']['CONFIG'])) { + DB_DataObject::_loadConfig(); + } + + // Only include the $_database property if the omit_database_var is unset or false - - - if (isset($options["database_{$this->_database}"])) { + if (isset($options["database_{$this->_database}"]) && empty($GLOBALS['_DB_DATAOBJECT']['CONFIG']['generator_omit_database_var'])) { $body .= " {$var} \$_database = '{$this->_database}'; {$p}// database name (used with database_{*} config)\n"; } @@ -904,9 +921,10 @@ class DB_DataObject_Generator extends DB_DataObject $padding = (30 - strlen($t->name)); if ($padding < 2) $padding =2; $p = str_repeat(' ',$padding) ; - - $body .=" {$var} \${$t->name}; {$p}// {$t->type}({$t->len}) {$t->flags}\n"; - + + $length = empty($t->len) ? '' : '('.$t->len.')'; + $body .=" {$var} \${$t->name}; {$p}// {$t->type}$length {$t->flags}\n"; + // can not do set as PEAR::DB table info doesnt support it. //if (substr($t->Type,0,3) == "set") // $sets[$t->Field] = "array".substr($t->Type,3); @@ -1185,7 +1203,7 @@ class DB_DataObject_Generator extends DB_DataObject $__DB->loadModule('Manager'); $__DB->loadModule('Reverse'); } - $quotedTable = !empty($options['quote_identifiers']) ? + $quotedTable = !empty($options['quote_identifiers_tableinfo']) ? $__DB->quoteIdentifier($table) : $table; if (!$is_MDB2) { diff --git a/extlib/DB/DataObject/createTables.php b/extlib/DB/DataObject/createTables.php index c0659574e..d54d28c24 100755..100644 --- a/extlib/DB/DataObject/createTables.php +++ b/extlib/DB/DataObject/createTables.php @@ -16,7 +16,7 @@ // | Author: Alan Knowles <alan@akbkhome.com> // +----------------------------------------------------------------------+ // -// $Id: createTables.php,v 1.24 2006/01/13 01:27:55 alan_k Exp $ +// $Id: createTables.php 277015 2009-03-12 05:51:03Z alan_k $ // // since this version doesnt use overload, diff --git a/extlib/Services/oEmbed.php b/extlib/Services/oEmbed.php index 5d38ed883..7d507b6f6 100644 --- a/extlib/Services/oEmbed.php +++ b/extlib/Services/oEmbed.php @@ -162,7 +162,7 @@ class Services_oEmbed } if ($this->options[self::OPTION_API] === null) { - $this->options[self::OPTION_API] = $this->discover(); + $this->options[self::OPTION_API] = $this->discover($url); } } @@ -319,7 +319,7 @@ class Services_oEmbed } } - return (isset($ret['json']) ? $ret['json'] : array_pop($ret)); + return (isset($ret['application/json']) ? $ret['application/json'] : array_pop($ret)); } /** diff --git a/extlib/htmLawed/htmLawed.php b/extlib/htmLawed/htmLawed.php new file mode 100644 index 000000000..17f6e98ca --- /dev/null +++ b/extlib/htmLawed/htmLawed.php @@ -0,0 +1,715 @@ +<?php + +/* +htmLawed 1.1.8.1, 16 July 2009 +Copyright Santosh Patnaik +GPL v3 license +A PHP Labware internal utility; www.bioinformatics.org/phplabware/internal_utilities/htmLawed + +See htmLawed_README.txt/htm +*/ + +function htmLawed($t, $C=1, $S=array()){ +$C = is_array($C) ? $C : array(); +if(!empty($C['valid_xhtml'])){ + $C['elements'] = empty($C['elements']) ? '*-center-dir-font-isindex-menu-s-strike-u' : $C['elements']; + $C['make_tag_strict'] = isset($C['make_tag_strict']) ? $C['make_tag_strict'] : 2; + $C['xml:lang'] = isset($C['xml:lang']) ? $C['xml:lang'] : 2; +} +// config eles +$e = array('a'=>1, 'abbr'=>1, 'acronym'=>1, 'address'=>1, 'applet'=>1, 'area'=>1, 'b'=>1, 'bdo'=>1, 'big'=>1, 'blockquote'=>1, 'br'=>1, 'button'=>1, 'caption'=>1, 'center'=>1, 'cite'=>1, 'code'=>1, 'col'=>1, 'colgroup'=>1, 'dd'=>1, 'del'=>1, 'dfn'=>1, 'dir'=>1, 'div'=>1, 'dl'=>1, 'dt'=>1, 'em'=>1, 'embed'=>1, 'fieldset'=>1, 'font'=>1, 'form'=>1, 'h1'=>1, 'h2'=>1, 'h3'=>1, 'h4'=>1, 'h5'=>1, 'h6'=>1, 'hr'=>1, 'i'=>1, 'iframe'=>1, 'img'=>1, 'input'=>1, 'ins'=>1, 'isindex'=>1, 'kbd'=>1, 'label'=>1, 'legend'=>1, 'li'=>1, 'map'=>1, 'menu'=>1, 'noscript'=>1, 'object'=>1, 'ol'=>1, 'optgroup'=>1, 'option'=>1, 'p'=>1, 'param'=>1, 'pre'=>1, 'q'=>1, 'rb'=>1, 'rbc'=>1, 'rp'=>1, 'rt'=>1, 'rtc'=>1, 'ruby'=>1, 's'=>1, 'samp'=>1, 'script'=>1, 'select'=>1, 'small'=>1, 'span'=>1, 'strike'=>1, 'strong'=>1, 'sub'=>1, 'sup'=>1, 'table'=>1, 'tbody'=>1, 'td'=>1, 'textarea'=>1, 'tfoot'=>1, 'th'=>1, 'thead'=>1, 'tr'=>1, 'tt'=>1, 'u'=>1, 'ul'=>1, 'var'=>1); // 86/deprecated+embed+ruby +if(!empty($C['safe'])){ + unset($e['applet'], $e['embed'], $e['iframe'], $e['object'], $e['script']); +} +$x = !empty($C['elements']) ? str_replace(array("\n", "\r", "\t", ' '), '', $C['elements']) : '*'; +if($x == '-*'){$e = array();} +elseif(strpos($x, '*') === false){$e = array_flip(explode(',', $x));} +else{ + if(isset($x[1])){ + preg_match_all('`(?:^|-|\+)[^\-+]+?(?=-|\+|$)`', $x, $m, PREG_SET_ORDER); + for($i=count($m); --$i>=0;){$m[$i] = $m[$i][0];} + foreach($m as $v){ + if($v[0] == '+'){$e[substr($v, 1)] = 1;} + if($v[0] == '-' && isset($e[($v = substr($v, 1))]) && !in_array('+'. $v, $m)){unset($e[$v]);} + } + } +} +$C['elements'] =& $e; +// config attrs +$x = !empty($C['deny_attribute']) ? str_replace(array("\n", "\r", "\t", ' '), '', $C['deny_attribute']) : ''; +$x = array_flip((isset($x[0]) && $x[0] == '*') ? explode('-', $x) : explode(',', $x. (!empty($C['safe']) ? ',on*' : ''))); +if(isset($x['on*'])){ + unset($x['on*']); + $x += array('onblur'=>1, 'onchange'=>1, 'onclick'=>1, 'ondblclick'=>1, 'onfocus'=>1, 'onkeydown'=>1, 'onkeypress'=>1, 'onkeyup'=>1, 'onmousedown'=>1, 'onmousemove'=>1, 'onmouseout'=>1, 'onmouseover'=>1, 'onmouseup'=>1, 'onreset'=>1, 'onselect'=>1, 'onsubmit'=>1); +} +$C['deny_attribute'] = $x; +// config URL +$x = (isset($C['schemes'][2]) && strpos($C['schemes'], ':')) ? strtolower($C['schemes']) : 'href: aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet; *:file, http, https'; +$C['schemes'] = array(); +foreach(explode(';', str_replace(array(' ', "\t", "\r", "\n"), '', $x)) as $v){ + $x = $x2 = null; list($x, $x2) = explode(':', $v, 2); + if($x2){$C['schemes'][$x] = array_flip(explode(',', $x2));} +} +if(!isset($C['schemes']['*'])){$C['schemes']['*'] = array('file'=>1, 'http'=>1, 'https'=>1,);} +if(!empty($C['safe']) && empty($C['schemes']['style'])){$C['schemes']['style'] = array('nil'=>1);} +$C['abs_url'] = isset($C['abs_url']) ? $C['abs_url'] : 0; +if(!isset($C['base_url']) or !preg_match('`^[a-zA-Z\d.+\-]+://[^/]+/(.+?/)?$`', $C['base_url'])){ + $C['base_url'] = $C['abs_url'] = 0; +} +// config rest +$C['and_mark'] = empty($C['and_mark']) ? 0 : 1; +$C['anti_link_spam'] = (isset($C['anti_link_spam']) && is_array($C['anti_link_spam']) && count($C['anti_link_spam']) == 2 && (empty($C['anti_link_spam'][0]) or hl_regex($C['anti_link_spam'][0])) && (empty($C['anti_link_spam'][1]) or hl_regex($C['anti_link_spam'][1]))) ? $C['anti_link_spam'] : 0; +$C['anti_mail_spam'] = isset($C['anti_mail_spam']) ? $C['anti_mail_spam'] : 0; +$C['balance'] = isset($C['balance']) ? (bool)$C['balance'] : 1; +$C['cdata'] = isset($C['cdata']) ? $C['cdata'] : (empty($C['safe']) ? 3 : 0); +$C['clean_ms_char'] = empty($C['clean_ms_char']) ? 0 : $C['clean_ms_char']; +$C['comment'] = isset($C['comment']) ? $C['comment'] : (empty($C['safe']) ? 3 : 0); +$C['css_expression'] = empty($C['css_expression']) ? 0 : 1; +$C['hexdec_entity'] = isset($C['hexdec_entity']) ? $C['hexdec_entity'] : 1; +$C['hook'] = (!empty($C['hook']) && function_exists($C['hook'])) ? $C['hook'] : 0; +$C['hook_tag'] = (!empty($C['hook_tag']) && function_exists($C['hook_tag'])) ? $C['hook_tag'] : 0; +$C['keep_bad'] = isset($C['keep_bad']) ? $C['keep_bad'] : 6; +$C['lc_std_val'] = isset($C['lc_std_val']) ? (bool)$C['lc_std_val'] : 1; +$C['make_tag_strict'] = isset($C['make_tag_strict']) ? $C['make_tag_strict'] : 1; +$C['named_entity'] = isset($C['named_entity']) ? (bool)$C['named_entity'] : 1; +$C['no_deprecated_attr'] = isset($C['no_deprecated_attr']) ? $C['no_deprecated_attr'] : 1; +$C['parent'] = isset($C['parent'][0]) ? strtolower($C['parent']) : 'body'; +$C['show_setting'] = !empty($C['show_setting']) ? $C['show_setting'] : 0; +$C['style_pass'] = empty($C['style_pass']) ? 0 : 1; +$C['tidy'] = empty($C['tidy']) ? 0 : $C['tidy']; +$C['unique_ids'] = isset($C['unique_ids']) ? $C['unique_ids'] : 1; +$C['xml:lang'] = isset($C['xml:lang']) ? $C['xml:lang'] : 0; + +if(isset($GLOBALS['C'])){$reC = $GLOBALS['C'];} +$GLOBALS['C'] = $C; +$S = is_array($S) ? $S : hl_spec($S); +if(isset($GLOBALS['S'])){$reS = $GLOBALS['S'];} +$GLOBALS['S'] = $S; + +$t = preg_replace('`[\x00-\x08\x0b-\x0c\x0e-\x1f]`', '', $t); +if($C['clean_ms_char']){ + $x = array("\x7f"=>'', "\x80"=>'€', "\x81"=>'', "\x83"=>'ƒ', "\x85"=>'…', "\x86"=>'†', "\x87"=>'‡', "\x88"=>'ˆ', "\x89"=>'‰', "\x8a"=>'Š', "\x8b"=>'‹', "\x8c"=>'Œ', "\x8d"=>'', "\x8e"=>'Ž', "\x8f"=>'', "\x90"=>'', "\x95"=>'•', "\x96"=>'–', "\x97"=>'—', "\x98"=>'˜', "\x99"=>'™', "\x9a"=>'š', "\x9b"=>'›', "\x9c"=>'œ', "\x9d"=>'', "\x9e"=>'ž', "\x9f"=>'Ÿ'); + $x = $x + ($C['clean_ms_char'] == 1 ? array("\x82"=>'‚', "\x84"=>'„', "\x91"=>'‘', "\x92"=>'’', "\x93"=>'“', "\x94"=>'”') : array("\x82"=>'\'', "\x84"=>'"', "\x91"=>'\'', "\x92"=>'\'', "\x93"=>'"', "\x94"=>'"')); + $t = strtr($t, $x); +} +if($C['cdata'] or $C['comment']){$t = preg_replace_callback('`<!(?:(?:--.*?--)|(?:\[CDATA\[.*?\]\]))>`sm', 'hl_cmtcd', $t);} +$t = preg_replace_callback('`&([A-Za-z][A-Za-z0-9]{1,30}|#(?:[0-9]{1,8}|[Xx][0-9A-Fa-f]{1,7}));`', 'hl_ent', str_replace('&', '&', $t)); +if($C['unique_ids'] && !isset($GLOBALS['hl_Ids'])){$GLOBALS['hl_Ids'] = array();} +if($C['hook']){$t = $C['hook']($t, $C, $S);} +if($C['show_setting'] && preg_match('`^[a-z][a-z0-9_]*$`i', $C['show_setting'])){ + $GLOBALS[$C['show_setting']] = array('config'=>$C, 'spec'=>$S, 'time'=>microtime()); +} +// main +$t = preg_replace_callback('`<(?:(?:\s|$)|(?:[^>]*(?:>|$)))|>`m', 'hl_tag', $t); +$t = $C['balance'] ? hl_bal($t, $C['keep_bad'], $C['parent']) : $t; +$t = (($C['cdata'] or $C['comment']) && strpos($t, "\x01") !== false) ? str_replace(array("\x01", "\x02", "\x03", "\x04", "\x05"), array('', '', '&', '<', '>'), $t) : $t; +$t = $C['tidy'] ? hl_tidy($t, $C['tidy'], $C['parent']) : $t; +unset($C, $e); +if(isset($reC)){$GLOBALS['C'] = $reC;} +if(isset($reS)){$GLOBALS['S'] = $reS;} +return $t; +// eof +} + +function hl_attrval($t, $p){ +// check attr val against $S +$o = 1; $l = strlen($t); +foreach($p as $k=>$v){ + switch($k){ + case 'maxlen':if($l > $v){$o = 0;} + break; case 'minlen': if($l < $v){$o = 0;} + break; case 'maxval': if((float)($t) > $v){$o = 0;} + break; case 'minval': if((float)($t) < $v){$o = 0;} + break; case 'match': if(!preg_match($v, $t)){$o = 0;} + break; case 'nomatch': if(preg_match($v, $t)){$o = 0;} + break; case 'oneof': + $m = 0; + foreach(explode('|', $v) as $n){if($t == $n){$m = 1; break;}} + $o = $m; + break; case 'noneof': + $m = 1; + foreach(explode('|', $v) as $n){if($t == $n){$m = 0; break;}} + $o = $m; + break; default: + break; + } + if(!$o){break;} +} +return ($o ? $t : (isset($p['default']) ? $p['default'] : 0)); +// eof +} + +function hl_bal($t, $do=1, $in='div'){ +// balance tags +// by content +$cB = array('blockquote'=>1, 'form'=>1, 'map'=>1, 'noscript'=>1); // Block +$cE = array('area'=>1, 'br'=>1, 'col'=>1, 'embed'=>1, 'hr'=>1, 'img'=>1, 'input'=>1, 'isindex'=>1, 'param'=>1); // Empty +$cF = array('button'=>1, 'del'=>1, 'div'=>1, 'dd'=>1, 'fieldset'=>1, 'iframe'=>1, 'ins'=>1, 'li'=>1, 'noscript'=>1, 'object'=>1, 'td'=>1, 'th'=>1); // Flow; later context-wise dynamic move of ins & del to $cI +$cI = array('a'=>1, 'abbr'=>1, 'acronym'=>1, 'address'=>1, 'b'=>1, 'bdo'=>1, 'big'=>1, 'caption'=>1, 'cite'=>1, 'code'=>1, 'dfn'=>1, 'dt'=>1, 'em'=>1, 'font'=>1, 'h1'=>1, 'h2'=>1, 'h3'=>1, 'h4'=>1, 'h5'=>1, 'h6'=>1, 'i'=>1, 'kbd'=>1, 'label'=>1, 'legend'=>1, 'p'=>1, 'pre'=>1, 'q'=>1, 'rb'=>1, 'rt'=>1, 's'=>1, 'samp'=>1, 'small'=>1, 'span'=>1, 'strike'=>1, 'strong'=>1, 'sub'=>1, 'sup'=>1, 'tt'=>1, 'u'=>1, 'var'=>1); // Inline +$cN = array('a'=>array('a'=>1), 'button'=>array('a'=>1, 'button'=>1, 'fieldset'=>1, 'form'=>1, 'iframe'=>1, 'input'=>1, 'label'=>1, 'select'=>1, 'textarea'=>1), 'fieldset'=>array('fieldset'=>1), 'form'=>array('form'=>1), 'label'=>array('label'=>1), 'noscript'=>array('script'=>1), 'pre'=>array('big'=>1, 'font'=>1, 'img'=>1, 'object'=>1, 'script'=>1, 'small'=>1, 'sub'=>1, 'sup'=>1), 'rb'=>array('ruby'=>1), 'rt'=>array('ruby'=>1)); // Illegal +$cN2 = array_keys($cN); +$cR = array('blockquote'=>1, 'dir'=>1, 'dl'=>1, 'form'=>1, 'map'=>1, 'menu'=>1, 'noscript'=>1, 'ol'=>1, 'optgroup'=>1, 'rbc'=>1, 'rtc'=>1, 'ruby'=>1, 'select'=>1, 'table'=>1, 'tbody'=>1, 'tfoot'=>1, 'thead'=>1, 'tr'=>1, 'ul'=>1); +$cS = array('colgroup'=>array('col'=>1), 'dir'=>array('li'), 'dl'=>array('dd'=>1, 'dt'=>1), 'menu'=>array('li'=>1), 'ol'=>array('li'=>1), 'optgroup'=>array('option'=>1), 'option'=>array('#pcdata'=>1), 'rbc'=>array('rb'=>1), 'rp'=>array('#pcdata'=>1), 'rtc'=>array('rt'=>1), 'ruby'=>array('rb'=>1, 'rbc'=>1, 'rp'=>1, 'rt'=>1, 'rtc'=>1), 'select'=>array('optgroup'=>1, 'option'=>1), 'script'=>array('#pcdata'=>1), 'table'=>array('caption'=>1, 'col'=>1, 'colgroup'=>1, 'tfoot'=>1, 'tbody'=>1, 'tr'=>1, 'thead'=>1), 'tbody'=>array('tr'=>1), 'tfoot'=>array('tr'=>1), 'textarea'=>array('#pcdata'=>1), 'thead'=>array('tr'=>1), 'tr'=>array('td'=>1, 'th'=>1), 'ul'=>array('li'=>1)); // Specific - immediate parent-child +$cO = array('address'=>array('p'=>1), 'applet'=>array('param'=>1), 'blockquote'=>array('script'=>1), 'fieldset'=>array('legend'=>1, '#pcdata'=>1), 'form'=>array('script'=>1), 'map'=>array('area'=>1), 'object'=>array('param'=>1, 'embed'=>1)); // Other +$cT = array('colgroup'=>1, 'dd'=>1, 'dt'=>1, 'li'=>1, 'option'=>1, 'p'=>1, 'td'=>1, 'tfoot'=>1, 'th'=>1, 'thead'=>1, 'tr'=>1); // Omitable closing +// block/inline type; ins & del both type; #pcdata: text +$eB = array('address'=>1, 'blockquote'=>1, 'center'=>1, 'del'=>1, 'dir'=>1, 'dl'=>1, 'div'=>1, 'fieldset'=>1, 'form'=>1, 'ins'=>1, 'h1'=>1, 'h2'=>1, 'h3'=>1, 'h4'=>1, 'h5'=>1, 'h6'=>1, 'hr'=>1, 'isindex'=>1, 'menu'=>1, 'noscript'=>1, 'ol'=>1, 'p'=>1, 'pre'=>1, 'table'=>1, 'ul'=>1); +$eI = array('#pcdata'=>1, 'a'=>1, 'abbr'=>1, 'acronym'=>1, 'applet'=>1, 'b'=>1, 'bdo'=>1, 'big'=>1, 'br'=>1, 'button'=>1, 'cite'=>1, 'code'=>1, 'del'=>1, 'dfn'=>1, 'em'=>1, 'embed'=>1, 'font'=>1, 'i'=>1, 'iframe'=>1, 'img'=>1, 'input'=>1, 'ins'=>1, 'kbd'=>1, 'label'=>1, 'map'=>1, 'object'=>1, 'param'=>1, 'q'=>1, 'ruby'=>1, 's'=>1, 'samp'=>1, 'select'=>1, 'script'=>1, 'small'=>1, 'span'=>1, 'strike'=>1, 'strong'=>1, 'sub'=>1, 'sup'=>1, 'textarea'=>1, 'tt'=>1, 'u'=>1, 'var'=>1); +$eN = array('a'=>1, 'big'=>1, 'button'=>1, 'fieldset'=>1, 'font'=>1, 'form'=>1, 'iframe'=>1, 'img'=>1, 'input'=>1, 'label'=>1, 'object'=>1, 'ruby'=>1, 'script'=>1, 'select'=>1, 'small'=>1, 'sub'=>1, 'sup'=>1, 'textarea'=>1); // Exclude from specific ele; $cN values +$eO = array('area'=>1, 'caption'=>1, 'col'=>1, 'colgroup'=>1, 'dd'=>1, 'dt'=>1, 'legend'=>1, 'li'=>1, 'optgroup'=>1, 'option'=>1, 'rb'=>1, 'rbc'=>1, 'rp'=>1, 'rt'=>1, 'rtc'=>1, 'script'=>1, 'tbody'=>1, 'td'=>1, 'tfoot'=>1, 'thead'=>1, 'th'=>1, 'tr'=>1); // Missing in $eB & $eI +$eF = $eB + $eI; + +// $in sets allowed child +$in = ((isset($eF[$in]) && $in != '#pcdata') or isset($eO[$in])) ? $in : 'div'; +if(isset($cE[$in])){ + return (!$do ? '' : str_replace(array('<', '>'), array('<', '>'), $t)); +} +if(isset($cS[$in])){$inOk = $cS[$in];} +elseif(isset($cI[$in])){$inOk = $eI; $cI['del'] = 1; $cI['ins'] = 1;} +elseif(isset($cF[$in])){$inOk = $eF; unset($cI['del'], $cI['ins']);} +elseif(isset($cB[$in])){$inOk = $eB; unset($cI['del'], $cI['ins']);} +if(isset($cO[$in])){$inOk = $inOk + $cO[$in];} +if(isset($cN[$in])){$inOk = array_diff_assoc($inOk, $cN[$in]);} + +$t = explode('<', $t); +$ok = $q = array(); // $q seq list of open non-empty ele +ob_start(); + +for($i=-1, $ci=count($t); ++$i<$ci;){ + // allowed $ok in parent $p + if($ql = count($q)){ + $p = array_pop($q); + $q[] = $p; + if(isset($cS[$p])){$ok = $cS[$p];} + elseif(isset($cI[$p])){$ok = $eI; $cI['del'] = 1; $cI['ins'] = 1;} + elseif(isset($cF[$p])){$ok = $eF; unset($cI['del'], $cI['ins']);} + elseif(isset($cB[$p])){$ok = $eB; unset($cI['del'], $cI['ins']);} + if(isset($cO[$p])){$ok = $ok + $cO[$p];} + if(isset($cN[$p])){$ok = array_diff_assoc($ok, $cN[$p]);} + }else{$ok = $inOk; unset($cI['del'], $cI['ins']);} + // bad tags, & ele content + if(isset($e) && ($do == 1 or (isset($ok['#pcdata']) && ($do == 3 or $do == 5)))){ + echo '<', $s, $e, $a, '>'; + } + if(isset($x[0])){ + if($do < 3 or isset($ok['#pcdata'])){echo $x;} + elseif(strpos($x, "\x02\x04")){ + foreach(preg_split('`(\x01\x02[^\x01\x02]+\x02\x01)`', $x, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY) as $v){ + echo (substr($v, 0, 2) == "\x01\x02" ? $v : ($do > 4 ? preg_replace('`\S`', '', $v) : '')); + } + }elseif($do > 4){echo preg_replace('`\S`', '', $x);} + } + // get markup + if(!preg_match('`^(/?)([a-zA-Z1-6]+)([^>]*)>(.*)`sm', $t[$i], $r)){$x = $t[$i]; continue;} + $s = null; $e = null; $a = null; $x = null; list($all, $s, $e, $a, $x) = $r; + // close tag + if($s){ + if(isset($cE[$e]) or !in_array($e, $q)){continue;} // Empty/unopen + if($p == $e){array_pop($q); echo '</', $e, '>'; unset($e); continue;} // Last open + $add = ''; // Nesting - close open tags that need to be + for($j=-1, $cj=count($q); ++$j<$cj;){ + if(($d = array_pop($q)) == $e){break;} + else{$add .= "</{$d}>";} + } + echo $add, '</', $e, '>'; unset($e); continue; + } + // open tag + // $cB ele needs $eB ele as child + if(isset($cB[$e]) && strlen(trim($x))){ + $t[$i] = "{$e}{$a}>"; + array_splice($t, $i+1, 0, 'div>'. $x); unset($e, $x); ++$ci; --$i; continue; + } + if((($ql && isset($cB[$p])) or (isset($cB[$in]) && !$ql)) && !isset($eB[$e]) && !isset($ok[$e])){ + array_splice($t, $i, 0, 'div>'); unset($e, $x); ++$ci; --$i; continue; + } + // if no open ele, $in = parent; mostly immediate parent-child relation should hold + if(!$ql or !isset($eN[$e]) or !array_intersect($q, $cN2)){ + if(!isset($ok[$e])){ + if($ql && isset($cT[$p])){echo '</', array_pop($q), '>'; unset($e, $x); --$i;} + continue; + } + if(!isset($cE[$e])){$q[] = $e;} + echo '<', $e, $a, '>'; unset($e); continue; + } + // specific parent-child + if(isset($cS[$p][$e])){ + if(!isset($cE[$e])){$q[] = $e;} + echo '<', $e, $a, '>'; unset($e); continue; + } + // nesting + $add = ''; + $q2 = array(); + for($k=-1, $kc=count($q); ++$k<$kc;){ + $d = $q[$k]; + $ok2 = array(); + if(isset($cS[$d])){$q2[] = $d; continue;} + $ok2 = isset($cI[$d]) ? $eI : $eF; + if(isset($cO[$d])){$ok2 = $ok2 + $cO[$d];} + if(isset($cN[$d])){$ok2 = array_diff_assoc($ok2, $cN[$d]);} + if(!isset($ok2[$e])){ + if(!$k && !isset($inOk[$e])){continue 2;} + $add = "</{$d}>"; + for(;++$k<$kc;){$add = "</{$q[$k]}>{$add}";} + break; + } + else{$q2[] = $d;} + } + $q = $q2; + if(!isset($cE[$e])){$q[] = $e;} + echo $add, '<', $e, $a, '>'; unset($e); continue; +} + +// end +if($ql = count($q)){ + $p = array_pop($q); + $q[] = $p; + if(isset($cS[$p])){$ok = $cS[$p];} + elseif(isset($cI[$p])){$ok = $eI; $cI['del'] = 1; $cI['ins'] = 1;} + elseif(isset($cF[$p])){$ok = $eF; unset($cI['del'], $cI['ins']);} + elseif(isset($cB[$p])){$ok = $eB; unset($cI['del'], $cI['ins']);} + if(isset($cO[$p])){$ok = $ok + $cO[$p];} + if(isset($cN[$p])){$ok = array_diff_assoc($ok, $cN[$p]);} +}else{$ok = $inOk; unset($cI['del'], $cI['ins']);} +if(isset($e) && ($do == 1 or (isset($ok['#pcdata']) && ($do == 3 or $do == 5)))){ + echo '<', $s, $e, $a, '>'; +} +if(isset($x[0])){ + if(strlen(trim($x)) && (($ql && isset($cB[$p])) or (isset($cB[$in]) && !$ql))){ + echo '<div>', $x, '</div>'; + } + elseif($do < 3 or isset($ok['#pcdata'])){echo $x;} + elseif(strpos($x, "\x02\x04")){ + foreach(preg_split('`(\x01\x02[^\x01\x02]+\x02\x01)`', $x, -1, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY) as $v){ + echo (substr($v, 0, 2) == "\x01\x02" ? $v : ($do > 4 ? preg_replace('`\S`', '', $v) : '')); + } + }elseif($do > 4){echo preg_replace('`\S`', '', $x);} +} +while(!empty($q) && ($e = array_pop($q))){echo '</', $e, '>';} +$o = ob_get_contents(); +ob_end_clean(); +return $o; +// eof +} + +function hl_cmtcd($t){ +// comment/CDATA sec handler +$t = $t[0]; +global $C; +if($t[3] == '-'){ + if(!$C['comment']){return $t;} + if($C['comment'] == 1){return '';} + if(substr(($t = preg_replace('`--+`', '-', substr($t, 4, -3))), -1) != ' '){$t .= ' ';} + $t = $C['comment'] == 2 ? str_replace(array('&', '<', '>'), array('&', '<', '>'), $t) : $t; + $t = "\x01\x02\x04!--$t--\x05\x02\x01"; +}else{ // CDATA + if(!$C['cdata']){return $t;} + if($C['cdata'] == 1){return '';} + $t = substr($t, 1, -1); + $t = $C['cdata'] == 2 ? str_replace(array('&', '<', '>'), array('&', '<', '>'), $t) : $t; + $t = "\x01\x01\x04$t\x05\x01\x01"; +} +return str_replace(array('&', '<', '>'), array("\x03", "\x04", "\x05"), $t); +// eof +} + +function hl_ent($t){ +// entitity handler +global $C; +$t = $t[1]; +static $U = array('quot'=>1,'amp'=>1,'lt'=>1,'gt'=>1); +static $N = array('fnof'=>'402', 'Alpha'=>'913', 'Beta'=>'914', 'Gamma'=>'915', 'Delta'=>'916', 'Epsilon'=>'917', 'Zeta'=>'918', 'Eta'=>'919', 'Theta'=>'920', 'Iota'=>'921', 'Kappa'=>'922', 'Lambda'=>'923', 'Mu'=>'924', 'Nu'=>'925', 'Xi'=>'926', 'Omicron'=>'927', 'Pi'=>'928', 'Rho'=>'929', 'Sigma'=>'931', 'Tau'=>'932', 'Upsilon'=>'933', 'Phi'=>'934', 'Chi'=>'935', 'Psi'=>'936', 'Omega'=>'937', 'alpha'=>'945', 'beta'=>'946', 'gamma'=>'947', 'delta'=>'948', 'epsilon'=>'949', 'zeta'=>'950', 'eta'=>'951', 'theta'=>'952', 'iota'=>'953', 'kappa'=>'954', 'lambda'=>'955', 'mu'=>'956', 'nu'=>'957', 'xi'=>'958', 'omicron'=>'959', 'pi'=>'960', 'rho'=>'961', 'sigmaf'=>'962', 'sigma'=>'963', 'tau'=>'964', 'upsilon'=>'965', 'phi'=>'966', 'chi'=>'967', 'psi'=>'968', 'omega'=>'969', 'thetasym'=>'977', 'upsih'=>'978', 'piv'=>'982', 'bull'=>'8226', 'hellip'=>'8230', 'prime'=>'8242', 'Prime'=>'8243', 'oline'=>'8254', 'frasl'=>'8260', 'weierp'=>'8472', 'image'=>'8465', 'real'=>'8476', 'trade'=>'8482', 'alefsym'=>'8501', 'larr'=>'8592', 'uarr'=>'8593', 'rarr'=>'8594', 'darr'=>'8595', 'harr'=>'8596', 'crarr'=>'8629', 'lArr'=>'8656', 'uArr'=>'8657', 'rArr'=>'8658', 'dArr'=>'8659', 'hArr'=>'8660', 'forall'=>'8704', 'part'=>'8706', 'exist'=>'8707', 'empty'=>'8709', 'nabla'=>'8711', 'isin'=>'8712', 'notin'=>'8713', 'ni'=>'8715', 'prod'=>'8719', 'sum'=>'8721', 'minus'=>'8722', 'lowast'=>'8727', 'radic'=>'8730', 'prop'=>'8733', 'infin'=>'8734', 'ang'=>'8736', 'and'=>'8743', 'or'=>'8744', 'cap'=>'8745', 'cup'=>'8746', 'int'=>'8747', 'there4'=>'8756', 'sim'=>'8764', 'cong'=>'8773', 'asymp'=>'8776', 'ne'=>'8800', 'equiv'=>'8801', 'le'=>'8804', 'ge'=>'8805', 'sub'=>'8834', 'sup'=>'8835', 'nsub'=>'8836', 'sube'=>'8838', 'supe'=>'8839', 'oplus'=>'8853', 'otimes'=>'8855', 'perp'=>'8869', 'sdot'=>'8901', 'lceil'=>'8968', 'rceil'=>'8969', 'lfloor'=>'8970', 'rfloor'=>'8971', 'lang'=>'9001', 'rang'=>'9002', 'loz'=>'9674', 'spades'=>'9824', 'clubs'=>'9827', 'hearts'=>'9829', 'diams'=>'9830', 'apos'=>'39', 'OElig'=>'338', 'oelig'=>'339', 'Scaron'=>'352', 'scaron'=>'353', 'Yuml'=>'376', 'circ'=>'710', 'tilde'=>'732', 'ensp'=>'8194', 'emsp'=>'8195', 'thinsp'=>'8201', 'zwnj'=>'8204', 'zwj'=>'8205', 'lrm'=>'8206', 'rlm'=>'8207', 'ndash'=>'8211', 'mdash'=>'8212', 'lsquo'=>'8216', 'rsquo'=>'8217', 'sbquo'=>'8218', 'ldquo'=>'8220', 'rdquo'=>'8221', 'bdquo'=>'8222', 'dagger'=>'8224', 'Dagger'=>'8225', 'permil'=>'8240', 'lsaquo'=>'8249', 'rsaquo'=>'8250', 'euro'=>'8364', 'nbsp'=>'160', 'iexcl'=>'161', 'cent'=>'162', 'pound'=>'163', 'curren'=>'164', 'yen'=>'165', 'brvbar'=>'166', 'sect'=>'167', 'uml'=>'168', 'copy'=>'169', 'ordf'=>'170', 'laquo'=>'171', 'not'=>'172', 'shy'=>'173', 'reg'=>'174', 'macr'=>'175', 'deg'=>'176', 'plusmn'=>'177', 'sup2'=>'178', 'sup3'=>'179', 'acute'=>'180', 'micro'=>'181', 'para'=>'182', 'middot'=>'183', 'cedil'=>'184', 'sup1'=>'185', 'ordm'=>'186', 'raquo'=>'187', 'frac14'=>'188', 'frac12'=>'189', 'frac34'=>'190', 'iquest'=>'191', 'Agrave'=>'192', 'Aacute'=>'193', 'Acirc'=>'194', 'Atilde'=>'195', 'Auml'=>'196', 'Aring'=>'197', 'AElig'=>'198', 'Ccedil'=>'199', 'Egrave'=>'200', 'Eacute'=>'201', 'Ecirc'=>'202', 'Euml'=>'203', 'Igrave'=>'204', 'Iacute'=>'205', 'Icirc'=>'206', 'Iuml'=>'207', 'ETH'=>'208', 'Ntilde'=>'209', 'Ograve'=>'210', 'Oacute'=>'211', 'Ocirc'=>'212', 'Otilde'=>'213', 'Ouml'=>'214', 'times'=>'215', 'Oslash'=>'216', 'Ugrave'=>'217', 'Uacute'=>'218', 'Ucirc'=>'219', 'Uuml'=>'220', 'Yacute'=>'221', 'THORN'=>'222', 'szlig'=>'223', 'agrave'=>'224', 'aacute'=>'225', 'acirc'=>'226', 'atilde'=>'227', 'auml'=>'228', 'aring'=>'229', 'aelig'=>'230', 'ccedil'=>'231', 'egrave'=>'232', 'eacute'=>'233', 'ecirc'=>'234', 'euml'=>'235', 'igrave'=>'236', 'iacute'=>'237', 'icirc'=>'238', 'iuml'=>'239', 'eth'=>'240', 'ntilde'=>'241', 'ograve'=>'242', 'oacute'=>'243', 'ocirc'=>'244', 'otilde'=>'245', 'ouml'=>'246', 'divide'=>'247', 'oslash'=>'248', 'ugrave'=>'249', 'uacute'=>'250', 'ucirc'=>'251', 'uuml'=>'252', 'yacute'=>'253', 'thorn'=>'254', 'yuml'=>'255'); +if($t[0] != '#'){ + return ($C['and_mark'] ? "\x06" : '&'). (isset($U[$t]) ? $t : (isset($N[$t]) ? (!$C['named_entity'] ? '#'. ($C['hexdec_entity'] > 1 ? 'x'. dechex($N[$t]) : $N[$t]) : $t) : 'amp;'. $t)). ';'; +} +if(($n = ctype_digit($t = substr($t, 1)) ? intval($t) : hexdec(substr($t, 1))) < 9 or ($n > 13 && $n < 32) or $n == 11 or $n == 12 or ($n > 126 && $n < 160 && $n != 133) or ($n > 55295 && ($n < 57344 or ($n > 64975 && $n < 64992) or $n == 65534 or $n == 65535 or $n > 1114111))){ + return ($C['and_mark'] ? "\x06" : '&'). "amp;#{$t};"; +} +return ($C['and_mark'] ? "\x06" : '&'). '#'. (((ctype_digit($t) && $C['hexdec_entity'] < 2) or !$C['hexdec_entity']) ? $n : 'x'. dechex($n)). ';'; +// eof +} + +function hl_prot($p, $c=null){ +// check URL scheme +global $C; +$b = $a = ''; +if($c == null){$c = 'style'; $b = $p[1]; $a = $p[3]; $p = trim($p[2]);} +$c = isset($C['schemes'][$c]) ? $C['schemes'][$c] : $C['schemes']['*']; +if(isset($c['*']) or !strcspn($p, '#?;')){return "{$b}{$p}{$a}";} // All ok, frag, query, param +if(preg_match('`^([a-z\d\-+.&#; ]+?)(:|&#(58|x3a);|%3a|\\\\0{0,4}3a).`i', $p, $m) && !isset($c[strtolower($m[1])])){ // Denied prot + return "{$b}denied:{$p}{$a}"; +} +if($C['abs_url']){ + if($C['abs_url'] == -1 && strpos($p, $C['base_url']) === 0){ // Make url rel + $p = substr($p, strlen($C['base_url'])); + }elseif(empty($m[1])){ // Make URL abs + if(substr($p, 0, 2) == '//'){$p = substr($C['base_url'], 0, strpos($C['base_url'], ':')+1). $p;} + elseif($p[0] == '/'){$p = preg_replace('`(^.+?://[^/]+)(.*)`', '$1', $C['base_url']). $p;} + elseif(strcspn($p, './')){$p = $C['base_url']. $p;} + else{ + preg_match('`^([a-zA-Z\d\-+.]+://[^/]+)(.*)`', $C['base_url'], $m); + $p = preg_replace('`(?<=/)\./`', '', $m[2]. $p); + while(preg_match('`(?<=/)([^/]{3,}|[^/.]+?|\.[^/.]|[^/.]\.)/\.\./`', $p)){ + $p = preg_replace('`(?<=/)([^/]{3,}|[^/.]+?|\.[^/.]|[^/.]\.)/\.\./`', '', $p); + } + $p = $m[1]. $p; + } + } +} +return "{$b}{$p}{$a}"; +// eof +} + +function hl_regex($p){ +// ?regex +if(empty($p)){return 0;} +if($t = ini_get('track_errors')){$o = isset($php_errormsg) ? $php_errormsg : null;} +else{ini_set('track_errors', 1);} +unset($php_errormsg); +if(($d = ini_get('display_errors'))){ini_set('display_errors', 0);} +preg_match($p, ''); +if($d){ini_set('display_errors', 1);} +$r = isset($php_errormsg) ? 0 : 1; +if($t){$php_errormsg = isset($o) ? $o : null;} +else{ini_set('track_errors', 0);} +return $r; +// eof +} + +function hl_spec($t){ +// final $spec +$s = array(); +$t = str_replace(array("\t", "\r", "\n", ' '), '', preg_replace('/"(?>(`.|[^"])*)"/sme', 'substr(str_replace(array(";", "|", "~", " ", ",", "/", "(", ")", \'`"\'), array("\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07", "\x08", "\""), "$0"), 1, -1)', trim($t))); +for($i = count(($t = explode(';', $t))); --$i>=0;){ + $w = $t[$i]; + if(empty($w) or ($e = strpos($w, '=')) === false or !strlen(($a = substr($w, $e+1)))){continue;} + $y = $n = array(); + foreach(explode(',', $a) as $v){ + if(!preg_match('`^([a-z:\-\*]+)(?:\((.*?)\))?`i', $v, $m)){continue;} + if(($x = strtolower($m[1])) == '-*'){$n['*'] = 1; continue;} + if($x[0] == '-'){$n[substr($x, 1)] = 1; continue;} + if(!isset($m[2])){$y[$x] = 1; continue;} + foreach(explode('/', $m[2]) as $m){ + if(empty($m) or ($p = strpos($m, '=')) == 0 or $p < 5){$y[$x] = 1; continue;} + $y[$x][strtolower(substr($m, 0, $p))] = str_replace(array("\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07", "\x08"), array(";", "|", "~", " ", ",", "/", "(", ")"), substr($m, $p+1)); + } + if(isset($y[$x]['match']) && !hl_regex($y[$x]['match'])){unset($y[$x]['match']);} + if(isset($y[$x]['nomatch']) && !hl_regex($y[$x]['nomatch'])){unset($y[$x]['nomatch']);} + } + if(!count($y) && !count($n)){continue;} + foreach(explode(',', substr($w, 0, $e)) as $v){ + if(!strlen(($v = strtolower($v)))){continue;} + if(count($y)){$s[$v] = $y;} + if(count($n)){$s[$v]['n'] = $n;} + } +} +return $s; +// eof +} + +function hl_tag($t){ +// tag/attribute handler +global $C; +$t = $t[0]; +// invalid < > +if($t == '< '){return '< ';} +if($t == '>'){return '>';} +if(!preg_match('`^<(/?)([a-zA-Z][a-zA-Z1-6]*)([^>]*?)\s?>$`m', $t, $m)){ + return str_replace(array('<', '>'), array('<', '>'), $t); +}elseif(!isset($C['elements'][($e = strtolower($m[2]))])){ + return (($C['keep_bad']%2) ? str_replace(array('<', '>'), array('<', '>'), $t) : ''); +} +// attr string +$a = str_replace(array("\xad", "\n", "\r", "\t"), ' ', trim($m[3])); +if(strpos($a, '&') !== false){ + str_replace(array('­', '­', '­'), ' ', $a); +} +// tag transform +static $eD = array('applet'=>1, 'center'=>1, 'dir'=>1, 'embed'=>1, 'font'=>1, 'isindex'=>1, 'menu'=>1, 's'=>1, 'strike'=>1, 'u'=>1); // Deprecated +if($C['make_tag_strict'] && isset($eD[$e])){ + $trt = hl_tag2($e, $a, $C['make_tag_strict']); + if(!$e){return (($C['keep_bad']%2) ? str_replace(array('<', '>'), array('<', '>'), $t) : '');} +} +// close tag +static $eE = array('area'=>1, 'br'=>1, 'col'=>1, 'embed'=>1, 'hr'=>1, 'img'=>1, 'input'=>1, 'isindex'=>1, 'param'=>1); // Empty ele +if(!empty($m[1])){ + return (!isset($eE[$e]) ? "</$e>" : (($C['keep_bad'])%2 ? str_replace(array('<', '>'), array('<', '>'), $t) : '')); +} + +// open tag & attr +static $aN = array('abbr'=>array('td'=>1, 'th'=>1), 'accept-charset'=>array('form'=>1), 'accept'=>array('form'=>1, 'input'=>1), 'accesskey'=>array('a'=>1, 'area'=>1, 'button'=>1, 'input'=>1, 'label'=>1, 'legend'=>1, 'textarea'=>1), 'action'=>array('form'=>1), 'align'=>array('caption'=>1, 'embed'=>1, 'applet'=>1, 'iframe'=>1, 'img'=>1, 'input'=>1, 'object'=>1, 'legend'=>1, 'table'=>1, 'hr'=>1, 'div'=>1, 'h1'=>1, 'h2'=>1, 'h3'=>1, 'h4'=>1, 'h5'=>1, 'h6'=>1, 'p'=>1, 'col'=>1, 'colgroup'=>1, 'tbody'=>1, 'td'=>1, 'tfoot'=>1, 'th'=>1, 'thead'=>1, 'tr'=>1), 'alt'=>array('applet'=>1, 'area'=>1, 'img'=>1, 'input'=>1), 'archive'=>array('applet'=>1, 'object'=>1), 'axis'=>array('td'=>1, 'th'=>1), 'bgcolor'=>array('embed'=>1, 'table'=>1, 'tr'=>1, 'td'=>1, 'th'=>1), 'border'=>array('table'=>1, 'img'=>1, 'object'=>1), 'bordercolor'=>array('table'=>1, 'td'=>1, 'tr'=>1), 'cellpadding'=>array('table'=>1), 'cellspacing'=>array('table'=>1), 'char'=>array('col'=>1, 'colgroup'=>1, 'tbody'=>1, 'td'=>1, 'tfoot'=>1, 'th'=>1, 'thead'=>1, 'tr'=>1), 'charoff'=>array('col'=>1, 'colgroup'=>1, 'tbody'=>1, 'td'=>1, 'tfoot'=>1, 'th'=>1, 'thead'=>1, 'tr'=>1), 'charset'=>array('a'=>1, 'script'=>1), 'checked'=>array('input'=>1), 'cite'=>array('blockquote'=>1, 'q'=>1, 'del'=>1, 'ins'=>1), 'classid'=>array('object'=>1), 'clear'=>array('br'=>1), 'code'=>array('applet'=>1), 'codebase'=>array('object'=>1, 'applet'=>1), 'codetype'=>array('object'=>1), 'color'=>array('font'=>1), 'cols'=>array('textarea'=>1), 'colspan'=>array('td'=>1, 'th'=>1), 'compact'=>array('dir'=>1, 'dl'=>1, 'menu'=>1, 'ol'=>1, 'ul'=>1), 'coords'=>array('area'=>1, 'a'=>1), 'data'=>array('object'=>1), 'datetime'=>array('del'=>1, 'ins'=>1), 'declare'=>array('object'=>1), 'defer'=>array('script'=>1), 'dir'=>array('bdo'=>1), 'disabled'=>array('button'=>1, 'input'=>1, 'optgroup'=>1, 'option'=>1, 'select'=>1, 'textarea'=>1), 'enctype'=>array('form'=>1), 'face'=>array('font'=>1), 'for'=>array('label'=>1), 'frame'=>array('table'=>1), 'frameborder'=>array('iframe'=>1), 'headers'=>array('td'=>1, 'th'=>1), 'height'=>array('embed'=>1, 'iframe'=>1, 'td'=>1, 'th'=>1, 'img'=>1, 'object'=>1, 'applet'=>1), 'href'=>array('a'=>1, 'area'=>1), 'hreflang'=>array('a'=>1), 'hspace'=>array('applet'=>1, 'img'=>1, 'object'=>1), 'ismap'=>array('img'=>1, 'input'=>1), 'label'=>array('option'=>1, 'optgroup'=>1), 'language'=>array('script'=>1), 'longdesc'=>array('img'=>1, 'iframe'=>1), 'marginheight'=>array('iframe'=>1), 'marginwidth'=>array('iframe'=>1), 'maxlength'=>array('input'=>1), 'method'=>array('form'=>1), 'model'=>array('embed'=>1), 'multiple'=>array('select'=>1), 'name'=>array('button'=>1, 'embed'=>1, 'textarea'=>1, 'applet'=>1, 'select'=>1, 'form'=>1, 'iframe'=>1, 'img'=>1, 'a'=>1, 'input'=>1, 'object'=>1, 'map'=>1, 'param'=>1), 'nohref'=>array('area'=>1), 'noshade'=>array('hr'=>1), 'nowrap'=>array('td'=>1, 'th'=>1), 'object'=>array('applet'=>1), 'onblur'=>array('a'=>1, 'area'=>1, 'button'=>1, 'input'=>1, 'label'=>1, 'select'=>1, 'textarea'=>1), 'onchange'=>array('input'=>1, 'select'=>1, 'textarea'=>1), 'onfocus'=>array('a'=>1, 'area'=>1, 'button'=>1, 'input'=>1, 'label'=>1, 'select'=>1, 'textarea'=>1), 'onreset'=>array('form'=>1), 'onselect'=>array('input'=>1, 'textarea'=>1), 'onsubmit'=>array('form'=>1), 'pluginspage'=>array('embed'=>1), 'pluginurl'=>array('embed'=>1), 'prompt'=>array('isindex'=>1), 'readonly'=>array('textarea'=>1, 'input'=>1), 'rel'=>array('a'=>1), 'rev'=>array('a'=>1), 'rows'=>array('textarea'=>1), 'rowspan'=>array('td'=>1, 'th'=>1), 'rules'=>array('table'=>1), 'scope'=>array('td'=>1, 'th'=>1), 'scrolling'=>array('iframe'=>1), 'selected'=>array('option'=>1), 'shape'=>array('area'=>1, 'a'=>1), 'size'=>array('hr'=>1, 'font'=>1, 'input'=>1, 'select'=>1), 'span'=>array('col'=>1, 'colgroup'=>1), 'src'=>array('embed'=>1, 'script'=>1, 'input'=>1, 'iframe'=>1, 'img'=>1), 'standby'=>array('object'=>1), 'start'=>array('ol'=>1), 'summary'=>array('table'=>1), 'tabindex'=>array('a'=>1, 'area'=>1, 'button'=>1, 'input'=>1, 'object'=>1, 'select'=>1, 'textarea'=>1), 'target'=>array('a'=>1, 'area'=>1, 'form'=>1), 'type'=>array('a'=>1, 'embed'=>1, 'object'=>1, 'param'=>1, 'script'=>1, 'input'=>1, 'li'=>1, 'ol'=>1, 'ul'=>1, 'button'=>1), 'usemap'=>array('img'=>1, 'input'=>1, 'object'=>1), 'valign'=>array('col'=>1, 'colgroup'=>1, 'tbody'=>1, 'td'=>1, 'tfoot'=>1, 'th'=>1, 'thead'=>1, 'tr'=>1), 'value'=>array('input'=>1, 'option'=>1, 'param'=>1, 'button'=>1, 'li'=>1), 'valuetype'=>array('param'=>1), 'vspace'=>array('applet'=>1, 'img'=>1, 'object'=>1), 'width'=>array('embed'=>1, 'hr'=>1, 'iframe'=>1, 'img'=>1, 'object'=>1, 'table'=>1, 'td'=>1, 'th'=>1, 'applet'=>1, 'col'=>1, 'colgroup'=>1, 'pre'=>1), 'wmode'=>array('embed'=>1), 'xml:space'=>array('pre'=>1, 'script'=>1, 'style'=>1)); // Ele-specific +static $aNE = array('checked'=>1, 'compact'=>1, 'declare'=>1, 'defer'=>1, 'disabled'=>1, 'ismap'=>1, 'multiple'=>1, 'nohref'=>1, 'noresize'=>1, 'noshade'=>1, 'nowrap'=>1, 'readonly'=>1, 'selected'=>1); // Empty +static $aNP = array('action'=>1, 'cite'=>1, 'classid'=>1, 'codebase'=>1, 'data'=>1, 'href'=>1, 'longdesc'=>1, 'model'=>1, 'pluginspage'=>1, 'pluginurl'=>1, 'usemap'=>1); // Need scheme check; excludes style, on* & src +static $aNU = array('class'=>array('param'=>1, 'script'=>1), 'dir'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'iframe'=>1, 'param'=>1, 'script'=>1), 'id'=>array('script'=>1), 'lang'=>array('applet'=>1, 'br'=>1, 'iframe'=>1, 'param'=>1, 'script'=>1), 'xml:lang'=>array('applet'=>1, 'br'=>1, 'iframe'=>1, 'param'=>1, 'script'=>1), 'onclick'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'ondblclick'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onkeydown'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onkeypress'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onkeyup'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onmousedown'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onmousemove'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onmouseout'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onmouseover'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'onmouseup'=>array('applet'=>1, 'bdo'=>1, 'br'=>1, 'font'=>1, 'iframe'=>1, 'isindex'=>1, 'param'=>1, 'script'=>1), 'style'=>array('param'=>1, 'script'=>1), 'title'=>array('param'=>1, 'script'=>1)); // Univ & exceptions + +if($C['lc_std_val']){ + // predef attr vals for $eAL & $aNE ele + static $aNL = array('all'=>1, 'baseline'=>1, 'bottom'=>1, 'button'=>1, 'center'=>1, 'char'=>1, 'checkbox'=>1, 'circle'=>1, 'col'=>1, 'colgroup'=>1, 'cols'=>1, 'data'=>1, 'default'=>1, 'file'=>1, 'get'=>1, 'groups'=>1, 'hidden'=>1, 'image'=>1, 'justify'=>1, 'left'=>1, 'ltr'=>1, 'middle'=>1, 'none'=>1, 'object'=>1, 'password'=>1, 'poly'=>1, 'post'=>1, 'preserve'=>1, 'radio'=>1, 'rect'=>1, 'ref'=>1, 'reset'=>1, 'right'=>1, 'row'=>1, 'rowgroup'=>1, 'rows'=>1, 'rtl'=>1, 'submit'=>1, 'text'=>1, 'top'=>1); + static $eAL = array('a'=>1, 'area'=>1, 'bdo'=>1, 'button'=>1, 'col'=>1, 'form'=>1, 'img'=>1, 'input'=>1, 'object'=>1, 'optgroup'=>1, 'option'=>1, 'param'=>1, 'script'=>1, 'select'=>1, 'table'=>1, 'td'=>1, 'tfoot'=>1, 'th'=>1, 'thead'=>1, 'tr'=>1, 'xml:space'=>1); + $lcase = isset($eAL[$e]) ? 1 : 0; +} + +$depTr = 0; +if($C['no_deprecated_attr']){ + // dep attr:applicable ele + static $aND = array('align'=>array('caption'=>1, 'div'=>1, 'h1'=>1, 'h2'=>1, 'h3'=>1, 'h4'=>1, 'h5'=>1, 'h6'=>1, 'hr'=>1, 'img'=>1, 'input'=>1, 'legend'=>1, 'object'=>1, 'p'=>1, 'table'=>1), 'bgcolor'=>array('table'=>1, 'td'=>1, 'th'=>1, 'tr'=>1), 'border'=>array('img'=>1, 'object'=>1), 'bordercolor'=>array('table'=>1, 'td'=>1, 'tr'=>1), 'clear'=>array('br'=>1), 'compact'=>array('dl'=>1, 'ol'=>1, 'ul'=>1), 'height'=>array('td'=>1, 'th'=>1), 'hspace'=>array('img'=>1, 'object'=>1), 'language'=>array('script'=>1), 'name'=>array('a'=>1, 'form'=>1, 'iframe'=>1, 'img'=>1, 'map'=>1), 'noshade'=>array('hr'=>1), 'nowrap'=>array('td'=>1, 'th'=>1), 'size'=>array('hr'=>1), 'start'=>array('ol'=>1), 'type'=>array('li'=>1, 'ol'=>1, 'ul'=>1), 'value'=>array('li'=>1), 'vspace'=>array('img'=>1, 'object'=>1), 'width'=>array('hr'=>1, 'pre'=>1, 'td'=>1, 'th'=>1)); + static $eAD = array('a'=>1, 'br'=>1, 'caption'=>1, 'div'=>1, 'dl'=>1, 'form'=>1, 'h1'=>1, 'h2'=>1, 'h3'=>1, 'h4'=>1, 'h5'=>1, 'h6'=>1, 'hr'=>1, 'iframe'=>1, 'img'=>1, 'input'=>1, 'legend'=>1, 'li'=>1, 'map'=>1, 'object'=>1, 'ol'=>1, 'p'=>1, 'pre'=>1, 'script'=>1, 'table'=>1, 'td'=>1, 'th'=>1, 'tr'=>1, 'ul'=>1); + $depTr = isset($eAD[$e]) ? 1 : 0; +} + +// attr name-vals +if(strpos($a, "\x01") !== false){$a = preg_replace('`\x01[^\x01]*\x01`', '', $a);} // No comment/CDATA sec +$mode = 0; $a = trim($a, ' /'); $aA = array(); +while(strlen($a)){ + $w = 0; + switch($mode){ + case 0: // Name + if(preg_match('`^[a-zA-Z][\-a-zA-Z:]+`', $a, $m)){ + $nm = strtolower($m[0]); + $w = $mode = 1; $a = ltrim(substr_replace($a, '', 0, strlen($m[0]))); + } + break; case 1: + if($a[0] == '='){ // = + $w = 1; $mode = 2; $a = ltrim($a, '= '); + }else{ // No val + $w = 1; $mode = 0; $a = ltrim($a); + $aA[$nm] = ''; + } + break; case 2: // Val + if(preg_match('`^"[^"]*"`', $a, $m) or preg_match("`^'[^']*'`", $a, $m) or preg_match("`^\s*[^\s\"']+`", $a, $m)){ + $m = $m[0]; $w = 1; $mode = 0; $a = ltrim(substr_replace($a, '', 0, strlen($m))); + $aA[$nm] = trim(($m[0] == '"' or $m[0] == '\'') ? substr($m, 1, -1) : $m); + } + break; + } + if($w == 0){ // Parse errs, deal with space, " & ' + $a = preg_replace('`^(?:"[^"]*("|$)|\'[^\']*(\'|$)|\S)*\s*`', '', $a); + $mode = 0; + } +} +if($mode == 1){$aA[$nm] = '';} + +// clean attrs +global $S; +$rl = isset($S[$e]) ? $S[$e] : array(); +$a = array(); $nfr = 0; +foreach($aA as $k=>$v){ + if(((isset($C['deny_attribute']['*']) ? isset($C['deny_attribute'][$k]) : !isset($C['deny_attribute'][$k])) or isset($rl[$k])) && ((!isset($rl['n'][$k]) && !isset($rl['n']['*'])) or isset($rl[$k])) && (isset($aN[$k][$e]) or (isset($aNU[$k]) && !isset($aNU[$k][$e])))){ + if(isset($aNE[$k])){$v = $k;} + elseif(!empty($lcase) && (($e != 'button' or $e != 'input') or $k == 'type')){ // Rather loose but ?not cause issues + $v = (isset($aNL[($v2 = strtolower($v))])) ? $v2 : $v; + } + if($k == 'style' && !$C['style_pass']){ + if(false !== strpos($v, '&#')){ + static $sC = array(' '=>' ', ' '=>' ', 'E'=>'e', 'E'=>'e', 'e'=>'e', 'e'=>'e', 'X'=>'x', 'X'=>'x', 'x'=>'x', 'x'=>'x', 'P'=>'p', 'P'=>'p', 'p'=>'p', 'p'=>'p', 'S'=>'s', 'S'=>'s', 's'=>'s', 's'=>'s', 'I'=>'i', 'I'=>'i', 'i'=>'i', 'i'=>'i', 'O'=>'o', 'O'=>'o', 'o'=>'o', 'o'=>'o', 'N'=>'n', 'N'=>'n', 'n'=>'n', 'n'=>'n', 'U'=>'u', 'U'=>'u', 'u'=>'u', 'u'=>'u', 'R'=>'r', 'R'=>'r', 'r'=>'r', 'r'=>'r', 'L'=>'l', 'L'=>'l', 'l'=>'l', 'l'=>'l', '('=>'(', '('=>'(', ')'=>')', ')'=>')', ' '=>':', ' '=>':', '"'=>'"', '"'=>'"', '''=>"'", '''=>"'", '/'=>'/', '/'=>'/', '*'=>'*', '*'=>'*', '\'=>'\\', '\'=>'\\'); + $v = strtr($v, $sC); + } + $v = preg_replace_callback('`(url(?:\()(?: )*(?:\'|"|&(?:quot|apos);)?)(.+)((?:\'|"|&(?:quot|apos);)?(?: )*(?:\)))`iS', 'hl_prot', $v); + $v = !$C['css_expression'] ? preg_replace('`expression`i', ' ', preg_replace('`\\\\\S|(/|(%2f))(\*|(%2a))`i', ' ', $v)) : $v; + }elseif(isset($aNP[$k]) or strpos($k, 'src') !== false or $k[0] == 'o'){ + $v = hl_prot($v, $k); + if($k == 'href'){ // X-spam + if($C['anti_mail_spam'] && strpos($v, 'mailto:') === 0){ + $v = str_replace('@', htmlspecialchars($C['anti_mail_spam']), $v); + }elseif($C['anti_link_spam']){ + $r1 = $C['anti_link_spam'][1]; + if(!empty($r1) && preg_match($r1, $v)){continue;} + $r0 = $C['anti_link_spam'][0]; + if(!empty($r0) && preg_match($r0, $v)){ + if(isset($a['rel'])){ + if(!preg_match('`\bnofollow\b`i', $a['rel'])){$a['rel'] .= ' nofollow';} + }elseif(isset($aA['rel'])){ + if(!preg_match('`\bnofollow\b`i', $aA['rel'])){$nfr = 1;} + }else{$a['rel'] = 'nofollow';} + } + } + } + } + if(isset($rl[$k]) && is_array($rl[$k]) && ($v = hl_attrval($v, $rl[$k])) === 0){continue;} + $a[$k] = str_replace('"', '"', $v); + } +} +if($nfr){$a['rel'] = isset($a['rel']) ? $a['rel']. ' nofollow' : 'nofollow';} + +// rqd attr +static $eAR = array('area'=>array('alt'=>'area'), 'bdo'=>array('dir'=>'ltr'), 'form'=>array('action'=>''), 'img'=>array('src'=>'', 'alt'=>'image'), 'map'=>array('name'=>''), 'optgroup'=>array('label'=>''), 'param'=>array('name'=>''), 'script'=>array('type'=>'text/javascript'), 'textarea'=>array('rows'=>'10', 'cols'=>'50')); +if(isset($eAR[$e])){ + foreach($eAR[$e] as $k=>$v){ + if(!isset($a[$k])){$a[$k] = isset($v[0]) ? $v : $k;} + } +} + +// depr attrs +if($depTr){ + $c = array(); + foreach($a as $k=>$v){ + if($k == 'style' or !isset($aND[$k][$e])){continue;} + if($k == 'align'){ + unset($a['align']); + if($e == 'img' && ($v == 'left' or $v == 'right')){$c[] = 'float: '. $v;} + elseif(($e == 'div' or $e == 'table') && $v == 'center'){$c[] = 'margin: auto';} + else{$c[] = 'text-align: '. $v;} + }elseif($k == 'bgcolor'){ + unset($a['bgcolor']); + $c[] = 'background-color: '. $v; + }elseif($k == 'border'){ + unset($a['border']); $c[] = "border: {$v}px"; + }elseif($k == 'bordercolor'){ + unset($a['bordercolor']); $c[] = 'border-color: '. $v; + }elseif($k == 'clear'){ + unset($a['clear']); $c[] = 'clear: '. ($v != 'all' ? $v : 'both'); + }elseif($k == 'compact'){ + unset($a['compact']); $c[] = 'font-size: 85%'; + }elseif($k == 'height' or $k == 'width'){ + unset($a[$k]); $c[] = $k. ': '. ($v[0] != '*' ? $v. (ctype_digit($v) ? 'px' : '') : 'auto'); + }elseif($k == 'hspace'){ + unset($a['hspace']); $c[] = "margin-left: {$v}px; margin-right: {$v}px"; + }elseif($k == 'language' && !isset($a['type'])){ + unset($a['language']); + $a['type'] = 'text/'. strtolower($v); + }elseif($k == 'name'){ + if($C['no_deprecated_attr'] == 2 or ($e != 'a' && $e != 'map')){unset($a['name']);} + if(!isset($a['id']) && preg_match('`[a-zA-Z][a-zA-Z\d.:_\-]*`', $v)){$a['id'] = $v;} + }elseif($k == 'noshade'){ + unset($a['noshade']); $c[] = 'border-style: none; border: 0; background-color: gray; color: gray'; + }elseif($k == 'nowrap'){ + unset($a['nowrap']); $c[] = 'white-space: nowrap'; + }elseif($k == 'size'){ + unset($a['size']); $c[] = 'size: '. $v. 'px'; + }elseif($k == 'start' or $k == 'value'){ + unset($a[$k]); + }elseif($k == 'type'){ + unset($a['type']); + static $ol_type = array('i'=>'lower-roman', 'I'=>'upper-roman', 'a'=>'lower-latin', 'A'=>'upper-latin', '1'=>'decimal'); + $c[] = 'list-style-type: '. (isset($ol_type[$v]) ? $ol_type[$v] : 'decimal'); + }elseif($k == 'vspace'){ + unset($a['vspace']); $c[] = "margin-top: {$v}px; margin-bottom: {$v}px"; + } + } + if(count($c)){ + $c = implode('; ', $c); + $a['style'] = isset($a['style']) ? rtrim($a['style'], ' ;'). '; '. $c. ';': $c. ';'; + } +} +// unique ID +if($C['unique_ids'] && isset($a['id'])){ + if(!preg_match('`^[A-Za-z][A-Za-z0-9_\-.:]*$`', ($id = $a['id'])) or (isset($GLOBALS['hl_Ids'][$id]) && $C['unique_ids'] == 1)){unset($a['id']); + }else{ + while(isset($GLOBALS['hl_Ids'][$id])){$id = $C['unique_ids']. $id;} + $GLOBALS['hl_Ids'][($a['id'] = $id)] = 1; + } +} +// xml:lang +if($C['xml:lang'] && isset($a['lang'])){ + $a['xml:lang'] = isset($a['xml:lang']) ? $a['xml:lang'] : $a['lang']; + if($C['xml:lang'] == 2){unset($a['lang']);} +} +// for transformed tag +if(!empty($trt)){ + $a['style'] = isset($a['style']) ? rtrim($a['style'], ' ;'). '; '. $trt : $trt; +} +// return with empty ele / +if(empty($C['hook_tag'])){ + $aA = ''; + foreach($a as $k=>$v){$aA .= " {$k}=\"{$v}\"";} + return "<{$e}{$aA}". (isset($eE[$e]) ? ' /' : ''). '>'; +} +else{return $C['hook_tag']($e, $a);} +// eof +} + +function hl_tag2(&$e, &$a, $t=1){ +// transform tag +if($e == 'center'){$e = 'div'; return 'text-align: center;';} +if($e == 'dir' or $e == 'menu'){$e = 'ul'; return '';} +if($e == 's' or $e == 'strike'){$e = 'span'; return 'text-decoration: line-through;';} +if($e == 'u'){$e = 'span'; return 'text-decoration: underline;';} +static $fs = array('0'=>'xx-small', '1'=>'xx-small', '2'=>'small', '3'=>'medium', '4'=>'large', '5'=>'x-large', '6'=>'xx-large', '7'=>'300%', '-1'=>'smaller', '-2'=>'60%', '+1'=>'larger', '+2'=>'150%', '+3'=>'200%', '+4'=>'300%'); +if($e == 'font'){ + $a2 = ''; + if(preg_match('`face\s*=\s*(\'|")([^=]+?)\\1`i', $a, $m) or preg_match('`face\s*=\s*([^"])(\S+)`i', $a, $m)){ + $a2 .= ' font-family: '. str_replace('"', '\'', trim($m[2])). ';'; + } + if(preg_match('`color\s*=\s*(\'|")?(.+?)(\\1|\s|$)`i', $a, $m)){ + $a2 .= ' color: '. trim($m[2]). ';'; + } + if(preg_match('`size\s*=\s*(\'|")?(.+?)(\\1|\s|$)`i', $a, $m) && isset($fs[($m = trim($m[2]))])){ + $a2 .= ' font-size: '. $fs[$m]. ';'; + } + $e = 'span'; return ltrim($a2); +} +if($t == 2){$e = 0; return 0;} +return ''; +// eof +} + +function hl_tidy($t, $w, $p){ +// Tidy/compact HTM +if(strpos(' pre,script,textarea', "$p,")){return $t;} +$t = str_replace(' </', '</', preg_replace(array('`(<\w[^>]*(?<!/)>)\s+`', '`\s+`', '`(<\w[^>]*(?<!/)>) `'), array(' $1', ' ', '$1'), preg_replace_callback(array('`(<(!\[CDATA\[))(.+?)(\]\]>)`sm', '`(<(!--))(.+?)(-->)`sm', '`(<(pre|script|textarea).*?>)(.+?)(</\2>)`sm'), create_function('$m', 'return $m[1]. str_replace(array("<", ">", "\n", "\r", "\t", " "), array("\x01", "\x02", "\x03", "\x04", "\x05", "\x07"), $m[3]). $m[4];'), $t))); +if(($w = strtolower($w)) == -1){ + return str_replace(array("\x01", "\x02", "\x03", "\x04", "\x05", "\x07"), array('<', '>', "\n", "\r", "\t", ' '), $t); +} +$s = strpos(" $w", 't') ? "\t" : ' '; +$s = preg_match('`\d`', $w, $m) ? str_repeat($s, $m[0]) : str_repeat($s, ($s == "\t" ? 1 : 2)); +$n = preg_match('`[ts]([1-9])`', $w, $m) ? $m[1] : 0; +$a = array('br'=>1); +$b = array('button'=>1, 'input'=>1, 'option'=>1); +$c = array('caption'=>1, 'dd'=>1, 'dt'=>1, 'h1'=>1, 'h2'=>1, 'h3'=>1, 'h4'=>1, 'h5'=>1, 'h6'=>1, 'isindex'=>1, 'label'=>1, 'legend'=>1, 'li'=>1, 'object'=>1, 'p'=>1, 'pre'=>1, 'td'=>1, 'textarea'=>1, 'th'=>1); +$d = array('address'=>1, 'blockquote'=>1, 'center'=>1, 'colgroup'=>1, 'dir'=>1, 'div'=>1, 'dl'=>1, 'fieldset'=>1, 'form'=>1, 'hr'=>1, 'iframe'=>1, 'map'=>1, 'menu'=>1, 'noscript'=>1, 'ol'=>1, 'optgroup'=>1, 'rbc'=>1, 'rtc'=>1, 'ruby'=>1, 'script'=>1, 'select'=>1, 'table'=>1, 'tfoot'=>1, 'thead'=>1, 'tr'=>1, 'ul'=>1); +ob_start(); +if(isset($d[$p])){echo str_repeat($s, ++$n);} +$t = explode('<', $t); +echo ltrim(array_shift($t)); +for($i=-1, $j=count($t); ++$i<$j;){ + $r = ''; list($e, $r) = explode('>', $t[$i]); + $x = $e[0] == '/' ? 0 : (substr($e, -1) == '/' ? 1 : ($e[0] != '!' ? 2 : -1)); + $y = !$x ? ltrim($e, '/') : ($x > 0 ? substr($e, 0, strcspn($e, ' ')) : 0); + $e = "<$e>"; + if(isset($d[$y])){ + if(!$x){echo "\n", str_repeat($s, --$n), "$e\n", str_repeat($s, $n);} + else{echo "\n", str_repeat($s, $n), "$e\n", str_repeat($s, ($x != 1 ? ++$n : $n));} + echo ltrim($r); continue; + } + $f = "\n". str_repeat($s, $n); + if(isset($c[$y])){ + if(!$x){echo $e, $f, ltrim($r);} + else{echo $f, $e, $r;} + }elseif(isset($b[$y])){echo $f, $e, $r; + }elseif(isset($a[$y])){echo $e, $f, ltrim($r); + }elseif(!$y){echo $f, $e, $f, ltrim($r); + }else{echo $e, $r;} +} +$t = preg_replace('`[\n]\s*?[\n]+`', "\n", ob_get_contents()); +ob_end_clean(); +if(($l = strpos(" $w", 'r') ? (strpos(" $w", 'n') ? "\r\n" : "\r") : 0)){ + $t = str_replace("\n", $l, $t); +} +return str_replace(array("\x01", "\x02", "\x03", "\x04", "\x05", "\x07"), array('<', '>', "\n", "\r", "\t", ' '), $t); +// eof +} + +function hl_version(){ +// rel +return '1.1.8.1'; +// eof +} + +function kses($t, $h, $p=array('http', 'https', 'ftp', 'news', 'nntp', 'telnet', 'gopher', 'mailto')){ +// kses compat +foreach($h as $k=>$v){ + $h[$k]['n']['*'] = 1; +} +$C['cdata'] = $C['comment'] = $C['make_tag_strict'] = $C['no_deprecated_attr'] = $C['unique_ids'] = 0; +$C['keep_bad'] = 1; +$C['elements'] = count($h) ? strtolower(implode(',', array_keys($h))) : '-*'; +$C['hook'] = 'kses_hook'; +$C['schemes'] = '*:'. implode(',', $p); +return htmLawed($t, $C, $h); +// eof +} + +function kses_hook($t, &$C, &$S){ +// kses compat +return $t; +// eof +}
\ No newline at end of file diff --git a/extlib/htmLawed/htmLawedTest.php b/extlib/htmLawed/htmLawedTest.php new file mode 100644 index 000000000..776828699 --- /dev/null +++ b/extlib/htmLawed/htmLawedTest.php @@ -0,0 +1,592 @@ +<?php + +/* +htmLawedTest.php, 16 July 2009 +htmLawed 1.1.8.1, 16 July 2009 +Copyright Santosh Patnaik +GPL v3 license +A PHP Labware internal utility - http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed + +Test htmLawed; user provides text input; input and processed input are shown as highlighted code and rendered HTML; also shown are execution time and peak memory usage +*/ + +// config +$_errs = 0; // display PHP errors +$_limit = 8000; // input character limit + +// more config +$_hlimit = 1000; // input character limit for showing hexdumps +$_hilite = 1; // 0 turns off slow Javascript-based code-highlighting, e.g., if $_limit is high +$_w3c_validate = 1; // 1 to show buttons to send input/output to w3c validator +$_sid = 'sid'; // session name; alphanum. +$_slife = 30; // session life in min. + +// errors +error_reporting(E_ALL | (defined('E_STRICT') ? E_STRICT : 1)); +ini_set('display_errors', $_errs); + +// session +session_name($_sid); +session_cache_limiter('private'); +session_cache_expire($_slife); +ini_set('session.gc_maxlifetime', $_slife * 60); +ini_set('session.use_only_cookies', 1); +ini_set('session.cookie_lifetime', 0); +session_start(); +if(!isset($_SESSION['token'])){ + $_SESSION['token'] = md5(uniqid(rand(), 1)); +} + +// slashes +if(get_magic_quotes_gpc()){ + foreach($_POST as $k => $v){ + $_POST[$k] = stripslashes($v); + } + ini_set('magic_quotes_gpc', 0); +} +set_magic_quotes_runtime(0); + +$_POST['enc'] = (isset($_POST['enc']) and preg_match('`^[-\w]+$`', $_POST['enc'])) ? $_POST['enc'] : 'utf-8'; + +// token for anti-CSRF +if(count($_POST)){ + if((empty($_GET['pre']) and ((!empty($_POST['token']) and !empty($_SESSION['token']) and $_POST['token'] != $_SESSION['token']) or empty($_POST[$_sid]) or $_POST[$_sid] != session_id() or empty($_COOKIE[$_sid]) or $_COOKIE[$_sid] != session_id())) or ($_POST[$_sid] != session_id())){ + $_POST = array('enc'=>'utf-8'); + } +} +if(empty($_GET['pre'])){ + $_SESSION['token'] = md5(uniqid(rand(), 1)); + $token = $_SESSION['token']; + session_regenerate_id(1); +} + +// compress +if(function_exists('gzencode') && isset($_SERVER['HTTP_ACCEPT_ENCODING']) && preg_match('`gzip|deflate`i', $_SERVER['HTTP_ACCEPT_ENCODING']) && !ini_get('zlib.output_compression')){ + ob_start('ob_gzhandler'); +} + +// HTM for unprocessed +if(isset($_POST['inputH'])){ + echo '<html><head><title>htmLawed test: HTML view of unprocessed input</title></head><body style="margin:0; padding: 0;"><p style="background-color: black; color: white; padding: 2px;"> Rendering of unprocessed input without an HTML doctype or charset declaration <small><a style="color: white; text-decoration: none;" href="1" onclick="javascript:window.close(this); return false;">close window</a> | <a style="color: white; text-decoration: none;" href="htmLawedTest.php" onclick="javascript: window.open(\'htmLawedTest.php\', \'hlmain\'); window.close(this); return false;">htmLawed test page</a></small></p><div>', $_POST['inputH'], '</div></body></html>'; + exit; +} + +// main +$_POST['text'] = isset($_POST['text']) ? $_POST['text'] : 'text to process; < '. $_limit. ' characters'. ($_hlimit ? ' (for binary hexdump view, < '. $_hlimit. ')' : ''); +$do = (!empty($_POST[$_sid]) && isset($_POST['text'][0]) && !isset($_POST['text'][$_limit])) ? 1 : 0; +$limit_exceeded = isset($_POST['text'][$_limit]) ? 1 : 0; +$pre_mem = memory_get_usage(); +$validation = (!empty($_POST[$_sid]) and isset($_POST['w3c_validate'][0])) ? 1 : 0; +include './htmLawed.php'; + +function format($t){ + $t = "\n". str_replace(array("\t", "\r\n", "\r", '&', '<', '>', "\n"), array(' ', "\n", "\n", '&', '<', '>', "<span class=\"newline\">¬</span><br />\n"), $t); + return str_replace(array('<br />', "\n ", ' '), array("\n<br />\n", "\n ", ' '), $t); +} + +function hexdump($d){ +// Mainly by Aidan Lister <aidan@php.net>, Peter Waller <iridum@php.net> + $hexi = ''; + $ascii = ''; + ob_start(); + echo '<pre>'; + $offset = 0; + $len = strlen($d); + for($i=$j=0; $i<$len; $i++) + { + // Convert to hexidecimal + $hexi .= sprintf("%02X ", ord($d[$i])); + // Replace non-viewable bytes with '.' + if(ord($d[$i]) >= 32){ + $ascii .= htmlspecialchars($d[$i]); + }else{ + $ascii .= '.'; + } + // Add extra column spacing + if($j == 7){ + $hexi .= ' '; + $ascii .= ' '; + } + // Add row + if(++$j == 16 || $i == $len-1){ + // Join the hexi / ascii output + echo sprintf("%04X %-49s %s", $offset, $hexi, $ascii); + // Reset vars + $hexi = $ascii = ''; + $offset += 16; + $j = 0; + // Add newline + if ($i !== $len-1){ + echo "\n"; + } + } + } + echo '</pre>'; + $o = ob_get_contents(); + ob_end_clean(); + return $o; +} +?> + +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html lang="en" xml:lang="en"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=<?php echo htmlspecialchars($_POST['enc']); ?>" /> +<meta name="description" content="htmLawed <?php echo hl_version();?> test page" /> +<style type="text/css"><!--/*--><![CDATA[/*><!--*/ +a, a.resizer{text-decoration:none;} +a:hover, a.resizer:hover{color:red;} +a.resizer{color:green; float:right;} +body{background-color:#efefef;} +body, button, div, html, input, p{font-size:13px; font-family:'Lucida grande', Verdana, Arial, Helvetica, sans-serif;} +button, input{font-size: 85%;} +div.help{border-top: 1px dotted gray; margin-top: 15px; padding-top: 15px; color:#999999;} +#inputC, #inputD, #inputF, #inputR, #outputD, #outputF, #outputH, #outputR, #settingF{display:block;} +#inputC, #settingF{background-color:white; border:1px gray solid; padding:3px;} +#inputC li{margin: 0; padding: 0;} +#inputC ul{margin: 0; padding: 0; margin-left: 14px;} +#inputC input{margin: 0; margin-left: 2px; margin-right: 2px; padding: 1px; vertical-align: middle;} +#inputD{overflow:auto; background-color:#ffff99; border:1px #cc9966 solid; padding:3px;} +#inputR{overflow:auto; background-color:#ffffcc; border:1px #ffcc99 solid; padding:3px;} +#inputC, #settingF, #inputD, #inputR, #outputD, #outputR, textarea{font-size:100%; font-family:'Bitstream vera sans mono', 'courier new', 'courier', monospace;} +#outputD{overflow:auto; background-color: #99ffcc; border:1px #66cc99 solid; padding:3px;} +#outputH{overflow:auto; background-color:white; padding:3px; border:1px #dcdcdc solid;} +#outputR{overflow:auto; background-color: #ccffcc; border:1px #99cc99 solid; padding:3px;} +span.cmtcdata{color: orange;} +span.ctag{color:red;} +span.ent{border-bottom:1px dotted #999999;} +span.etag{color:purple;} +span.help{color:#999999;} +span.newline{color:#dcdcdc;} +span.notice{color:green;} +span.otag{color:blue;} +#topmost{margin:auto; width:98%;} +/*]]>*/--></style> +<script type="text/javascript"><!--//--><![CDATA[//><!-- +window.name = 'hlmain'; +function hl(i){ + <?php if(!$_hilite){echo 'return;'; }?> + var e = document.getElementById(i); + if(!e){return;} + run(e, '</[a-z1-6]+>', 'ctag'); + run(e, '<[a-z]+(?:[^>]*)/>', 'etag'); + run(e, '<[a-z1-6]+(?:[^>]*)>', 'otag'); + run(e, '&[#a-z0-9]+;', 'ent'); + run(e, '<!(?:(?:--(?:.|\n)*?--)|(?:\\[CDATA\\[(?:.|\n)*?\\]\\]))>', 'cmtcdata'); +} +function sndProc(){ + var f = document.getElementById('testform'); + if(!f){return;} + var e = document.createElement('input'); + e.type = 'hidden'; + e.name = '<?php echo htmlspecialchars($_sid); ?>'; + e.id = '<?php echo htmlspecialchars($_sid); ?>'; + e.value = readCookie('<?php echo htmlspecialchars($_sid); ?>'); + f.appendChild(e); + f.submit(); +} +function readCookie(n){ + var ne = n + '='; + var ca = document.cookie.split(';'); + for(var i=0;i < ca.length;i++){ + var c = ca[i]; + while(c.charAt(0)==' '){ + c = c.substring(1,c.length); + } + if(c.indexOf(ne) == 0){ + return c.substring(ne.length,c.length); + } + } + return null; +} +function run(e, q, c){ + var q = new RegExp(q); + if(e.firstChild == null){ + var m = q.exec(e.data); + if(m){ + var v = m[0]; + var k2 = e.splitText(m.index); + var k3 = k2.splitText(v.length); + var s = e.ownerDocument.createElement('span'); + e.parentNode.replaceChild(s, k2); + s.className = c; s.appendChild(k2); + } + } + for(var k = e.firstChild; k != null; k = k.nextSibling){ + if(k.nodeType == 3){ + var m = q.exec(k.data); + if(m){ + var v = m[0]; + var k2 = k.splitText(m.index); + var k3 = k2.splitText(v.length); + var s = k.ownerDocument.createElement('span'); + k.parentNode.replaceChild(s, k2); + s.className = c; s.appendChild(k2); + } + } + else if(c == 'ent' && k.nodeType == 1){ + var d = k.firstChild; + if(d){ + var m = q.exec(d.data); + if(m){ + var v = m[0]; + var d2 = d.splitText(m.index); + var d3 = d2.splitText(v.length); + var s = d.ownerDocument.createElement('span'); + d.parentNode.replaceChild(s, d2); + s.className = c; s.appendChild(d2); + } + } + } + } +} +function toggle(i){ + var e = document.getElementById(i); + if(!e){return;} + if(e.style){ + var a = e.style.display; + if(a == 'block'){e.style.display = 'none'; return;} + if(a == 'none'){e.style.display = 'block';} + else{e.style.display = 'none';} + return; + } + var a = e.visibility; + if(a == 'hidden'){e.visibility = 'show'; return;} + if(a == 'show'){e.visibility = 'hidden';} +} +function sndUnproc(){ + var i = document.getElementById('text'); + if(!i){return;} + i = i.value; + i = i.replace(/>/g, '>'); + i = i.replace(/</g, '<'); + i = i.replace(/"/g, '"'); + var w = window.open('htmLawedTest.php?pre=1', 'hlprehtm'); + var f = document.createElement('form'); + f.enctype = 'application/x-www-form-urlencoded'; + f.method = 'post'; + f.acceptCharset = '<?php echo htmlspecialchars($_POST['enc']); ?>'; + if(f.style){f.style.display = 'none';} + else{f.visibility = 'hidden';} + f.innerHTML = '<p style="display:none;"><input style="display:none;" type="hidden" name="token" id="token" value="<?php echo $token; ?>" /><input style="display:none;" type="hidden" name="<?php echo htmlspecialchars($_sid); ?>" id="<?php echo htmlspecialchars($_sid); ?>" value="' + readCookie('<?php echo htmlspecialchars($_sid); ?>') + '" /><input style="display:none;" type="hidden" name="inputH" id="inputH" value="'+ i+ '" /></p>'; + f.action = 'htmLawedTest.php?pre=1'; + f.target = 'hlprehtm'; + f.method = 'post'; + var b = document.getElementsByTagName('body')[0]; + b.appendChild(f); + f.submit(); + w.focus; +} +function sndValidn(id, type){ + var i = document.getElementById(id); + if(!i){return;} + i = i.value; + i = i.replace(/>/g, '>'); + i = i.replace(/</g, '<'); + i = i.replace(/"/g, '"'); + var w = window.open('http://validator.w3.org/check', 'validate'+id+type); + var f = document.createElement('form'); + f.enctype = 'application/x-www-form-urlencoded'; + f.method = 'post'; + f.acceptCharset = '<?php echo htmlspecialchars($_POST['enc']); ?>'; + if(f.style){f.style.display = 'none';} + else{f.visibility = 'hidden';} + f.innerHTML = '<p style="display:none;"><input style="display:none;" type="hidden" name="fragment" id="fragment" value="'+ i+ '" /><input style="display:none;" type="hidden" name="prefill" id="prefill" value="1" /><input style="display:none;" type="hidden" name="prefill_doctype" id="prefill_doctype" value="'+ type+ '" /><input style="display:none;" type="hidden" name="group" id="group" value="1" /><input type="hidden" name="ss" id="ss" value="1" /></p>'; + f.action = 'http://validator.w3.org/check'; + f.target = 'validate'+id+type; + var b = document.getElementsByTagName('body')[0]; + b.appendChild(f); + f.submit(); + w.focus; +} +tRs = { + formEl: null, + resizeClass: 'textarea', + adEv: function(t,ev,fn){ + if(typeof document.addEventListener != 'undefined'){ + t.addEventListener(ev,fn,false); + }else{ + t.attachEvent('on' + ev, fn); + } + }, + rmEv: function(t,ev,fn){ + if(typeof document.removeEventListener != 'undefined'){ + t.removeEventListener(ev,fn,false); + }else + { + t.detachEvent('on' + ev, fn); + } + }, + adBtn: function(){ + var textareas = document.getElementsByTagName('textarea'); + for(var i = 0; i < textareas.length; i++){ + var txtclass=textareas[i].className; + if(txtclass.substring(0,tRs.resizeClass.length)==tRs.resizeClass || + txtclass.substring(txtclass.length -tRs.resizeClass.length)==tRs.resizeClass){ + var a = document.createElement('a'); + a.appendChild(document.createTextNode("\u2195")); + a.style.cursor = 'n-resize'; + a.className= 'resizer'; + a.title = 'click-drag to resize' + tRs.adEv(a, 'mousedown', tRs.initResize); + textareas[i].parentNode.appendChild(a); + } + } + }, + initResize: function(event){ + if(typeof event == 'undefined'){ + event = window.event; + } + if(event.srcElement){ + var target = event.srcElement.previousSibling; + }else{ + var target = event.target.previousSibling; + } + if(target.nodeName.toLowerCase() == 'textarea' || (target.nodeName.toLowerCase() == 'input' && target.type == 'text')){ + tRs.formEl = target; + tRs.formEl.startHeight = tRs.formEl.clientHeight; + tRs.formEl.startY = event.clientY; + tRs.adEv(document, 'mousemove', tRs.resize); + tRs.adEv(document, 'mouseup', tRs.stopResize); + tRs.formEl.parentNode.style.cursor = 'n-resize'; + tRs.formEl.style.cursor = 'n-resize'; + try{ + event.preventDefault(); + }catch(e){ + } + } + }, + resize: function(event){ + if(typeof event == 'undefined'){ + event = window.event; + } + if(tRs.formEl.nodeName.toLowerCase() == 'textarea'){ + tRs.formEl.style.height = event.clientY - tRs.formEl.startY + tRs.formEl.startHeight + 'px'; + } + }, + stopResize: function(event){ + tRs.rmEv(document, 'mousedown', tRs.initResize); + tRs.rmEv(document, 'mousemove', tRs.resize); + tRs.formEl.style.cursor = 'text'; + tRs.formEl.parentNode.style.cursor = 'auto'; + return false; + } +}; +tRs.adEv(window, 'load', tRs.adBtn); +//--><!]]></script> +<title>htmLawed (<?php echo hl_version();?>) test</title> +</head> +<body> +<div id="topmost"> + +<h5 style="float: left; display: inline; margin-top: 0; margin-bottom: 5px;"><a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed/index.php" title="htmLawed home">HTM<big><big>L</big></big>AWED</a> <?php echo hl_version();?> <a href="htmLawedTest.php" title="test home">TEST</a></h5> +<span style="float: right;" class="help"><a href="htmLawed_README.htm"><span class="notice">htm</span></a> / <a href="htmLawed_README.txt"><span class="notice">txt</span></a> documentation</span><br style="clear:both;" /> + +<a href="htmLawedTest.php" title="[toggle visibility] type or copy-paste" onclick="javascript:toggle('inputF'); return false;"><span class="notice">Input »</span> <span class="help" title="limit lower with multibyte characters<?php echo (($_hlimit < $_limit && $_hlimit)? '; limit is '. $_hlimit. ' for viewing binaries' : ''); ?>"><small>(max. <?php echo htmlspecialchars($_limit);?> chars)</small></span></a> + +<form id="testform" name="testform" action="htmLawedTest.php" method="post" accept-charset="<?php echo htmlspecialchars($_POST['enc']); ?>" style="padding:0; margin: 0; display:inline;"> + +<div id="inputF" style="display: block;"> + +<input type="hidden" name="token" id="token" value="<?php echo $token; ?>" /> +<div><textarea id="text" class="textarea" name="text" rows="5" cols="100" style="width: 100%;"><?php echo htmlspecialchars($_POST['text']);?></textarea></div> +<input type="submit" id="submitF" name="submitF" value="Process" style="float:left;" title="filter using htmLawed" onclick="javascript: sndProc(); return false;" onkeypress="javascript: sndProc(); return false;" /> + +<?php +if($do){ + if($validation){ + echo '<input type="hidden" value="1" name="w3c_validate" id="w3c_validate" />'; + } +?> + +<button type="button" title="rendered as web-page without a doctype or charset declaration" style="float: right;" onclick="javascript: sndUnproc(); return false;" onkeypress="javascript: sndUnproc(); return false;">View unprocessed</button> +<button type="button" onclick="javascript:document.getElementById('text').focus();document.getElementById('text').select()" title="select all to copy" style="float:right;">Select all</button> + +<?php +if($_w3c_validate && $validation){ +?> + +<button type="button" title="HTML 4.01 W3C online validation" style="float: right;" onclick="javascript: sndValidn('text', 'html401'); return false;" onkeypress="javascript: sndValidn('text', 'html401'); return false;">Check HTML</button> +<button type="button" title="XHTML 1.1 W3C online validation" style="float: right;" onclick="javascript: sndValidn('text', 'xhtml110'); return false;" onkeypress="javascript: sndValidn('text', 'xhtml110'); return false;">Check XHTML</button> + +<?php + } +} +else{ + if($_w3c_validate){ + echo '<span style="float: right;" class="help" title="for direct submission of input or output code to W3C validator for (X)HTML validation"><span style="font-size: 85%;"> Validator tools: </span><input type="checkbox" value="1" name="w3c_validate" id="w3c_validate" style="vertical-align: middle;"', ($validation ? ' checked="checked"' : ''), ' /></span>'; + } +} +?> + +<span style="float:right;" class="help"><span style="font-size: 85%;">Encoding: </span><input type="text" size="8" id="enc" name="enc" style="vertical-align: middle;" value="<?php echo htmlspecialchars($_POST['enc']); ?>" title="IANA-recognized name of the input character-set; can be multiple ;- or space-separated values; may not work in some browsers" /></span> + +</div> +<br style="clear:both;" /> + +<?php +if($limit_exceeded){ + echo '<br /><strong>Input text is too long!</strong><br />'; +} +?> + +<br /> + +<a href="htmLawedTest.php" title="[toggle visibility] htmLawed configuration" onclick="javascript:toggle('inputC'); return false;"><span class="notice">Settings »</span></a> + +<div id="inputC" style="display: none;"> +<table summary="none"> +<tr> +<td><span class="help" title="$config argument">Config:</span></td> +<td><ul> + +<?php +$cfg = array( +'abs_url'=>array('3', '0', 'absolute/relative URL conversion', '-1'), +'and_mark'=>array('2', '0', 'mark original <em>&</em> chars', '0', 'd'=>1), // 'd' to disable +'anti_link_spam'=>array('1', '0', 'modify <em>href</em> values as an anti-link spam measure', '0', array(array('30', '1', '', 'regex for extra <em>rel</em>'), array('30', '2', '', 'regex for no <em>href</em>'))), +'anti_mail_spam'=>array('1', '0', 'replace <em>@</em> in <em>mailto:</em> URLs', '0', '8', 'NO@SPAM', 'replacement'), +'balance'=>array('2', '1', 'fix nestings and balance tags', '0'), +'base_url'=>array('', '', 'base URL', '25'), +'cdata'=>array('4', 'nil', 'allow <em>CDATA</em> sections', 'nil'), +'clean_ms_char'=>array('3', '0', 'replace bad characters introduced by Microsoft apps. like <em>Word</em>', '0'), +'comment'=>array('4', 'nil', 'allow HTML comments', 'nil'), +'css_expression'=>array('2', 'nil', 'allow dynamic expressions in CSS style properties', 'nil'), +'deny_attribute'=>array('1', '0', 'denied attributes', '0', '50', '', 'these'), +'elements'=>array('', '', 'allowed elements', '50'), +'hexdec_entity'=>array('3', '1', 'convert hexadecimal numeric entities to decimal ones, or vice versa', '0'), +'hook'=>array('', '', 'name of hook function', '25'), +'hook_tag'=>array('', '', 'name of custom function to further check attribute values', '25'), +'keep_bad'=>array('7', '6', 'keep, or remove <em>bad</em> tag content', '0'), +'lc_std_val'=>array('2', '1', 'lower-case std. attribute values like <em>radio</em>', '0'), +'make_tag_strict'=>array('3', 'nil', 'transform deprecated elements', 'nil'), +'named_entity'=>array('2', '1', 'allow named entities, or convert numeric ones', '0'), +'no_deprecated_attr'=>array('3', '1', 'allow deprecated attributes, or transform them', '0'), +'parent'=>array('', 'div', 'name of parent element', '25'), +'safe'=>array('2', '0', 'for most <em>safe</em> HTML', '0'), +'schemes'=>array('', 'href: aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet; *:file, http, https', 'allowed URL protocols', '50'), +'show_setting'=>array('', 'htmLawed_setting', 'variable name to record <em>finalized</em> htmLawed settings', '25', 'd'=>1), +'style_pass'=>array('2', 'nil', 'do not look at <em>style</em> attribute values', 'nil'), +'tidy'=>array('3', '0', 'beautify/compact', '-1', '8', '1t1', 'format'), +'unique_ids'=>array('2', '1', 'unique <em>id</em> values', '0', '8', 'my_', 'prefix'), +'valid_xhtml'=>array('2', 'nil', 'auto-set various parameters for most valid XHTML', 'nil'), +'xml:lang'=>array('3', 'nil', 'auto-add <em>xml:lang</em> attribute', '0'), +); +foreach($cfg as $k=>$v){ + echo '<li>', $k, ': '; + if(!empty($v[0])){ // input radio + $j = $v[3]; + for($i = $j-1; ++$i < $v[0]+$v[3];++$j){ + echo '<input type="radio" name="h', $k, '" value="', $i, '"', (!isset($_POST['h'. $k]) ? ($v[1] == $i ? ' checked="checked"' : '') : ($_POST['h'. $k] == $i ? ' checked="checked"' : '')), (isset($v['d']) ? ' disabled="disabled"' : ''), ' />', $i, ' '; + } + if($v[1] == 'nil'){ + echo '<input type="radio" name="h', $k, '" value="nil"', ((!isset($_POST['h'. $k]) or $_POST['h'. $k] == 'nil') ? ' checked="checked"' : ''), (isset($v['d']) ? ' disabled="disabled"' : ''), ' />not set '; + } + if(!empty($v[4])){ // + input text box + echo '<input type="radio" name="h', $k, '" value="', $j, '"', (((isset($_POST['h'. $k]) && $_POST['h'. $k] == $j) or (!isset($_POST['h'. $k]) && $j == $v[1])) ? ' checked="checked"' : ''), (isset($v['d']) ? ' disabled="disabled"' : ''), ' />'; + if(!is_array($v[4])){ + echo $v[6], ': <input type="text" size="', $v[4], '" name="h', $k. $j, '" value="', htmlspecialchars(isset($_POST['h'. $k. $j][0]) ? $_POST['h'. $k. $j] : $v[5]), '"', (isset($v['d']) ? ' disabled="disabled"' : ''), ' />'; + } + else{ + foreach($v[4] as $z){ + echo ' ', $z[3], ': <input type="text" size="', $z[0], '" name="h', $k. $j. $z[1], '" value="', htmlspecialchars(isset($_POST['h'. $k. $j. $z[1]][0]) ? $_POST['h'. $k. $j. $z[1]] : $z[2]), '"', (isset($v['d']) ? ' disabled="disabled"' : ''), ' />'; + } + } + } + } + elseif(ctype_digit($v[3])){ // input text + echo '<input type="text" size="', $v[3], '" name="h', $k, '" value="', htmlspecialchars(isset($_POST['h'. $k][0]) ? $_POST['h'. $k] : $v[1]), '"', (isset($v['d']) ? ' disabled="disabled"' : ''), ' />'; + } + else{} // text-area + echo ' <span class="help">', $v[2], '</span></li>'; +} +echo '</ul></td></tr><tr><td><span style="vertical-align: top;" class="help" title="$spec argument: element-specific attribute rules">Spec:</span></td><td><textarea name="spec" id="spec" cols="70" rows="3" style="width:80%;">', htmlspecialchars((isset($_POST['spec']) ? $_POST['spec'] : '')), '</textarea></td></tr></table>'; +?> + +</div> +</form> + +<?php +if($do){ + $cfg = array(); + foreach($_POST as $k=>$v){ + if($k[0] == 'h' && $v != 'nil'){ + $cfg[substr($k, 1)] = $v; + } + } + + if($cfg['anti_link_spam'] && (!empty($cfg['anti_link_spam11']) or !empty($cfg['anti_link_spam12']))){ + $cfg['anti_link_spam'] = array($cfg['anti_link_spam11'], $cfg['anti_link_spam12']); + } + unset($cfg['anti_link_spam11'], $cfg['anti_link_spam12']); + if($cfg['anti_mail_spam'] == 1){ + $cfg['anti_mail_spam'] = isset($cfg['anti_mail_spam1'][0]) ? $cfg['anti_mail_spam1'] : 0; + } + unset($cfg['anti_mail_spam11']); + if($cfg['deny_attribute'] == 1){ + $cfg['deny_attribute'] = isset($cfg['deny_attribute1'][0]) ? $cfg['deny_attribute1'] : 0; + } + unset($cfg['deny_attribute1']); + if($cfg['tidy'] == 2){ + $cfg['tidy'] = isset($cfg['tidy2'][0]) ? $cfg['tidy2'] : 0; + } + unset($cfg['tidy2']); + if($cfg['unique_ids'] == 2){ + $cfg['unique_ids'] = isset($cfg['unique_ids2'][0]) ? $cfg['unique_ids2'] : 1; + } + unset($cfg['unique_ids2']); + unset($cfg['and_mark']); // disabling and_mark + + $cfg['show_setting'] = 'hlcfg'; + $st = microtime(); + $out = htmLawed($_POST['text'], $cfg, str_replace(array('$', '{'), '', $_POST['spec'])); + $et = microtime(); + echo '<br /><a href="htmLawedTest.php" title="[toggle visibility] syntax-highlighted" onclick="javascript:toggle(\'inputR\'); return false;"><span class="notice">Input code »</span></a> <span class="help" title="tags estimated as half of total > and < chars; values may be inaccurate for non-ASCII text"><small><big>', strlen($_POST['text']), '</big> chars, ~<big>', round((substr_count($_POST['text'], '>') + substr_count($_POST['text'], '<'))/2), '</big> tags</small> </span><div id="inputR" style="display: none;">', format($_POST['text']), '</div><script type="text/javascript">hl(\'inputR\');</script>', (!isset($_POST['text'][$_hlimit]) ? ' <a href="htmLawedTest.php" title="[toggle visibility] hexdump; non-viewable characters like line-returns are shown as dots" onclick="javascript:toggle(\'inputD\'); return false;"><span class="notice">Input binary » </span></a><div id="inputD" style="display: none;">'. hexdump($_POST['text']). '</div>' : ''), ' <a href="htmLawedTest.php" title="[toggle visibility] finalized internal settings as interpreted by htmLawed; for developers" onclick="javascript:toggle(\'settingF\'); return false;"><span class="notice">Finalized internal settings » </span></a> <div id="settingF" style="display: none;">', str_replace(array(' ', "\t", ' '), array(' ', ' ', ' '), nl2br(htmlspecialchars(print_r($GLOBALS['hlcfg']['config'], true)))), '</div><script type="text/javascript">hl(\'settingF\');</script>', '<br /><a href="htmLawedTest.php" title="[toggle visibility] suitable for copy-paste" onclick="javascript:toggle(\'outputF\'); return false;"><span class="notice">Output »</span></a> <span class="help" title="approx., server-specific value excluding the \'include()\' call"><small>htmLawed processing time <big>', number_format(((substr($et,0,9)) + (substr($et,-10)) - (substr($st,0,9)) - (substr($st,-10))),4), '</big> s</small></span>', (($mem = memory_get_peak_usage()) !== false ? '<span class="help"><small>, peak memory usage <big>'. round(($mem-$pre_mem)/1048576, 2). '</big> <small>MB</small>' : ''), '</small></span><div id="outputF" style="display: block;"><div><textarea id="text2" class="textarea" name="text2" rows="5" cols="100" style="width: 100%;">', htmlspecialchars($out), '</textarea></div><button type="button" onclick="javascript:document.getElementById(\'text2\').focus();document.getElementById(\'text2\').select()" title="select all to copy" style="float:right;">Select all</button>'; + if($_w3c_validate && $validation) + { +?> + +<button type="button" title="HTML 4.01 W3C online validation" style="float: right;" onclick="javascript: sndValidn('text2', 'html401'); return false;" onkeypress="javascript: sndValidn('text2', 'html401'); return false;">Check HTML</button> +<button type="button" title="XHTML 1.1 W3C online validation" style="float: right;" onclick="javascript: sndValidn('text2', 'xhtml110'); return false;" onkeypress="javascript: sndValidn('text2', 'xhtml110'); return false;">Check XHTML</button> + +<?php + } + echo '</div><br /><a href="htmLawedTest.php" title="[toggle visibility] syntax-highlighted" onclick="javascript:toggle(\'outputR\'); return false;"><span class="notice">Output code »</span></a><div id="outputR" style="display: block;">', format($out), '</div><script type="text/javascript">hl(\'outputR\');</script>', (!isset($_POST['text'][$_hlimit]) ? '<br /><a href="htmLawedTest.php" title="[toggle visibility] hexdump; non-viewable characters like line-returns are shown as dots" onclick="javascript:toggle(\'outputD\'); return false;"><span class="notice">Output binary »</span></a><div id="outputD" style="display: none;">'. hexdump($out). '</div>' : ''), '<br /><a href="htmLawedTest.php" title="[toggle visibility] XHTML 1 Transitional doctype" onclick="javascript:toggle(\'outputH\'); return false;"><span class="notice">Output rendered »</span></a><div id="outputH" style="display: block;">', $out, '</div>'; +} +else{ +?> + +<br /> + +<div class="help">Use with a Javascript- and cookie-enabled, relatively new version of a common browser. <em>Submitted input will also be HTML-rendered (XHTML 1) after htmLawed-filtering.</em> + +<?php echo (file_exists('./htmLawed_TESTCASE.txt') ? '<br /><br />You can use text from <a href="htmLawed_TESTCASE.txt"><span class="notice">this collection of test-cases</span></a> in the input. Set the character encoding of the browser to Unicode/utf-8 before copying.' : ''); ?> + +<br /><br />For anti-XSS tests, try the <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed/htmLawedSafeModeTest.php"><span class="notice">special test-page</span></a> or see <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed/rsnake/RSnakeXSSTest.htm"><span class="notice">these results</span></a>. + +<br /><br /><small>Change <em>Encoding</em> to reflect the character encoding of the input text. Even then, it may not work or some characters may not display properly because of variable browser support and because of the form interface. Developers can write some PHP code to capture the filtered input to a file if this is important. +<br /><br />Refer to the htmLawed documentation (<a href="htmLawed_README.htm"><span class="notice">htm</span></a>/<a href="htmLawed_README.txt"><span class="notice">txt</span></a>) for details about <em>Settings</em>, and htmLawed's behavior and limitations. For <em>Settings</em>, incorrectly-specified values like regular expressions are silently ignored. One or more settings form-fields may have been disabled. Some characters are not allowed in the <em>Spec</em> field. + + +<br /><br />Hovering the mouse over some of the text can provide additional information in some browsers.</small> + +<?php +if($_w3c_validate){ +?> + +<small><br /><br />Because of character-encoding issues, the W3C validator (anyway not perfect) may reject validation requests or invalidate otherwise-valid code, esp. if text was copy-pasted in the input box. Local applications like the <em>HTML Validator</em> Firefox browser add-on may be useful in such cases.</small> + +<?php +} +?> + +</div> + +<?php +} +?> + +</div> +</body> +</html>
\ No newline at end of file diff --git a/extlib/htmLawed/htmLawed_README.htm b/extlib/htmLawed/htmLawed_README.htm new file mode 100644 index 000000000..e560e2eb2 --- /dev/null +++ b/extlib/htmLawed/htmLawed_README.htm @@ -0,0 +1,1979 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta http-equiv="Content-Language" content="en" /> +<meta name="description" content="htmLawed PHP software is a free, open-source, customizable HTML input purifier and filter - htmLawed_README.txt - presented with rTxt2htm, a PHP Labware utility" /> +<meta name="keywords" content="htmLawed, HTM, HTML, HTML Tidy, converter, filter, formatter, purifier, sanitizer, XSS, input, PHP, software, code, script, security, cross-site scripting, hack, sanitize, remove, standards, tags, attributes, elements, htmLawed_README.txt, rTxt2htm, PHP Labware" /> +<style type="text/css" media="all"> +<!--/*--><![CDATA[/*><!--*/ +a {text-decoration:none; color: blue;} +a:hover {color: red;} +a:visited {color: blue;} +body {margin: 0; padding: 0;} +body, div, html, p {font-family: Georgia, 'Times new roman', Times;} +code.code {font-family: 'Bitstream vera sans mono', 'Courier New', 'Courier', monospace;} +div.comment {padding: 5px; color: #999999; font-size: 80%;} +div.comment a {color: #6699cc;} +div#body {width: 70%; margin: 5px; padding: 5px;} /* holds non-toc content */ +div#toc {position: fixed; top: 5px; left: 73%; z-index: 2; margin-top: 5px; margin-left: 5px; border: 1px solid gray; padding: 5px; background-color: #ededed; width: 23%; overflow: auto; max-height:94%; font-size: 90%;} /* holds content table (toc) */ +div#top {font-size: 14px; margin: 5px; padding: 5px;} /* holds all content */ +div.monospace {overflow: auto; font-family: 'Bitstream vera sans mono', 'Courier New', 'Courier', monospace;} +div.sub-section {padding-left: 15px;} +div.sub-sub-section {padding-left: 30px;} +h1 {font-size: 22px; margin-top: 5px; margin-bottom: 5px;} +h2 {font-size: 20px; float: left; margin-top: 15px; margin-bottom: 5px;} +h3 {font-size: 18px; float: left; margin-top: 15px; margin-bottom: 5px;} +h4 {font-size: 16px; float: left; margin-top: 15px; margin-bottom: 5px;} +hr {margin-top: 15px; margin-bottom: 5px;} +input, textarea {font-family: 'Bitstream vera sans mono', 'Courier New', 'Courier', monospace;} +p.subtle {color: gray; padding: 0; padding-top: 10px; margin: 0;} +p.subtle a, p.subtle a:visited {color: #6699cc;} +span.item-no {color: black;} +span.subtle {color: gray; margin: 0; padding:0;} +span.subtle a, span.subtle a:visited {color: #6699cc;} +span.term {font-family: 'Bitstream vera sans mono', 'Courier New', 'Courier', monospace;} +span.toc-item {color: black;} +span.totop {float: right; margin-top: 15px; margin-bottom: 5px;} +span.totop a, span.totop a:visited {color: #6699cc;} +@media screen { /* fixes for old IE */ + * html, * html body {overflow-y: auto!important; height: 100%; margin: 0; padding: 0;} + * html div#body {height: 100%; overflow-y: auto; position: relative;} + * html div#toc {position: absolute;} +} +/*]]>*/--> +</style> +<title>htmLawed documentation | htmLawed PHP software is a free, open-source, customizable HTML input purifier and filter</title> +</head> +<body> +<div id="top"> +<h1><a id="peak" name="peak"></a>htmLawed documentation</h1> + +<div id="toc"><span class="toc-item"><a href="#s1"><span class="item-no">1</span>  About htmLawed</a></span><br /> +  <span class="toc-item"><a href="#s1.1"><span class="item-no">1.1</span>  Example uses</a></span><br /> +  <span class="toc-item"><a href="#s1.2"><span class="item-no">1.2</span>  Features</a></span><br /> +  <span class="toc-item"><a href="#s1.3"><span class="item-no">1.3</span>  History</a></span><br /> +  <span class="toc-item"><a href="#s1.4"><span class="item-no">1.4</span>  License & copyright</a></span><br /> +  <span class="toc-item"><a href="#s1.5"><span class="item-no">1.5</span>  Terms used here</a></span><br /> +<span class="toc-item"><a href="#s2"><span class="item-no">2</span>  Usage</a></span><br /> +  <span class="toc-item"><a href="#s2.1"><span class="item-no">2.1</span>  Simple</a></span><br /> +  <span class="toc-item"><a href="#s2.2"><span class="item-no">2.2</span>  Configuring htmLawed using the <span class="term">$config</span> parameter</a></span><br /> +  <span class="toc-item"><a href="#s2.3"><span class="item-no">2.3</span>  Extra HTML specifications using the <span class="term">$spec</span> parameter</a></span><br /> +  <span class="toc-item"><a href="#s2.4"><span class="item-no">2.4</span>  Performance time & memory usage</a></span><br /> +  <span class="toc-item"><a href="#s2.5"><span class="item-no">2.5</span>  Some security risks to keep in mind</a></span><br /> +  <span class="toc-item"><a href="#s2.6"><span class="item-no">2.6</span>  Use without modifying old <span class="term">kses()</span> code</a></span><br /> +  <span class="toc-item"><a href="#s2.7"><span class="item-no">2.7</span>  Tolerance for ill-written HTML</a></span><br /> +  <span class="toc-item"><a href="#s2.8"><span class="item-no">2.8</span>  Limitations & work-arounds</a></span><br /> +  <span class="toc-item"><a href="#s2.9"><span class="item-no">2.9</span>  Examples</a></span><br /> +<span class="toc-item"><a href="#s3"><span class="item-no">3</span>  Details</a></span><br /> +  <span class="toc-item"><a href="#s3.1"><span class="item-no">3.1</span>  Invalid/dangerous characters</a></span><br /> +  <span class="toc-item"><a href="#s3.2"><span class="item-no">3.2</span>  Character references/entities</a></span><br /> +  <span class="toc-item"><a href="#s3.3"><span class="item-no">3.3</span>  HTML elements</a></span><br /> +    <span class="toc-item"><a href="#s3.3.1"><span class="item-no">3.3.1</span>  HTML comments and <span class="term">CDATA</span> sections</a></span><br /> +    <span class="toc-item"><a href="#s3.3.2"><span class="item-no">3.3.2</span>  Tag-transformation for better XHTML-Strict</a></span><br /> +    <span class="toc-item"><a href="#s3.3.3"><span class="item-no">3.3.3</span>  Tag balancing and proper nesting</a></span><br /> +    <span class="toc-item"><a href="#s3.3.4"><span class="item-no">3.3.4</span>  Elements requiring child elements</a></span><br /> +    <span class="toc-item"><a href="#s3.3.5"><span class="item-no">3.3.5</span>  Beautify or compact HTML</a></span><br /> +  <span class="toc-item"><a href="#s3.4"><span class="item-no">3.4</span>  Attributes</a></span><br /> +    <span class="toc-item"><a href="#s3.4.1"><span class="item-no">3.4.1</span>  Auto-addition of XHTML-required attributes</a></span><br /> +    <span class="toc-item"><a href="#s3.4.2"><span class="item-no">3.4.2</span>  Duplicate/invalid <span class="term">id</span> values</a></span><br /> +    <span class="toc-item"><a href="#s3.4.3"><span class="item-no">3.4.3</span>  URL schemes (protocols) and scripts in attribute values</a></span><br /> +    <span class="toc-item"><a href="#s3.4.4"><span class="item-no">3.4.4</span>  Absolute & relative URLs</a></span><br /> +    <span class="toc-item"><a href="#s3.4.5"><span class="item-no">3.4.5</span>  Lower-cased, standard attribute values</a></span><br /> +    <span class="toc-item"><a href="#s3.4.6"><span class="item-no">3.4.6</span>  Transformation of deprecated attributes</a></span><br /> +    <span class="toc-item"><a href="#s3.4.7"><span class="item-no">3.4.7</span>  Anti-spam & <span class="term">href</span></a></span><br /> +    <span class="toc-item"><a href="#s3.4.8"><span class="item-no">3.4.8</span>  Inline style properties</a></span><br /> +    <span class="toc-item"><a href="#s3.4.9"><span class="item-no">3.4.9</span>  Hook function for tag content</a></span><br /> +  <span class="toc-item"><a href="#s3.5"><span class="item-no">3.5</span>  Simple configuration directive for most valid XHTML</a></span><br /> +  <span class="toc-item"><a href="#s3.6"><span class="item-no">3.6</span>  Simple configuration directive for most <em>safe</em> HTML</a></span><br /> +  <span class="toc-item"><a href="#s3.7"><span class="item-no">3.7</span>  Using a hook function</a></span><br /> +  <span class="toc-item"><a href="#s3.8"><span class="item-no">3.8</span>  Obtaining <em>finalized</em> parameter values</a></span><br /> +  <span class="toc-item"><a href="#s3.9"><span class="item-no">3.9</span>  Retaining non-HTML tags in input with mixed markup</a></span><br /> +<span class="toc-item"><a href="#s4"><span class="item-no">4</span>  Other</a></span><br /> +  <span class="toc-item"><a href="#s4.1"><span class="item-no">4.1</span>  Support</a></span><br /> +  <span class="toc-item"><a href="#s4.2"><span class="item-no">4.2</span>  Known issues</a></span><br /> +  <span class="toc-item"><a href="#s4.3"><span class="item-no">4.3</span>  Change-log</a></span><br /> +  <span class="toc-item"><a href="#s4.4"><span class="item-no">4.4</span>  Testing</a></span><br /> +  <span class="toc-item"><a href="#s4.5"><span class="item-no">4.5</span>  Upgrade, & old versions</a></span><br /> +  <span class="toc-item"><a href="#s4.6"><span class="item-no">4.6</span>  Comparison with <span class="term">HTMLPurifier</span></a></span><br /> +  <span class="toc-item"><a href="#s4.7"><span class="item-no">4.7</span>  Use through application plug-ins/modules</a></span><br /> +  <span class="toc-item"><a href="#s4.8"><span class="item-no">4.8</span>  Use in non-PHP applications</a></span><br /> +  <span class="toc-item"><a href="#s4.9"><span class="item-no">4.9</span>  Donate</a></span><br /> +  <span class="toc-item"><a href="#s4.10"><span class="item-no">4.10</span>  Acknowledgements</a></span><br /> +<span class="toc-item"><a href="#s5"><span class="item-no">5</span>  Appendices</a></span><br /> +  <span class="toc-item"><a href="#s5.1"><span class="item-no">5.1</span>  Characters discouraged in HTML</a></span><br /> +  <span class="toc-item"><a href="#s5.2"><span class="item-no">5.2</span>  Valid attribute-element combinations</a></span><br /> +  <span class="toc-item"><a href="#s5.3"><span class="item-no">5.3</span>  CSS 2.1 properties accepting URLs</a></span><br /> +  <span class="toc-item"><a href="#s5.4"><span class="item-no">5.4</span>  Microsoft Windows 1252 character replacements</a></span><br /> +  <span class="toc-item"><a href="#s5.5"><span class="item-no">5.5</span>  URL format</a></span><br /> +  <span class="toc-item"><a href="#s5.6"><span class="item-no">5.6</span>  Brief on htmLawed code</a></span></div><!-- ended div toc --> + +<div id="body"> +<br /> +<div class="comment">htmLawed_README.txt, 16 July 2009<br /> +htmLawed 1.1.8.1, 16 July 2009 <br /> +Copyright Santosh Patnaik<br /> +GPL v3 license<br /> +A PHP Labware internal utility - <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed">http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed</a> </div> +<br /> + +<div class="section"><h2> +<a name="s1" id="s1"></a><span class="item-no">1</span>  About htmLawed +</h2><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed is a highly customizable single-file PHP script to make text secure, and standard- and admin policy-compliant for use in the body of HTML 4, XHTML 1 or 1.1, or generic XML documents. It is thus a configurable input (X)HTML filter, processor, purifier, sanitizer, beautifier, etc., and an alternative to the <a href="http://tidy.sourceforge.net">HTMLTidy</a> application.<br /> +<br /> +  The <em>lawing in</em> of input text is needed to ensure that HTML code in the text is standard-compliant, does not introduce security vulnerabilities, and does not break the aesthetics, design or layout of web-pages. htmLawed tries to do this by, for example, making HTML well-formed with balanced and properly nested tags, neutralizing code that may be used for cross-site scripting (<span class="term">XSS</span>) attacks, and allowing only specified HTML elements/tags and attributes.<br /> + +<div class="sub-section"><h3> +<a name="s1.1" id="s1.1"></a><span class="item-no">1.1</span>  Example uses +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  *  Filtering of text submitted as comments on blogs to allow only certain HTML elements<br /> +<br /> +  *  Making RSS/Atom newsfeed item-content standard-compliant: often one uses an excerpt from an HTML document for the content, and with unbalanced tags, non-numerical entities, etc., such excerpts may not be XML-compliant<br /> +<br /> +  *  Text processing for stricter XML standard-compliance: e.g., to have lowercased <span class="term">x</span> in hexadecimal numeric entities becomes necessary if an XHTML document with MathML content needs to be served as <span class="term">application/xml</span><br /> +<br /> +  *  Scraping text or data from web-pages<br /> +<br /> +  *  Pretty-printing HTML code<br /> + +</div> +<div class="sub-section"><h3> +<a name="s1.2" id="s1.2"></a><span class="item-no">1.2</span>  Features +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Key: <span class="term">*</span> security feature, <span class="term">^</span> standard compliance, <span class="term">~</span> requires setting right options, <span class="term">`</span> different from <span class="term">Kses</span><br /> +<br /> +  *  make input more <strong>secure</strong> and <strong>standard-compliant</strong><br /> +  *  use for HTML 4, XHTML 1.0 or 1.1, or even generic <strong>XML</strong> documents  ^~`<br /> +<br /> +  *  <strong>beautify</strong> or <strong>compact</strong> HTML  ^~`<br /> +<br /> +  *  <strong>restrict elements</strong>  ^~`<br /> +  *  proper closure of empty elements like <span class="term">img</span>  ^`<br /> +  *  <strong>transform deprecated elements</strong> like <span class="term">u</span>  ^~`<br /> +  *  HTML <strong>comments</strong> and <span class="term">CDATA</span> sections can be permitted  ^~`<br /> +  *  elements like <span class="term">script</span>, <span class="term">object</span> and <span class="term">form</span> can be permitted  ~<br /> +<br /> +  *  <strong>restrict attributes</strong>, including <strong>element-specifically</strong>  ^~`<br /> +  *  remove <strong>invalid attributes</strong>  ^`<br /> +  *  element and attribute names are <strong>lower-cased</strong>  ^<br /> +  *  provide <strong>required attributes</strong>, like <span class="term">alt</span> for <span class="term">image</span>  ^`<br /> +  *  <strong>transform deprecated attributes</strong>  ^~`<br /> +  *  attributes <strong>declared only once</strong>  ^`<br /> +<br /> +  *  <strong>restrict attribute values</strong>, including <strong>element-specifically</strong>  ^~`<br /> +  *  a value is declared for <em>empty</em> (<em>minimized</em>) attributes like <span class="term">checked</span>  ^<br /> +  *  check for potentially dangerous attribute values  *~<br /> +  *  ensure <strong>unique</strong> <span class="term">id</span> attribute values  ^~`<br /> +  *  <strong>double-quote</strong> attribute values  ^<br /> +  *  lower-case <strong>standard attribute values</strong> like <span class="term">password</span>  ^`<br /> +<br /> +  *  <strong>attribute-specific URL protocol/scheme restriction</strong>  *~`<br /> +  *  disable <strong>dynamic expressions</strong> in <span class="term">style</span> values  *~`<br /> +<br /> +  *  neutralize invalid named character entities  ^`<br /> +  *  <strong>convert</strong> hexadecimal numeric entities to decimal ones, or vice versa  ^~`<br /> +  *  convert named entities to numeric ones for generic XML use  ^~`<br /> +<br /> +  *  remove <strong>null</strong> characters  *<br /> +  *  neutralize potentially dangerous proprietary Netscape <strong>Javascript entities</strong>  *<br /> +  *  replace potentially dangerous <strong>soft-hyphen</strong> character in attribute values with spaces  *<br /> +<br /> +  *  remove common <strong>invalid characters</strong> not allowed in HTML or XML  ^`<br /> +  *  replace <strong>characters from Microsoft applications</strong> like <span class="term">Word</span> that are discouraged in HTML or XML  ^~`<br /> +  *  neutralize entities for characters invalid or discouraged in HTML or XML  ^`<br /> +  *  appropriately neutralize <span class="term"><</span>, <span class="term">&</span>, <span class="term">"</span>, and <span class="term">></span> characters  ^*`<br /> +<br /> +  *  understands improperly spaced tag content (like, spread over more than a line) and properly spaces them  `<br /> +  *  attempts to <strong>balance tags</strong> for well-formedness  ^~`<br /> +  *  understands when <strong>omitable closing tags</strong> like <span class="term"></p></span> (allowed in HTML 4, transitional, e.g.) are missing  ^~`<br /> +  *  attempts to permit only <strong>validly nested tags</strong>  ^~`<br /> +  *  option to <strong>remove or neutralize bad content</strong> ^~`<br /> +  *  attempts to <strong>rectify common errors of plain-text misplacement</strong> (e.g., directly inside <span class="term">blockquote</span>) ^~`<br /> +<br /> +  *  fast, <strong>non-OOP</strong> code of ~45 kb incurring peak basal memory usage of ~0.5 MB<br /> +  *  <strong>compatible</strong> with pre-existing code using <span class="term">Kses</span> (the filter used by <span class="term">WordPress</span>)<br /> +<br /> +  *  optional <strong>anti-spam</strong> measures such as addition of <span class="term">rel="nofollow"</span> and link-disabling  ~`<br /> +  *  optionally makes <strong>relative URLs absolute</strong>, and vice versa  ~`<br /> +<br /> +  *  optionally mark <span class="term">&</span> to identify the entities for <span class="term">&</span>, <span class="term"><</span> and <span class="term">></span> introduced by htmLawed  ~`<br /> +<br /> +  *  allows deployment of powerful <strong>hook functions</strong> to <strong>inject</strong> HTML, <strong>consolidate</strong> <span class="term">style</span> attributes to <span class="term">class</span>, finely check attribute values, etc.  ~`<br /> +<br /> +  *  <strong>independent of character encoding</strong> of input and does not affect it<br /> +<br /> +  *  <strong>tolerance for ill-written HTML</strong> to a certain degree<br /> + +</div> +<div class="sub-section"><h3> +<a name="s1.3" id="s1.3"></a><span class="item-no">1.3</span>  History +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed was developed for use with <span class="term">LabWiki</span>, a wiki software developed at PHP Labware, as a suitable software could not be found. Existing PHP software like <span class="term">Kses</span> and <span class="term">HTMLPurifier</span> were deemed inadequate, slow, resource-intensive, or dependent on external applications like <span class="term">HTML Tidy</span>.<br /> +<br /> +  htmLawed started as a modification of Ulf Harnhammar's <span class="term">Kses</span> (version 0.2.2) software, and is compatible with code that uses <span class="term">Kses</span>; see <a href="#s2.6">section 2.6</a>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s1.4" id="s1.4"></a><span class="item-no">1.4</span>  License & copyright +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed is free and open-source software licensed under GPL license version <a href="http://www.gnu.org/licenses/gpl-3.0.txt">3</a>, and copyrighted by Santosh Patnaik, MD, PhD.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s1.5" id="s1.5"></a><span class="item-no">1.5</span>  Terms used here +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  *  <em>administrator</em> - or admin; person setting up the code to pass input through htmLawed; also, <em>user</em><br /> +  *  <em>attributes</em> - name-value pairs like <span class="term">href="http://x.com"</span> in opening tags<br /> +  *  <em>author</em> - <em>writer</em><br /> +  *  <em>character</em> - atomic unit of text; internally represented by a numeric <em>code-point</em> as specified by the <em>encoding</em> or <em>charset</em> in use<br /> +  *  <em>entity</em> - markup like <span class="term">&gt;</span> and <span class="term">&#160;</span> used to refer to a character<br /> +  *  <em>element</em> - HTML element like <span class="term">a</span> and <span class="term">img</span><br /> +  *  <em>element content</em> -  content between the opening and closing tags of an element, like <span class="term">click</span> of <span class="term"><a href="x">click</a></span><br /> +  *  <em>HTML</em> - implies XHTML unless specified otherwise<br /> +  *  <em>input</em> - text string given to htmLawed to process<br /> +  *  <em>processing</em> - involves filtering, correction, etc., of input<br /> +  *  <em>safe</em> - absence or reduction of certain characters and HTML elements and attributes in the input that can otherwise potentially and circumstantially expose web-site users to security vulnerabilities like cross-site scripting attacks (XSS)<br /> +  *  <em>scheme</em> - URL protocol like <span class="term">http</span> and <span class="term">ftp</span><br /> +  *  <em>specs</em> - standard specifications<br /> +  *  <em>style property</em> - terms like <span class="term">border</span> and <span class="term">height</span> for which declarations are made in values for the <span class="term">style</span> attribute of elements<br /> +  *  <em>tag</em> - markers like <span class="term"><a href="x"></span> and <span class="term"></a></span> delineating element content; the opening tag can contain attributes<br /> +  *  <em>tag content</em> - consists of tag markers <span class="term"><</span> and <span class="term">></span>, element names like <span class="term">div</span>, and possibly attributes<br /> +  *  <em>user</em> - administrator<br /> +  *  <em>writer</em> - end-user like a blog commenter providing the input that is to be processed; also, <em>author</em><br /> + +</div> +</div> +<div class="section"><h2> +<a name="s2" id="s2"></a><span class="item-no">2</span>  Usage +</h2><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed should work with PHP 4.3 and higher. Either <span class="term">include()</span> the <span class="term">htmLawed.php</span> file or copy-paste the entire code.<br /> +<br /> +  To easily <strong>test</strong> htmLawed using a form-based interface, use the provided <a href="htmLawedTest.php">demo</a> (<span class="term">htmLawed.php</span> and <span class="term">htmLawedTest.php</span> should be in the same directory on the web-server).<br /> + +<div class="sub-section"><h3> +<a name="s2.1" id="s2.1"></a><span class="item-no">2.1</span>  Simple +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  The input text to be processed, <span class="term">$text</span>, is passed as an argument of type string; <span class="term">htmLawed()</span> returns the processed string:<br /> +<br /> + +<code class="code">    $processed = htmLawed($text);</code> +<br /> +<br /> +  <strong>Note</strong>: If input is from a <span class="term">$_GET</span> or <span class="term">$_POST</span> value, and <span class="term">magic quotes</span> are enabled on the PHP setup, run <span class="term">stripslashes()</span> on the input before passing to htmLawed.<br /> +<br /> +  By default, htmLawed will process the text allowing all valid HTML elements/tags, secure URL scheme/CSS style properties, etc. It will allow <span class="term">CDATA</span> sections and HTML comments, balance tags, and ensure proper nesting of elements. Such actions can be configured using two other optional arguments -- <span class="term">$config</span> and <span class="term">$spec</span>:<br /> +<br /> + +<code class="code">    $processed = htmLawed($text, $config, $spec);</code> +<br /> +<br /> +  These extra parameters are detailed below. Some examples are shown in <a href="#s2.9">section 2.9</a>.<br /> +<br /> +  <strong>Note</strong>: For maximum protection against <span class="term">XSS</span> and other scripting attacks (e.g., by disallowing Javascript code), consider using the <span class="term">safe</span> parameter; see <a href="#s3.6">section 3.6</a>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.2" id="s2.2"></a><span class="item-no">2.2</span>  Configuring htmLawed using the <span class="term">$config</span> parameter +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  <span class="term">$config</span> instructs htmLawed on how to tackle certain tasks. When <span class="term">$config</span> is not specified, or not set as an array (e.g., <span class="term">$config = 1</span>), htmLawed will take default actions. One or many of the task-action or value-specification pairs can be specified in <span class="term">$config</span> as array key-value pairs. If a parameter is not specified, htmLawed will use the default value/action indicated further below.<br /> +<br /> + +<code class="code">    $config = array('comment'=>0, 'cdata'=>1);</code> +<br /> + +<code class="code">    $processed = htmLawed($text, $config);</code> +<br /> +<br /> +  Or,<br /> +<br /> + +<code class="code">    $processed = htmLawed($text, array('comment'=>0, 'cdata'=>1));</code> +<br /> +<br /> +  Below are the possible value-specification combinations. In PHP code, values that are integers should not be quoted and should be used as numeric types (unless meant as string/text).<br /> +<br /> +  Key: <span class="term">*</span> default, <span class="term">^</span> different default when htmLawed is used in the Kses-compatible mode (see <a href="#s2.6">section 2.6</a>), <span class="term">~</span> different default when <span class="term">valid_xhtml</span> is set to <span class="term">1</span> (see <a href="#s3.5">section 3.5</a>), <span class="term">"</span> different default when <span class="term">safe</span> is set to <span class="term">1</span> (see <a href="#s3.6">section 3.6</a>)<br /> +<br /> +  <strong>abs_url</strong><br /> +  Make URLs absolute or relative; <span class="term">$config["base_url"]</span> needs to be set; see <a href="#s3.4.4">section 3.4.4</a><br /> +<br /> +  <span class="term">-1</span> - make relative<br /> +  <span class="term">0</span> - no action  *<br /> +  <span class="term">1</span> - make absolute<br /> +<br /> +  <strong>and_mark</strong><br /> +  Mark <span class="term">&</span> characters in the original input; see <a href="#s3.2">section 3.2</a><br /> +<br /> +  <strong>anti_link_spam</strong><br /> +  Anti-link-spam measure; see <a href="#s3.4.7">section 3.4.7</a><br /> +<br /> +  <span class="term">0</span> - no measure taken  *<br /> +  <span class="term">array("regex1", "regex2")</span> - will ensure a <span class="term">rel</span> attribute with <span class="term">nofollow</span> in its value in case the <span class="term">href</span> attribute value matches the regular expression pattern <span class="term">regex1</span>, and/or will remove <span class="term">href</span> if its value matches the regular expression pattern <span class="term">regex2</span>. E.g., <span class="term">array("/./", "/://\W*(?!(abc\.com|xyz\.org))/")</span>; see <a href="#s3.4.7">section 3.4.7</a> for more.<br /> +<br /> +  <strong>anti_mail_spam</strong><br /> +  Anti-mail-spam measure; see <a href="#s3.4.7">section 3.4.7</a><br /> +<br /> +  <span class="term">0</span> - no measure taken  *<br /> +  <span class="term">word</span> - <span class="term">@</span> in mail address in <span class="term">href</span> attribute value is replaced with specified <span class="term">word</span><br /> +<br /> +  <strong>balance</strong><br /> +  Balance tags for well-formedness and proper nesting; see <a href="#s3.3.3">section 3.3.3</a><br /> +<br /> +  <span class="term">0</span> - no<br /> +  <span class="term">1</span> - yes  *<br /> +<br /> +  <strong>base_url</strong><br /> +  Base URL value that needs to be set if <span class="term">$config["abs_url"]</span> is not <span class="term">0</span>; see <a href="#s3.4.4">section 3.4.4</a><br /> +<br /> +  <strong>cdata</strong><br /> +  Handling of <span class="term">CDATA</span> sections; see <a href="#s3.3.1">section 3.3.1</a><br /> +<br /> +  <span class="term">0</span> - don't consider <span class="term">CDATA</span> sections as markup and proceed as if plain text  ^"<br /> +  <span class="term">1</span> - remove<br /> +  <span class="term">2</span> - allow, but neutralize any <span class="term"><</span>, <span class="term">></span>, and <span class="term">&</span> inside by converting them to named entities<br /> +  <span class="term">3</span> - allow  *<br /> +<br /> +  <strong>clean_ms_char</strong><br /> +  Replace discouraged characters introduced by Microsoft Word, etc.; see <a href="#s3.1">section 3.1</a><br /> +<br /> +  <span class="term">0</span> - no  *<br /> +  <span class="term">1</span> - yes<br /> +  <span class="term">2</span> - yes, but replace special single & double quotes with ordinary ones<br /> +<br /> +  <strong>comment</strong><br /> +  Handling of HTML comments; see <a href="#s3.3.1">section 3.3.1</a><br /> +<br /> +  <span class="term">0</span> - don't consider comments as markup and proceed as if plain text  ^"<br /> +  <span class="term">1</span> - remove<br /> +  <span class="term">2</span> - allow, but neutralize any <span class="term"><</span>, <span class="term">></span>, and <span class="term">&</span> inside by converting to named entities<br /> +  <span class="term">3</span> - allow  *<br /> +<br /> +  <strong>css_expression</strong><br /> +  Allow dynamic CSS expression by not removing the expression from CSS property values in <span class="term">style</span> attributes; see <a href="#s3.4.8">section 3.4.8</a><br /> +<br /> +  <span class="term">0</span> - remove  *<br /> +  <span class="term">1</span> - allow<br /> +<br /> +  <strong>deny_attribute</strong><br /> +  Denied HTML attributes; see <a href="#s3.4">section 3.4</a><br /> +<br /> +  <span class="term">0</span> - none  *<br /> +  <span class="term">string</span> - dictated by values in <span class="term">string</span><br /> +  <span class="term">on*</span> (like <span class="term">onfocus</span>) attributes not allowed - "<br /> +<br /> +  <strong>elements</strong><br /> +  Allowed HTML elements; see <a href="#s3.3">section 3.3</a><br /> +<br /> +  <span class="term">* -center -dir -font -isindex -menu -s -strike -u</span> -  ~<br /> +  <span class="term">applet, embed, iframe, object, script</span> not allowed - "<br /> +<br /> +  <strong>hexdec_entity</strong><br /> +  Allow hexadecimal numeric entities and do not convert to the more widely accepted decimal ones, or convert decimal to hexadecimal ones; see <a href="#s3.2">section 3.2</a><br /> +<br /> +  <span class="term">0</span> - no<br /> +  <span class="term">1</span> - yes  *<br /> +  <span class="term">2</span> - convert decimal to hexadecimal ones<br /> +<br /> +  <strong>hook</strong><br /> +  Name of an optional hook function to alter the input string, <span class="term">$config</span> or <span class="term">$spec</span> before htmLawed starts its main work; see <a href="#s3.7">section 3.7</a><br /> +<br /> +  <span class="term">0</span> - no hook function  *<br /> +  <span class="term">name</span> - <span class="term">name</span> is name of the hook function (<span class="term">kses_hook</span>  ^)<br /> +<br /> +  <strong>hook_tag</strong><br /> +  Name of an optional hook function to alter tag content finalized by htmLawed; see <a href="#s3.4.9">section 3.4.9</a><br /> +<br /> +  <span class="term">0</span> - no hook function  *<br /> +  <span class="term">name</span> - <span class="term">name</span> is name of the hook function<br /> +<br /> +  <strong>keep_bad</strong><br /> +  Neutralize bad tags by converting <span class="term"><</span> and <span class="term">></span> to entities, or remove them; see <a href="#s3.3.3">section 3.3.3</a><br /> +<br /> +  <span class="term">0</span> - remove  ^<br /> +  <span class="term">1</span> - neutralize both tags and element content<br /> +  <span class="term">2</span> - remove tags but neutralize element content<br /> +  <span class="term">3</span> and <span class="term">4</span> - like <span class="term">1</span> and <span class="term">2</span> but remove if text (<span class="term">pcdata</span>) is invalid in parent element<br /> +  <span class="term">5</span> and <span class="term">6</span> * -  like <span class="term">3</span> and <span class="term">4</span> but line-breaks, tabs and spaces are left<br /> +<br /> +  <strong>lc_std_val</strong><br /> +  For XHTML compliance, predefined, standard attribute values, like <span class="term">get</span> for the <span class="term">method</span> attribute of <span class="term">form</span>, must be lowercased; see <a href="#s3.4.5">section 3.4.5</a><br /> +<br /> +  <span class="term">0</span> - no<br /> +  <span class="term">1</span> - yes  *<br /> +<br /> +  <strong>make_tag_strict</strong><br /> +  Transform/remove these non-strict XHTML elements, even if they are allowed by the admin: <span class="term">applet</span> <span class="term">center</span> <span class="term">dir</span> <span class="term">embed</span> <span class="term">font</span> <span class="term">isindex</span> <span class="term">menu</span> <span class="term">s</span> <span class="term">strike</span> <span class="term">u</span>; see <a href="#s3.3.2">section 3.3.2</a><br /> +<br /> +  <span class="term">0</span> - no  ^<br /> +  <span class="term">1</span> - yes, but leave <span class="term">applet</span>, <span class="term">embed</span> and <span class="term">isindex</span> elements that currently can't be transformed  *<br /> +  <span class="term">2</span> - yes, removing <span class="term">applet</span>, <span class="term">embed</span> and <span class="term">isindex</span> elements and their contents (nested elements remain)  ~<br /> +<br /> +  <strong>named_entity</strong><br /> +  Allow non-universal named HTML entities, or convert to numeric ones; see <a href="#s3.2">section 3.2</a><br /> +<br /> +  <span class="term">0</span> - convert<br /> +  <span class="term">1</span> - allow  *<br /> +<br /> +  <strong>no_deprecated_attr</strong><br /> +  Allow deprecated attributes or transform them; see <a href="#s3.4.6">section 3.4.6</a><br /> +<br /> +  <span class="term">0</span> - allow  ^<br /> +  <span class="term">1</span> - transform, but <span class="term">name</span> attributes for <span class="term">a</span> and <span class="term">map</span> are retained  *<br /> +  <span class="term">2</span> - transform<br /> +<br /> +  <strong>parent</strong><br /> +  Name of the parent element, possibly imagined, that will hold the input; see <a href="#s3.3">section 3.3</a><br /> +<br /> +  <strong>safe</strong><br /> +  Magic parameter to make input the most secure against XSS without needing to specify other relevant <span class="term">$config</span> parameters; see <a href="#s3.6">section 3.6</a><br /> +<br /> +  <span class="term">0</span> - no  *<br /> +  <span class="term">1</span> - will auto-adjust other relevant <span class="term">$config</span> parameters (indicated by <span class="term">"</span> in this list)<br /> +<br /> +  <strong>schemes</strong><br /> +  Array of attribute-specific, comma-separated, lower-cased list of schemes (protocols) allowed in attributes accepting URLs; <span class="term">*</span> covers all unspecified attributes; see <a href="#s3.4.3">section 3.4.3</a><br /> +<br /> +  <span class="term">href: aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet; *:file, http, https</span>  *<br /> +  <span class="term">*: ftp, gopher, http, https, mailto, news, nntp, telnet</span>  ^<br /> +  <span class="term">href: aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet; style: nil; *:file, http, https</span>  "<br /> +<br /> +  <strong>show_setting</strong><br /> +  Name of a PHP variable to assign the <em>finalized</em> <span class="term">$config</span> and <span class="term">$spec</span> values; see <a href="#s3.8">section 3.8</a><br /> +<br /> +  <strong>style_pass</strong><br /> +  Do not look at <span class="term">style</span> attribute values, letting them through without any alteration<br /> +<br /> +  <span class="term">0</span> - no *<br /> +  <span class="term">1</span> - htmLawed will let through any <span class="term">style</span> value; see <a href="#s3.4.8">section 3.4.8</a><br /> +<br /> +  <strong>tidy</strong><br /> +  Beautify or compact HTML code; see <a href="#s3.3.5">section 3.3.5</a><br /> +<br /> +  <span class="term">-1</span> - compact<br /> +  <span class="term">0</span> - no  *<br /> +  <span class="term">1</span> or <span class="term">string</span> - beautify (custom format specified by <span class="term">string</span>)<br /> +<br /> +  <strong>unique_ids</strong><br /> +  <span class="term">id</span> attribute value checks; see <a href="#s3.4.2">section 3.4.2</a><br /> +<br /> +  <span class="term">0</span> - no  ^<br /> +  <span class="term">1</span> - remove duplicate and/or invalid ones  *<br /> +  <span class="term">word</span> - remove invalid ones and replace duplicate ones with new and unique ones based on the <span class="term">word</span>; the admin-specified <span class="term">word</span>, like <span class="term">my_</span>, should begin with a letter (a-z) and can contain letters, digits, <span class="term">.</span>, <span class="term">_</span>, <span class="term">-</span>, and <span class="term">:</span>.<br /> +<br /> +  <strong>valid_xhtml</strong><br /> +  Magic parameter to make input the most valid XHTML without needing to specify other relevant <span class="term">$config</span> parameters; see <a href="#s3.5">section 3.5</a><br /> +<br /> +  <span class="term">0</span> - no  *<br /> +  <span class="term">1</span> - will auto-adjust other relevant <span class="term">$config</span> parameters (indicated by <span class="term">~</span> in this list)<br /> +<br /> +  <strong>xml:lang</strong><br /> +  Auto-adding <span class="term">xml:lang</span> attribute; see <a href="#s3.4.1">section 3.4.1</a><br /> +<br /> +  <span class="term">0</span> - no  *<br /> +  <span class="term">1</span> - add if <span class="term">lang</span> attribute is present<br /> +  <span class="term">2</span> - add if <span class="term">lang</span> attribute is present, and remove <span class="term">lang</span>  ~<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.3" id="s2.3"></a><span class="item-no">2.3</span>  Extra HTML specifications using the $spec parameter +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  The <span class="term">$spec</span> argument can be used to disallow an otherwise legal attribute for an element, or to restrict the attribute's values. This can also be helpful as a security measure (e.g., in certain versions of browsers, certain values can cause buffer overflows and denial of service attacks), or in enforcing admin policy compliance. <span class="term">$spec</span> is specified as a string of text containing one or more <em>rules</em>, with multiple rules separated from each other by a semi-colon (<span class="term">;</span>). E.g.,<br /> +<br /> + +<code class="code">    $spec = 'i=-*; td, tr=style, id, -*; a=id(match="/[a-z][a-z\d.:\-`"]*/i"/minval=2), href(maxlen=100/minlen=34); img=-width,-alt';</code> +<br /> + +<code class="code">    $processed = htmLawed($text, $config, $spec);</code> +<br /> +<br /> +  Or,<br /> +<br /> + +<code class="code">    $processed = htmLawed($text, $config, 'i=-*; td, tr=style, id, -*; a=id(match="/[a-z][a-z\d.:\-`"]*/i"/minval=2), href(maxlen=100/minlen=34); img=-width,-alt');</code> +<br /> +<br /> +  A rule begins with an HTML <strong>element</strong> name(s) (<em>rule-element</em>), for which the rule applies, followed by an equal (<span class="term">=</span>) sign. A rule-element may represent multiple elements if comma (,)-separated element names are used. E.g., <span class="term">th,td,tr=</span>.<br /> +<br /> +  Rest of the rule consists of comma-separated HTML <strong>attribute names</strong>. A minus (<span class="term">-</span>) character before an attribute means that the attribute is not permitted inside the rule-element. E.g., <span class="term">-width</span>. To deny all attributes, <span class="term">-*</span> can be used.<br /> +<br /> +  Following shows examples of rule excerpts with rule-element <span class="term">a</span> and the attributes that are being permitted:<br /> +<br /> +  *  <span class="term">a=</span> - all<br /> +  *  <span class="term">a=id</span> - all<br /> +  *  <span class="term">a=href, title, -id, -onclick</span> - all except <span class="term">id</span> and <span class="term">onclick</span><br /> +  *  <span class="term">a=*, id, -id</span> - all except <span class="term">id</span><br /> +  *  <span class="term">a=-*</span> - none<br /> +  *  <span class="term">a=-*, href, title</span> - none except <span class="term">href</span> and <span class="term">title</span><br /> +  *  <span class="term">a=-*, -id, href, title</span> - none except <span class="term">href</span> and <span class="term">title</span><br /> +<br /> +  Rules regarding <strong>attribute values</strong> are optionally specified inside round brackets after attribute names in slash ('/')-separated <em>parameter = value</em> pairs. E.g., <span class="term">title(maxlen=30/minlen=5)</span>. None, or one or more of the following parameters may be specified:<br /> +<br /> +  *  <span class="term">oneof</span> - one or more choices separated by <span class="term">|</span> that the value should match; if only one choice is provided, then the value must match that choice<br /> +<br /> +  *  <span class="term">noneof</span> - one or more choices separated by <span class="term">|</span> that the value should not match<br /> +<br /> +  *  <span class="term">maxlen</span> and <span class="term">minlen</span> - upper and lower limits for the number of characters in the attribute value; specified in numbers<br /> +<br /> +  *  <span class="term">maxval</span> and <span class="term">minval</span> - upper and lower limits for the numerical value specified in the attribute value; specified in numbers<br /> +<br /> +  *  <span class="term">match</span> and <span class="term">nomatch</span> - pattern that the attribute value should or should not match; specified as PHP/PCRE-compatible regular expressions with delimiters and possibly modifiers<br /> +<br /> +  *  <span class="term">default</span> - a value to force on the attribute if the value provided by the writer does not fit any of the specified parameters<br /> +<br /> +  If <span class="term">default</span> is not set and the attribute value does not satisfy any of the specified parameters, then the attribute is removed. The <span class="term">default</span> value can also be used to force all attribute declarations to take the same value (by getting the values declared illegal by setting, e.g., <span class="term">maxlen</span> to <span class="term">-1</span>).<br /> +<br /> +  Examples with <em>input</em> <span class="term"><input title="WIDTH" value="10em" /><input title="length" value="5" /></span> are shown below.<br /> +<br /> +  <em>Rule</em>: <span class="term">input=title(maxlen=60/minlen=6), value</span><br /> +  <em>Output</em>: <span class="term"><input value="10em" /><input title="length" value="5" /></span><br /> +<br /> +  <em>Rule</em>: <span class="term">input=title(), value(maxval=8/default=6)</span><br /> +  <em>Output</em>: <span class="term"><input title="WIDTH" value="6" /><input title="length" value="5" /></span><br /> +<br /> +  <em>Rule</em>: <span class="term">input=title(nomatch=$w.d$i), value(match=$em$/default=6em)</span><br /> +  <em>Output</em>: <span class="term"><input value="10em" /><input title="length" value="6em" /></span><br /> +<br /> +  <em>Rule</em>: <span class="term">input=title(oneof=height|depth/default=depth), value(noneof=5|6)</span><br /> +  <em>Output</em>: <span class="term"><input title="depth" value="10em" /><input title="depth" /></span><br /> +<br /> +  <strong>Special characters</strong>: The characters <span class="term">;</span>, <span class="term">,</span>, <span class="term">/</span>, <span class="term">(</span>, <span class="term">)</span>, <span class="term">|</span>, <span class="term">~</span> and space have special meanings in the rules. Words in the rules that use such characters, or the characters themselves, should be <em>escaped</em> by enclosing in pairs of double-quotes (<span class="term">"</span>). A back-tick (<span class="term">`</span>) can be used to escape a literal <span class="term">"</span>. An example rule illustrating this is <span class="term">input=value(maxlen=30/match="/^\w/"/default="your `"ID`"")</span>.<br /> +<br /> +  <strong>Note</strong>: To deny an attribute for all elements for which it is legal, <span class="term">$config["deny_attribute"]</span> (see <a href="#s3.4">section 3.4</a>) can be used instead of <span class="term">$spec</span>. Also, attributes can be allowed element-specifically through <span class="term">$spec</span> while being denied globally through <span class="term">$config["deny_attribute"]</span>. The <span class="term">hook_tag</span> parameter (<a href="#s3.4.9">section 3.4.9</a>) can also be used to implement the <span class="term">$spec</span> functionality.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.4" id="s2.4"></a><span class="item-no">2.4</span>  Performance time & memory usage +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  The time and memory used by htmLawed depends on its configuration and the size of the input, and the amount, nestedness and well-formedness of the HTML markup within it. In particular, tag balancing and beautification each can increase the processing time by about a quarter.<br /> +<br /> +  The htmLawed <a href="htmLawedTest.php">demo</a> can be used to evaluate the performance and effects of different types of input and <span class="term">$config</span>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.5" id="s2.5"></a><span class="item-no">2.5</span>  Some security risks to keep in mind +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  When setting the parameters/arguments (like those to allow certain HTML elements) for use with htmLawed, one should bear in mind that the setting may let through potentially <em>dangerous</em> HTML code. (This may not be a problem if the authors are trusted.)<br /> +<br /> +  For example, following increase security risks:<br /> +<br /> +  *  Allowing <span class="term">script</span>, <span class="term">applet</span>, <span class="term">embed</span>, <span class="term">iframe</span> or <span class="term">object</span> elements, or certain of their attributes like <span class="term">allowscriptaccess</span><br /> +<br /> +  *  Allowing HTML comments (some Internet Explorer versions are vulnerable with, e.g., <span class="term"><!--[if gte IE 4]><script>alert("xss");</script><![endif]--></span><br /> +<br /> +  *  Allowing dynamic CSS expressions (a feature of the IE browser)<br /> +<br /> +  <em>Unsafe</em> HTML can be removed by setting <span class="term">$config</span> appropriately. E.g., <span class="term">$config["elements"] = "* -script"</span> (<a href="#s3.3">section 3.3</a>), <span class="term">$config["safe"] = 1</span> (<a href="#s3.6">section 3.6</a>), etc.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.6" id="s2.6"></a><span class="item-no">2.6</span>  Use without modifying old <span class="term">kses()</span> code +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  The <span class="term">Kses</span> PHP script is used by many applications (like <span class="term">WordPress</span>). It is possible to have such applications use htmLawed instead, since it is compatible with code that calls the <span class="term">kses()</span> function declared in the <span class="term">Kses</span> file (usually named <span class="term">kses.php</span>). E.g., application code like this will continue to work after replacing <span class="term">Kses</span> with htmLawed:<br /> +<br /> + +<code class="code">    $comment_filtered = kses($comment_input, array('a'=>array(), 'b'=>array(), 'i'=>array()));</code> +<br /> +<br /> +  For some of the <span class="term">$config</span> parameters, htmLawed will use values other than the default ones. These are indicated by <span class="term">^</span> in <a href="#s2.2">section 2.2</a>. To force htmLawed to use other values, function <span class="term">kses()</span> in the htmLawed code should be edited -- a few configurable parameters/variables need to be changed.<br /> +<br /> +  If the application uses a <span class="term">Kses</span> file that has the <span class="term">kses()</span> function declared, then, to have the application use htmLawed instead of <span class="term">Kses</span>, simply rename <span class="term">htmLawed.php</span> (to <span class="term">kses.php</span>, e.g.) and replace the <span class="term">Kses</span> file (or just replace the code in the <span class="term">Kses</span> file with the htmLawed code). If the <span class="term">kses()</span> function in the <span class="term">Kses</span> file had been renamed by the application developer (e.g., in <span class="term">WordPress</span>, it is named <span class="term">wp_kses()</span>), then appropriately rename the <span class="term">kses()</span> function in the htmLawed code.<br /> +<br /> +  If the <span class="term">Kses</span> file used by the application has been highly altered by the application developers, then one may need a different approach. E.g., with <span class="term">WordPress</span>, it is best to copy the htmLawed code to <span class="term">wp_includes/kses.php</span>, rename the newly added function <span class="term">kses()</span> to <span class="term">wp_kses()</span>, and delete the code for the original <span class="term">wp_kses()</span> function.<br /> +<br /> +  If the <span class="term">Kses</span> code has a non-empty hook function (e.g., <span class="term">wp_kses_hook()</span> in case of <span class="term">WordPress</span>), then the code for htmLawed's <span class="term">kses_hook()</span> function should be appropriately edited. However, the requirement of the hook function should be re-evaluated considering that htmLawed has extra capabilities. With <span class="term">WordPress</span>, the hook function is an essential one. The following code is suggested for the htmLawed <span class="term">kses_hook()</span> in case of <span class="term">WordPress</span>:<br /> +<br /> + +<code class="code">    function kses_hook($string, &$cf, &$spec){</code> +<br /> + +<code class="code">    // kses compatibility</code> +<br /> + +<code class="code">    $allowed_html = $spec;</code> +<br /> + +<code class="code">    $allowed_protocols = array();</code> +<br /> + +<code class="code">    foreach($cf['schemes'] as $v){</code> +<br /> + +<code class="code">     foreach($v as $k2=>$v2){</code> +<br /> + +<code class="code">      if(!in_array($k2, $allowed_protocols)){</code> +<br /> + +<code class="code">       $allowed_protocols[] = $k2;</code> +<br /> + +<code class="code">      }</code> +<br /> + +<code class="code">     }</code> +<br /> + +<code class="code">    }</code> +<br /> + +<code class="code">    return wp_kses_hook($string, $allowed_html, $allowed_protocols);</code> +<br /> + +<code class="code">    // eof</code> +<br /> + +<code class="code">    }</code> +<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.7" id="s2.7"></a><span class="item-no">2.7</span>  Tolerance for ill-written HTML +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed can work with ill-written HTML code in the input. However, HTML that is too ill-written may not be <em>read</em> as HTML, and be considered mere plain text instead. Following statements indicate the degree of <em>looseness</em> that htmLawed can work with, and can be provided in instructions to writers:<br /> +<br /> +  *  Tags must be flanked by <span class="term"><</span> and <span class="term">></span> with no <span class="term">></span> inside -- any needed <span class="term">></span> should be put in as <span class="term">&gt;</span>. It is possible for tag content (element name and attributes) to be spread over many lines instead of being on one. A space may be present between the tag content and <span class="term">></span>, like <span class="term"><div ></span> and <span class="term"><img / ></span>, but not after the <span class="term"><</span>.<br /> +<br /> +  *  Element and attribute names need not be lower-cased.<br /> +<br /> +  *  Attribute string of elements may be liberally spaced with tabs, line-breaks, etc.<br /> +<br /> +  *  Attribute values may not be double-quoted, or may be single-quoted.<br /> +<br /> +  *  Left-padding of numeric entities (like, <span class="term">&#0160;</span>, <span class="term">&x07ff;</span>) with <span class="term">0</span> is okay as long as the number of characters between between the <span class="term">&</span> and the <span class="term">;</span> does not exceed 8. All entities must end with <span class="term">;</span> though.<br /> +<br /> +  *  Named character entities must be properly cased. E.g., <span class="term">&Lt;</span> or <span class="term">&TILDE;</span> will not be let through without modification.<br /> +<br /> +  *  HTML comments should not be inside element tags (okay between tags), and should begin with <span class="term"><!--</span> and end with <span class="term">--></span>. Characters like <span class="term"><</span>, <span class="term">></span>, and <span class="term">&</span> may be allowed inside depending on <span class="term">$config</span>, but any <span class="term">--></span> inside should be put in as <span class="term">--&gt;</span>. Any <span class="term">--</span> inside will be automatically converted to <span class="term">-</span>, and a space will be added before the comment delimiter <span class="term">--></span>.<br /> +<br /> +  *  <span class="term">CDATA</span> sections should not be inside element tags, and can be in element content only if plain text is allowed for that element. They should begin with <span class="term"><[CDATA[</span> and end with <span class="term">]]></span>. Characters like <span class="term"><</span>, <span class="term">></span>, and <span class="term">&</span> may be allowed inside depending on <span class="term">$config</span>, but any <span class="term">]]></span> inside should be put in as <span class="term">]]&gt;</span>.<br /> +<br /> +  *  For attribute values, character entities <span class="term">&lt;</span>, <span class="term">&gt;</span> and <span class="term">&amp;</span> should be used instead of characters <span class="term"><</span> and <span class="term">></span>, and <span class="term">&</span> (when <span class="term">&</span> is not part of a character entity). This applies even for Javascript code in values of attributes like <span class="term">onclick</span>.<br /> +<br /> +  *  Characters <span class="term"><</span>, <span class="term">></span>, <span class="term">&</span> and <span class="term">"</span> that are part of actual Javascript, etc., code in <span class="term">script</span> elements should be used as such and not be put in as entities like <span class="term">&gt;</span>. Otherwise, though the HTML will be valid, the code may fail to work. Further, if such characters have to be used, then they should be put inside <span class="term">CDATA</span> sections.<br /> +<br /> +  *  Simple instructions like "an opening tag cannot be present between two closing tags" and "nested elements should be closed in the reverse order of how they were opened" can help authors write balanced HTML. If tags are imbalanced, htmLawed will try to balance them, but in the process, depending on <span class="term">$config["keep_bad"]</span>, some code/text may be lost.<br /> +<br /> +  *  Input authors should be notified of admin-specified allowed elements, attributes, configuration values (like conversion of named entities to numeric ones), etc.<br /> +<br /> +  *  With <span class="term">$config["unique_ids"]</span> not <span class="term">0</span> and the <span class="term">id</span> attribute being permitted, writers should carefully avoid using duplicate or invalid <span class="term">id</span> values as even though htmLawed will correct/remove the values, the final output may not be the one desired. E.g., when <span class="term"><a id="home"></a><input id="home" /><label for="home"></label></span> is processed into<br /> +<span class="term"><a id="home"></a><input id="prefix_home" /><label for="home"></label></span>.<br /> +<br /> +  *  Note that even if intended HTML is lost in a highly ill-written input, the processed output will be more secure and standard-compliant.<br /> +<br /> +  *  For URLs, unless <span class="term">$config["scheme"]</span> is appropriately set, writers should avoid using escape characters or entities in schemes. E.g., <span class="term">htt&#112;</span> (which many browsers will read as the harmless <span class="term">http</span>) may be considered bad by htmLawed.<br /> +<br /> +  *  htmLawed will attempt to put plain text present directly inside <span class="term">blockquote</span>, <span class="term">form</span>, <span class="term">map</span> and <span class="term">noscript</span> elements (illegal as per the specs) inside auto-generated <span class="term">div</span> elements.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.8" id="s2.8"></a><span class="item-no">2.8</span>  Limitations & work-arounds +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed's main objective is to make the input text <em>more</em> standard-compliant, secure for web-page readers, and free of HTML elements and attributes considered undesirable by the administrator. Some of its current limitations, regardless of this objective, are noted below along with work-arounds.<br /> +<br /> +  It should be borne in mind that no browser application is 100% standard-compliant, and that some of the standard specs (like asking for normalization of white-spacing within <span class="term">textarea</span> elements) are clearly wrong. Regarding security, note that <em>unsafe</em> HTML code is not necessarily legally invalid.<br /> +<br /> +  *  htmLawed is meant for input that goes into the <span class="term">body</span> of HTML documents. HTML's head-level elements are not supported, nor are the frameset elements <span class="term">frameset</span>, <span class="term">frame</span> and <span class="term">noframes</span>.<br /> +<br /> +  *  It cannot transform the non-standard <span class="term">embed</span> elements to the standard-compliant <span class="term">object</span> elements. Yet, it can allow <span class="term">embed</span> elements if permitted (<span class="term">embed</span> is widely used and supported). Admins can certainly use the <span class="term">hook_tag</span> parameter (<a href="#s3.4.9">section 3.4.9</a>) to deploy a custom embed-to-object converter function.<br /> +<br /> +  *  The only non-standard element that may be permitted is <span class="term">embed</span>; others like <span class="term">noembed</span> and <span class="term">nobr</span> cannot be permitted without modifying the htmLawed code.<br /> +<br /> +  *  It cannot handle input that has non-HTML code like <span class="term">SVG</span> and <span class="term">MathML</span>. One way around is to break the input into pieces and passing only those without non-HTML code to htmLawed. Another is described in <a href="#s3.9">section 3.9</a>. A third way may be to some how take advantage of the <span class="term">$config["and_mark"]</span> parameter (see <a href="#s3.2">section 3.2</a>).<br /> +<br /> +  *  By default, htmLawed won't check many attribute values for standard compliance. E.g., <span class="term">width="20m"</span> with the dimension in non-standard <span class="term">m</span> is let through. Implementing universal and strict attribute value checks can make htmLawed slow and resource-intensive. Admins should look at the <span class="term">hook_tag</span> parameter (<a href="#s3.4.9">section 3.4.9</a>) or <span class="term">$spec</span> to enforce finer checks.<br /> +<br /> +  *  The attributes, deprecated (which can be transformed too) or not, that it supports are largely those that are in the specs. Only a few of the proprietary attributes are supported.<br /> +<br /> +  *  Except for contained URLs and dynamic expressions (also optional), htmLawed does not check CSS style property values. Admins should look at using the <span class="term">hook_tag</span> parameter (<a href="#s3.4.9">section 3.4.9</a>) or <span class="term">$spec</span> for finer checks. Perhaps the best option is to disallow <span class="term">style</span> but allow <span class="term">class</span> attributes with the right <span class="term">oneof</span> or <span class="term">match</span> values for <span class="term">class</span>, and have the various class style properties in <span class="term">.css</span> CSS stylesheet files.<br /> +<br /> +  *  htmLawed does not parse emoticons, decode <em>BBcode</em>, or <em>wikify</em>, auto-converting text to proper HTML. Similarly, it won't convert line-breaks to <span class="term">br</span> elements. Such functions are beyond its purview. Admins should use other code to pre- or post-process the input for such purposes.<br /> +<br /> +  *  htmLawed cannot be used to have links force-opened in new windows (by auto-adding appropriate <span class="term">target</span> and <span class="term">onclick</span> attributes to <span class="term">a</span>). Admins should look at Javascript-based DOM-modifying solutions for this. Admins may also be able to use a custom hook function to enforce such checks (<span class="term">hook_tag</span> parameter; see <a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  *  Nesting-based checks are not possible. E.g., one cannot disallow <span class="term">p</span> elements specifically inside <span class="term">td</span> while permitting it elsewhere. Admins may be able to use a custom hook function to enforce such checks (<span class="term">hook_tag</span> parameter; see <a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  *  Except for optionally converting absolute or relative URLs to the other type, htmLawed will not alter URLs (e.g., to change the value of query strings or to convert <span class="term">http</span> to <span class="term">https</span>. Having absolute URLs may be a standard-requirement, e.g., when HTML is embedded in email messages, whereas altering URLs for other purposes is beyond htmLawed's goals. Admins may be able to use a custom hook function to enforce such checks (<span class="term">hook_tag</span> parameter; see <a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  *  Pairs of opening and closing tags that do not enclose any content (like <span class="term"><em></em></span>) are not removed. This may be against the standard specs for certain elements (e.g., <span class="term">table</span>). However, presence of such standard-incompliant code will not break the display or layout of content. Admins can also use simple regex-based code to filter out such code.<br /> +<br /> +  *  htmLawed does not check for certain element orderings described in the standard specs (e.g., in a <span class="term">table</span>, <span class="term">tbody</span> is allowed before <span class="term">tfoot</span>). Admins may be able to use a custom hook function to enforce such checks (<span class="term">hook_tag</span> parameter; see <a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  *  htmLawed does not check the number of nested elements. E.g., it will allow two <span class="term">caption</span> elements in a <span class="term">table</span> element, illegal as per the specs. Admins may be able to use a custom hook function to enforce such checks (<span class="term">hook_tag</span> parameter; see <a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  *  htmLawed might convert certain entities to actual characters and remove backslashes and CSS comment-markers (<span class="term">/*</span>) in <span class="term">style</span> attribute values in order to detect malicious HTML like crafted IE-specific dynamic expressions like <span class="term">&#101;xpression...</span>. If this is too harsh, admins can allow CSS expressions through htmLawed core but then use a custom function through the <span class="term">hook_tag</span> parameter (<a href="#s3.4.9">section 3.4.9</a>) to more specifically identify CSS expressions in the <span class="term">style</span> attribute values. Also, using <span class="term">$config["style_pass"]</span>, it is possible to have htmLawed pass <span class="term">style</span> attribute values without even looking at them (<a href="#s3.4.8">section 3.4.8</a>).<br /> +<br /> +  *  htmLawed does not correct certain possible attribute-based security vulnerabilities (e.g., <span class="term"><a href="http://x%22+style=%22background-image:xss">x</a></span>). These arise when browsers mis-identify markup in <em>escaped</em> text, defeating the very purpose of escaping text (a bad browser will read the given example as <span class="term"><a href="http://x" style="background-image:xss">x</a></span>).<br /> +<br /> +  *  Because of poor Unicode support in PHP, htmLawed does not remove the <em>high value</em> HTML-invalid characters with multi-byte code-points. Such characters however are extremely unlikely to be in the input. (see <a href="#s3.1">section 3.1</a>).<br /> +<br /> +  *  Like any script using PHP's PCRE regex functions, PHP setup-specific low PCRE limit values can cause htmLawed to at least partially fail with very long input texts.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s2.9" id="s2.9"></a><span class="item-no">2.9</span>  Examples +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  <strong>1.</strong> A blog administrator wants to allow only <span class="term">a</span>, <span class="term">em</span>, <span class="term">strike</span>, <span class="term">strong</span> and <span class="term">u</span> in comments, but needs <span class="term">strike</span> and <span class="term">u</span> transformed to <span class="term">span</span> for better XHTML 1-strict compliance, and, he wants the <span class="term">a</span> links to be to <span class="term">http</span> or <span class="term">https</span> resources:<br /> +<br /> + +<code class="code">    $processed = htmLawed($in, array('elements'=>'a, em, strike, strong, u', 'make_tag_strict'=>1, 'safe'=>1, 'schemes'=>'*:http, https'), 'a=href');</code> +<br /> +<br /> +  <strong>2.</strong> An author uses a custom-made web application to load content on his web-site. He is the only one using that application and the content he generates has all types of HTML, including scripts. The web application uses htmLawed primarily as a tool to correct errors that creep in while writing HTML and to take care of the occasional <em>bad</em> characters in copy-paste text introduced by Microsoft Office. The web application provides a preview before submitted input is added to the content. For the previewing process, htmLawed is set up as follows:<br /> +<br /> + +<code class="code">    $processed = htmLawed($in, array('css_expression'=>1, 'keep_bad'=>1, 'make_tag_strict'=>1, 'schemes'=>'*:*', 'valid_xhtml'=>1));</code> +<br /> +<br /> +  For the final submission process, <span class="term">keep_bad</span> is set to <span class="term">6</span>. A value of <span class="term">1</span> for the preview process allows the author to note and correct any HTML mistake without losing any of the typed text.<br /> +<br /> +  <strong>3.</strong> A data-miner is scraping information in a specific table of similar web-pages and is collating the data rows, and uses htmLawed to reduce unnecessary markup and white-spaces:<br /> +<br /> + +<code class="code">    $processed = htmLawed($in, array('elements'=>'tr, td', 'tidy'=>-1), 'tr, td =');</code> +<br /> + +</div> +</div> +<div class="section"><h2> +<a name="s3" id="s3"></a><span class="item-no">3</span>  Details +</h2><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<div class="sub-section"><h3> +<a name="s3.1" id="s3.1"></a><span class="item-no">3.1</span>  Invalid/dangerous characters +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Valid characters (more correctly, their code-points) in HTML or XML are, hexadecimally, <span class="term">9</span>, <span class="term">a</span>, <span class="term">d</span>, <span class="term">20</span> to <span class="term">d7ff</span>, and <span class="term">e000</span> to <span class="term">10ffff</span>, except <span class="term">fffe</span> and <span class="term">ffff</span> (decimally, <span class="term">9</span>, <span class="term">10</span>, <span class="term">13</span>, <span class="term">32</span> to <span class="term">55295</span>, and <span class="term">57344</span> to <span class="term">1114111</span>, except <span class="term">65534</span> and <span class="term">65535</span>). htmLawed removes the invalid characters <span class="term">0</span> to <span class="term">8</span>, <span class="term">b</span>, <span class="term">c</span>, and <span class="term">e</span> to <span class="term">1f</span>.<br /> +<br /> +  Because of PHP's poor native support for multi-byte characters, htmLawed cannot check for the remaining invalid code-points. However, for various reasons, it is very unlikely for any of those characters to be in the input.<br /> +<br /> +  Characters that are discouraged (see <a href="#s5.1">section 5.1</a>) but not invalid are not removed by htmLawed.<br /> +<br /> +  It (function <span class="term">hl_tag()</span>) also replaces the potentially dangerous (in some Mozilla [Firefox] and Opera browsers) soft-hyphen character (code-point, hexadecimally, <span class="term">ad</span>, or decimally, <span class="term">173</span>) in attribute values with spaces. Where required, the characters <span class="term"><</span>, <span class="term">></span>, <span class="term">&</span>, and <span class="term">"</span> are converted to entities.<br /> +<br /> +  With <span class="term">$config["clean_ms_char"]</span> set as <span class="term">1</span> or <span class="term">2</span>, many of the discouraged characters (decimal code-points <span class="term">127</span> to <span class="term">159</span> except <span class="term">133</span>) that many Microsoft applications incorrectly use (as per the <span class="term">Windows 1252</span> [<span class="term">Cp-1252</span>] or a similar encoding system), and the character for decimal code-point <span class="term">133</span>, are converted to appropriate decimal numerical entities (or removed for a few cases)-- see appendix in <a href="#s5.4">section 5.4</a>. This can help avoid some display issues arising from copying-pasting of content.<br /> +<br /> +  With <span class="term">$config["clean_ms_char"]</span> set as <span class="term">2</span>, characters for the hexadecimal code-points <span class="term">82</span>, <span class="term">91</span>, and <span class="term">92</span> (for special single-quotes), and <span class="term">84</span>, <span class="term">93</span>, and <span class="term">94</span> (for special double-quotes) are converted to ordinary single and double quotes respectively and not to entities.<br /> +<br /> +  The character values are replaced with entities/characters and not character values referred to by the entities/characters to keep this task independent of the character-encoding of input text.<br /> +<br /> +  The <span class="term">$config["clean_ms_char"]</span> parameter need not be used if authors do not copy-paste Microsoft-created text or if the input text is not believed to use the <span class="term">Windows 1252</span> or a similar encoding. Further, the input form and the web-pages displaying it or its content should have the character encoding appropriately marked-up.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.2" id="s3.2"></a><span class="item-no">3.2</span>  Character references/entities +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Valid character entities take the form <span class="term">&*;</span> where <span class="term">*</span> is <span class="term">#x</span> followed by a hexadecimal number (hexadecimal numeric entity; like <span class="term">&#xA0;</span> for non-breaking space), or alphanumeric like <span class="term">gt</span> (external or named entity; like <span class="term">&nbsp;</span> for non-breaking space), or <span class="term">#</span> followed by a number (decimal numeric entity; like <span class="term">&#160;</span> for non-breaking space). Character entities referring to the soft-hyphen character (the <span class="term">&shy;</span> or <span class="term">\xad</span> character; hexadecimal code-point <span class="term">ad</span> [decimal <span class="term">173</span>]) in attribute values are always replaced with spaces; soft-hyphens in attribute values introduce vulnerabilities in some older versions of the Opera and Mozilla [Firefox] browsers.<br /> +<br /> +  htmLawed (function <span class="term">hl_ent()</span>):<br /> +<br /> +  *  Neutralizes entities with multiple leading zeroes or missing semi-colons (potentially dangerous)<br /> +<br /> +  *  Lowercases the <span class="term">X</span> (for XML-compliance) and <span class="term">A-F</span> of hexadecimal numeric entities<br /> +<br /> +  *  Neutralizes entities referring to characters that are HTML-invalid (see <a href="#s3.1">section 3.1</a>)<br /> +<br /> +  *  Neutralizes entities referring to characters that are HTML-discouraged (code-points, hexadecimally, <span class="term">7f</span> to <span class="term">84</span>, <span class="term">86</span> to <span class="term">9f</span>, and <span class="term">fdd0</span> to <span class="term">fddf</span>, or decimally, <span class="term">127</span> to <span class="term">132</span>, <span class="term">134</span> to <span class="term">159</span>, and <span class="term">64991</span> to <span class="term">64976</span>). Entities referring to the remaining discouraged characters (see <a href="#s5.1">section 5.1</a> for a full list) are let through.<br /> +<br /> +  *  Neutralizes named entities that are not in the specs.<br /> +<br /> +  *  Optionally converts valid HTML-specific named entities except <span class="term">&gt;</span>, <span class="term">&lt;</span>, <span class="term">&quot;</span>, and <span class="term">&amp;</span> to decimal numeric ones (hexadecimal if $config["hexdec_entity"] is <span class="term">2</span>) for generic XML-compliance. For this, <span class="term">$config["named_entity"]</span> should be <span class="term">1</span>.<br /> +<br /> +  *  Optionally converts hexadecimal numeric entities to the more widely supported decimal ones. For this, <span class="term">$config["hexdec_entity"]</span> should be <span class="term">0</span>.<br /> +<br /> +  *  Optionally converts decimal numeric entities to the hexadecimal ones. For this, <span class="term">$config["hexdec_entity"]</span> should be <span class="term">2</span>.<br /> +<br /> +  <em>Neutralization</em> refers to the <em>entitification</em> of <span class="term">&</span> to <span class="term">&amp;</span>.<br /> +<br /> +  <strong>Note</strong>: htmLawed does not convert entities to the actual characters represented by them; one can pass the htmLawed output through PHP's <span class="term">html_entity_decode</span> <a href="http://www.php.net/html_entity_decode">function</a> for that.<br /> +<br /> +  <strong>Note</strong>: If <span class="term">$config["and_mark"]</span> is set, and set to a value other than <span class="term">0</span>, then the <span class="term">&</span> characters in the original input are replaced with the control character for the hexadecimal code-point <span class="term">6</span> (<span class="term">\x06</span>; <span class="term">&</span> characters introduced by htmLawed, e.g., after converting <span class="term"><</span> to <span class="term">&lt;</span>, are not affected). This allows one to distinguish, say, an <span class="term">&gt;</span> introduced by htmLawed and an <span class="term">&gt;</span> put in by the input writer, and can be helpful in further processing of the htmLawed-processed text (e.g., to identify the character sequence <span class="term">o(><)o</span> to generate an emoticon image). When this feature is active, admins should ensure that the htmLawed output is not directly used in web pages or XML documents as the presence of the <span class="term">\x06</span> can break documents. Before use in such documents, and preferably before any storage, any remaining <span class="term">\x06</span> should be changed back to <span class="term">&</span>, e.g., with:<br /> +<br /> + +<code class="code">    $final = str_replace("\x06", '&', $prelim);</code> +<br /> +<br /> +  Also, see <a href="#s3.9">section 3.9</a>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.3" id="s3.3"></a><span class="item-no">3.3</span>  HTML elements +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed can be configured to allow only certain HTML elements (tags) in the input. Disallowed elements (just tag-content, and not element-content), based on <span class="term">$config["keep_bad"]</span>, are either <em>neutralized</em> (converted to plain text by entitification of <span class="term"><</span> and <span class="term">></span>) or removed.<br /> +<br /> +  E.g., with only <span class="term">em</span> permitted:<br /> +<br /> +  Input:<br /> +<br /> + +<code class="code">      <em>My</em> website is <a href="http://a.com>a.com</a>.</code> +<br /> +<br /> +  Output, with <span class="term">$config["keep_bad"] = 0</span>:<br /> +<br /> + +<code class="code">      <em>My</em> website is a.com.</code> +<br /> +<br /> +  Output, with <span class="term">$config["keep_bad"]</span> not <span class="term">0</span>:<br /> +<br /> + +<code class="code">      <em>My</em> website is &lt;a href=""&gt;a.com&lt;/a&gt;.</code> +<br /> +<br /> +  See <a href="#s3.3.3">section 3.3.3</a> for differences between the various non-zero <span class="term">$config["keep_bad"]</span> values.<br /> +<br /> +  htmLawed by default permits these 86 elements:<br /> +<br /> + +<code class="code">    a, abbr, acronym, address, applet, area, b, bdo, big, blockquote, br, button, caption, center, cite, code, col, colgroup, dd, del, dfn, dir, div, dl, dt, em, embed, fieldset, font, form, h1, h2, h3, h4, h5, h6, hr, i, iframe, img, input, ins, isindex, kbd, label, legend, li, map, menu, noscript, object, ol, optgroup, option, p, param, pre, q, rb, rbc, rp, rt, rtc, ruby, s, samp, script, select, small, span, strike, strong, sub, sup, table, tbody, td, textarea, tfoot, th, thead, tr, tt, u, ul, var</code> +<br /> +<br /> +  Except for <span class="term">embed</span> (included because of its wide-spread use) and the Ruby elements (<span class="term">rb</span>, <span class="term">rbc</span>, <span class="term">rp</span>, <span class="term">rt</span>, <span class="term">rtc</span>, <span class="term">ruby</span>; part of XHTML 1.1), these are all the elements in the HTML 4/XHTML 1 specs. Strict-specific specs. exclude <span class="term">center</span>, <span class="term">dir</span>, <span class="term">font</span>, <span class="term">isindex</span>, <span class="term">menu</span>, <span class="term">s</span>, <span class="term">strike</span>, and <span class="term">u</span>.<br /> +<br /> +  With <span class="term">$config["safe"] = 1</span>, the default set will exclude <span class="term">applet</span>, <span class="term">embed</span>, <span class="term">iframe</span>, <span class="term">object</span> and <span class="term">script</span>; see <a href="#s3.6">section 3.6</a>.<br /> +<br /> +  When <span class="term">$config["elements"]</span>, which specifies allowed elements, is <em>properly</em> defined, and neither empty nor set to <span class="term">0</span> or <span class="term">*</span>, the default set is not used. To have elements added to or removed from the default set, a <span class="term">+/-</span> notation is used. E.g., <span class="term">*-script-object</span> implies that only <span class="term">script</span> and <span class="term">object</span> are disallowed, whereas <span class="term">*+embed</span> means that <span class="term">noembed</span> is also allowed. Elements can also be specified as comma separated names. E.g., <span class="term">a, b, i</span> means only <span class="term">a</span>, <span class="term">b</span> and <span class="term">i</span> are permitted. In this notation, <span class="term">*</span>, <span class="term">+</span> and <span class="term">-</span> have no significance and can actually cause a mis-reading.<br /> +<br /> +  Some more examples of <span class="term">$config["elements"]</span> values indicating permitted elements (note that empty spaces are liberally allowed for clarity):<br /> +<br /> +  *  <span class="term">a, blockquote, code, em, strong</span> -- only <span class="term">a</span>, <span class="term">blockquote</span>, <span class="term">code</span>, <span class="term">em</span>, and <span class="term">strong</span><br /> +  *  <span class="term">*-script</span> -- all excluding <span class="term">script</span><br /> +  *  <span class="term">* -center -dir -font -isindex -menu -s -strike -u</span> -- only XHTML-Strict elements<br /> +  *  <span class="term">*+noembed-script</span> -- all including <span class="term">noembed</span> excluding <span class="term">script</span><br /> +<br /> +  Some mis-usages (and the resulting permitted elements) that can be avoided:<br /> +<br /> +  *  <span class="term">-*</span> -- none; instead of htmLawed, one might just use, e.g., the <span class="term">htmlspecialchars()</span> PHP function<br /> +  *  <span class="term">*, -script</span> -- all except <span class="term">script</span>; admin probably meant <span class="term">*-script</span><br /> +  *  <span class="term">-*, a, em, strong</span> -- all; admin probably meant <span class="term">a, em, strong</span><br /> +  *  <span class="term">*</span> -- all; admin need not have set <span class="term">elements</span><br /> +  *  <span class="term">*-form+form</span> -- all; a <span class="term">+</span> will always over-ride any <span class="term">-</span><br /> +  *  <span class="term">*, noembed</span> -- only <span class="term">noembed</span>; admin probably meant <span class="term">*+noembed</span><br /> +  *  <span class="term">a, +b, i</span> -- only <span class="term">a</span> and <span class="term">i</span>; admin probably meant <span class="term">a, b, i</span><br /> +<br /> +  Basically, when using the <span class="term">+/-</span> notation, commas (<span class="term">,</span>) should not be used, and vice versa, and <span class="term">*</span> should be used with the former but not the latter.<br /> +<br /> +  <strong>Note</strong>: Even if an element that is not in the default set is allowed through <span class="term">$config["elements"]</span>, like <span class="term">noembed</span> in the last example, it will eventually be removed during tag balancing unless such balancing is turned off (<span class="term">$config["balance"]</span> set to <span class="term">0</span>). Currently, the only way around this, which actually is simple, is to edit the various arrays in the function <span class="term">hl_bal()</span> to accommodate the element and its nesting properties.<br /> +<br /> +  <strong>A possibly second way to specify allowed elements</strong> is to set <span class="term">$config["parent"]</span> to an element name that supposedly will hold the input, and to set <span class="term">$config["balance"]</span> to <span class="term">1</span>. During tag balancing (see <a href="#s3.3.3">section 3.3.3</a>), all elements that cannot legally nest inside the parent element will be removed. The parent element is auto-reset to <span class="term">div</span> if <span class="term">$config["parent"]</span> is empty, <span class="term">body</span>, or an element not in htmLawed's default set of 86 elements.<br /> +<br /> +  <em>Tag transformation</em> is possible for improving XHTML-Strict compliance -- most of the deprecated elements are removed or converted to valid XHTML-Strict ones; see <a href="#s3.3.2">section 3.3.2</a>.<br /> + +<div class="sub-sub-section"><h4> +<a name="s3.3.1" id="s3.3.1"></a><span class="item-no">3.3.1</span>  Handling of comments and CDATA sections +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  <span class="term">CDATA</span> sections have the format <span class="term"><![CDATA[...anything but not "]]>"...]]></span>, and HTML comments, <span class="term"><!--...anything but not "-->"... --></span>. Neither HTML comments nor <span class="term">CDATA</span> sections can reside inside tags. HTML comments can exist anywhere else, but <span class="term">CDATA</span> sections can exist only where plain text is allowed (e.g., immediately inside <span class="term">td</span> element content but not immediately inside <span class="term">tr</span> element content).<br /> +<br /> +  htmLawed (function <span class="term">hl_cmtcd()</span>) handles HTML comments or <span class="term">CDATA</span> sections depending on the values of <span class="term">$config["comment"]</span> or <span class="term">$config["cdata"]</span>. If <span class="term">0</span>, such markup is not looked for and the text is processed like plain text. If <span class="term">1</span>, it is removed completely. If <span class="term">2</span>, it is preserved but any <span class="term"><</span>, <span class="term">></span> and <span class="term">&</span> inside are changed to entities. If <span class="term">3</span>, they are left as such.<br /> +<br /> +  Note that for the last two cases, HTML comments and <span class="term">CDATA</span> sections will always be removed from tag content (function <span class="term">hl_tag()</span>).<br /> +<br /> +  Examples:<br /> +<br /> +  Input:<br /> + +<code class="code">    <!-- home link --><a href="home.htm"><![CDATA[x=&y]]>Home</a></code> +<br /> +  Output (<span class="term">$config["comment"] = 0, $config["cdata"] = 2</span>):<br /> + +<code class="code">    &lt;-- home link --&gt;<a href="home.htm"><![CDATA[x=&amp;y]]>Home</a></code> +<br /> +  Output (<span class="term">$config["comment"] = 1, $config["cdata"] = 2</span>):<br /> + +<code class="code">    <a href="home.htm"><![CDATA[x=&amp;y]]>Home</a></code> +<br /> +  Output (<span class="term">$config["comment"] = 2, $config["cdata"] = 2</span>):<br /> + +<code class="code">    <!-- home link --><a href="home.htm"><![CDATA[x=&amp;y]]>Home</a></code> +<br /> +  Output (<span class="term">$config["comment"] = 2, $config["cdata"] = 1</span>):<br /> + +<code class="code">    <!-- home link --><a href="home.htm">Home</a></code> +<br /> +  Output (<span class="term">$config["comment"] = 3, $config["cdata"] = 3</span>):<br /> + +<code class="code">    <!-- home link --><a href="home.htm"><![CDATA[x=&y]]>Home</a></code> +<br /> +<br /> +  For standard-compliance, comments are given the form <span class="term"><!--comment --></span>, and any <span class="term">--</span> in the content is made <span class="term">-</span>.<br /> +<br /> +  When <span class="term">$config["safe"] = 1</span>, CDATA sections and comments are considered plain text unless <span class="term">$config["comment"]</span> or <span class="term">$config["cdata"]</span> is explicitly specified; see <a href="#s3.6">section 3.6</a>.<br /> + +</div> +<div class="sub-sub-section"><h4> +<a name="s3.3.2" id="s3.3.2"></a><span class="item-no">3.3.2</span>  Tag-transformation for better XHTML-Strict +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  If <span class="term">$config["make_tag_strict"]</span> is set and not <span class="term">0</span>, following non-XHTML-Strict elements (and attributes), even if admin-permitted, are mutated as indicated (element content remains intact; function <span class="term">hl_tag2()</span>):<br /> +<br /> +  *  applet - (based on <span class="term">$config["make_tag_strict"]</span>, unchanged (<span class="term">1</span>) or removed (<span class="term">2</span>))<br /> +  *  center - <span class="term">div style="text-align: center;"</span><br /> +  *  dir - <span class="term">ul</span><br /> +  *  embed - (based on <span class="term">$config["make_tag_strict"]</span>, unchanged (<span class="term">1</span>) or removed (<span class="term">2</span>))<br /> +  *  font (face, size, color) -    <span class="term">span style="font-family: ; font-size: ; color: ;"</span> (size transformation <a href="http://style.cleverchimp.com/font_size_intervals/altintervals.html">reference</a>)<br /> +  *  isindex - (based on <span class="term">$config["make_tag_strict"]</span>, unchanged (<span class="term">1</span>) or removed (<span class="term">2</span>))<br /> +  *  menu - <span class="term">ul</span><br /> +  *  s - <span class="term">span style="text-decoration: line-through;"</span><br /> +  *  strike - <span class="term">span style="text-decoration: line-through;"</span><br /> +  *  u - <span class="term">span style="text-decoration: underline;"</span><br /> +<br /> +  For an element with a pre-existing <span class="term">style</span> attribute value, the extra style properties are appended.<br /> +<br /> +  Example input:<br /> +<br /> + +<code class="code">    <center></code> +<br /> + +<code class="code">     The PHP <s>software</s> script used for this <strike>web-page</strike> web-page is <font style="font-weight: bold " face=arial size='+3' color   =  "red  ">htmLawedTest.php</font>, from <u style= 'color:green'>PHP Labware</u>.</code> +<br /> + +<code class="code">    </center></code> +<br /> +<br /> +  The output:<br /> +<br /> + +<code class="code">    <div style="text-align: center;"></code> +<br /> + +<code class="code">     The PHP <span style="text-decoration: line-through;">software</span> script used for this <span style="text-decoration: line-through;">web-page</span> web-page is <span style="font-weight: bold; font-family: arial; color: red; font-size: 200%;">htmLawedTest.php</span>, from <span style="color:green; text-decoration: underline;">PHP Labware</span>.</code> +<br /> + +<code class="code">    </div></code> +<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.3.3" id="s3.3.3"></a><span class="item-no">3.3.3</span>  Tag balancing and proper nesting +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  If <span class="term">$config["balance"]</span> is set to <span class="term">1</span>, htmLawed (function <span class="term">hl_bal()</span>) checks and corrects the input to have properly balanced tags and legal element content (i.e., any element nesting should be valid, and plain text may be present only in the content of elements that allow them).<br /> +<br /> +  Depending on the value of <span class="term">$config["keep_bad"]</span> (see <a href="#s2.2">section 2.2</a> and <a href="#s3.3">section 3.3</a>), illegal content may be removed or neutralized to plain text by converting < and > to entities:<br /> +<br /> +  <span class="term">0</span> - remove; this option is available only to maintain Kses-compatibility and should not be used otherwise (see <a href="#s2.6">section 2.6</a>)<br /> +  <span class="term">1</span> - neutralize tags and keep element content<br /> +  <span class="term">2</span> - remove tags but keep element content<br /> +  <span class="term">3</span> and <span class="term">4</span> - like <span class="term">1</span> and <span class="term">2</span>, but keep element content only if text (<span class="term">pcdata</span>) is valid in parent element as per specs<br /> +  <span class="term">5</span> and <span class="term">6</span> -  like <span class="term">3</span> and <span class="term">4</span>, but line-breaks, tabs and spaces are left<br /> +<br /> +  Example input (disallowing the <span class="term">p</span> element):<br /> +<br /> + +<code class="code">    <*> Pseudo-tags <*></code> +<br /> + +<code class="code">    <xml>Non-HTML tag xml</xml></code> +<br /> + +<code class="code">    <p></code> +<br /> + +<code class="code">    Disallowed tag p</code> +<br /> + +<code class="code">    </p></code> +<br /> + +<code class="code">    <ul>Bad<li>OK</li></ul></code> +<br /> +<br /> +  The output with <span class="term">$config["keep_bad"] = 1</span>:<br /> +<br /> + +<code class="code">    &lt;*&gt; Pseudo-tags &lt;*&gt;</code> +<br /> + +<code class="code">    &lt;xml&gt;Non-HTML tag xml&lt;/xml&gt;</code> +<br /> + +<code class="code">    &lt;p&gt;</code> +<br /> + +<code class="code">    Disallowed tag p</code> +<br /> + +<code class="code">    &lt;/p&gt;</code> +<br /> + +<code class="code">    <ul>Bad<li>OK</li></ul></code> +<br /> +<br /> +  The output with <span class="term">$config["keep_bad"] = 3</span>:<br /> +<br /> + +<code class="code">    &lt;*&gt; Pseudo-tags &lt;*&gt;</code> +<br /> + +<code class="code">    &lt;xml&gt;Non-HTML tag xml&lt;/xml&gt;</code> +<br /> + +<code class="code">    &lt;p&gt;</code> +<br /> + +<code class="code">    Disallowed tag p</code> +<br /> + +<code class="code">    &lt;/p&gt;</code> +<br /> + +<code class="code">    <ul><li>OK</li></ul></code> +<br /> +<br /> +  The output with <span class="term">$config["keep_bad"] = 6</span>:<br /> +<br /> + +<code class="code">    &lt;*&gt; Pseudo-tags &lt;*&gt;</code> +<br /> + +<code class="code">    Non-HTML tag xml</code> +<br /> +<br /> + +<code class="code">    Disallowed tag p</code> +<br /> +<br /> + +<code class="code">    <ul><li>OK</li></ul></code> +<br /> +<br /> +  An option like <span class="term">1</span> is useful, e.g., when a writer previews his submission, whereas one like <span class="term">3</span> is useful before content is finalized and made available to all.<br /> +<br /> +  <strong>Note:</strong> In the example above, unlike <span class="term"><*></span>, <span class="term"><xml></span> gets considered as a tag (even though there is no HTML element named <span class="term">xml</span>). In general, text matching the regular expression pattern <span class="term"><(/?)([a-zA-Z][a-zA-Z1-6]*)([^>]*?)\s?></span> is considered a tag (phrase enclosed by the angled brackets <span class="term"><</span> and <span class="term">></span>, and starting [with an optional slash preceding] with an alphanumeric word that starts with an alphabet...).<br /> +<br /> +  Nesting/content rules for each of the 86 elements in htmLawed's default set (see <a href="#s3.3">section 3.3</a>) are defined in function <span class="term">hl_bal()</span>. This means that if a non-standard element besides <span class="term">embed</span> is being permitted through <span class="term">$config["elements"]</span>, the element's tag content will end up getting removed if <span class="term">$config["balance"]</span> is set to <span class="term">1</span>.<br /> +<br /> +  Plain text and/or certain elements nested inside <span class="term">blockquote</span>, <span class="term">form</span>, <span class="term">map</span> and <span class="term">noscript</span> need to be in block-level elements. This point is often missed during manual writing of HTML code. htmLawed attempts to address this during balancing. E.g., if the parent container is set as <span class="term">form</span>, the input <span class="term">B:<input type="text" value="b" />C:<input type="text" value="c" /></span> is converted to <span class="term"><div>B:<input type="text" value="b" />C:<input type="text" value="c" /></div></span>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.3.4" id="s3.3.4"></a><span class="item-no">3.3.4</span>  Elements requiring child elements +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  As per specs, the following elements require legal child elements nested inside them:<br /> +<br /> + +<code class="code">    blockquote, dir, dl, form, map, menu, noscript, ol, optgroup, rbc, rtc, ruby, select, table, tbody, tfoot, thead, tr, ul</code> +<br /> +<br /> +  In some cases, the specs stipulate the number and/or the ordering of the child elements. A <span class="term">table</span> can have 0 or 1 <span class="term">caption</span>, <span class="term">tbody</span>, <span class="term">tfoot</span>, and <span class="term">thead</span>, but they must be in this order: <span class="term">caption</span>, <span class="term">thead</span>, <span class="term">tfoot</span>, <span class="term">tbody</span>.<br /> +<br /> +  htmLawed currently does not check for conformance to these rules. Note that any non-compliance in this regard will not introduce security vulnerabilities, crash browser applications, or affect the rendering of web-pages.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.3.5" id="s3.3.5"></a><span class="item-no">3.3.5</span>  Beautify or compact HTML +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  By default, htmLawed will neither <em>beautify</em> HTML code by formatting it with indentations, etc., nor will it make it compact by removing un-needed white-space.(It does always properly white-space tag content.)<br /> +<br /> +  As per the HTML standards, spaces, tabs and line-breaks in web-pages (except those inside <span class="term">pre</span> elements) are all considered equivalent, and referred to as <em>white-spaces</em>. Browser applications are supposed to consider contiguous white-spaces as just a single space, and to disregard white-spaces trailing opening tags or preceding closing tags. This white-space <em>normalization</em> allows the use of text/code beautifully formatted with indentations and line-spacings for readability. Such <em>pretty</em> HTML can, however, increase the size of web-pages, or make the extraction or scraping of plain text cumbersome.<br /> +<br /> +  With the <span class="term">$config</span> parameter <span class="term">tidy</span>, htmLawed can be used to beautify or compact the input text. Input with just plain text and no HTML markup is also subject to this. Besides <span class="term">pre</span>, the <span class="term">script</span> and <span class="term">textarea</span> elements, CDATA sections, and HTML comments are not subjected to the tidying process.<br /> +<br /> +  To <em>compact</em>, use <span class="term">$config["tidy"] = -1</span>; single instances or runs of white-spaces are replaced with a single space, and white-spaces trailing and leading open and closing tags, respectively, are removed.<br /> +<br /> +  To <em>beautify</em>, <span class="term">$config["tidy"]</span> is set as <span class="term">1</span>, or for customized tidying, as a string like <span class="term">2s2n</span>. The <span class="term">s</span> or <span class="term">t</span> character specifies the use of spaces or tabs for indentation. The first and third characters, any of the digits 0-9, specify the number of spaces or tabs per indentation, and any parental lead spacing (extra indenting of the whole block of input text). The <span class="term">r</span> and <span class="term">n</span> characters are used to specify line-break characters: <span class="term">n</span> for <span class="term">\n</span> (Unix/Mac OS X line-breaks), <span class="term">rn</span> or <span class="term">nr</span> for <span class="term">\r\n</span> (Windows/DOS line-breaks), or <span class="term">r</span> for <span class="term">\r</span>.<br /> +<br /> +  The <span class="term">$config["tidy"]</span> value of <span class="term">1</span> is equivalent to <span class="term">2s0n</span>. Other <span class="term">$config["tidy"]</span> values are read loosely: a value of <span class="term">4</span> is equivalent to <span class="term">4s0n</span>; <span class="term">t2</span>, to <span class="term">1t2n</span>; <span class="term">s</span>, to <span class="term">2s0n</span>; <span class="term">2TR</span>, to <span class="term">2t0r</span>; <span class="term">T1</span>, to <span class="term">1t1n</span>; <span class="term">nr3</span>, to <span class="term">3s0nr</span>, and so on. Except in the indentations and line-spacings, runs of white-spaces are replaced with a single space during beautification.<br /> +<br /> +  Input formatting using <span class="term">$config["tidy"]</span> is not recommended when input text has mixed markup (like HTML + PHP).<br /> + +</div> +</div> +<div class="sub-section"><h3> +<a name="s3.4" id="s3.4"></a><span class="item-no">3.4</span>  Attributes +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed will only permit attributes described in the HTML specs (including deprecated ones). It also permits some attributes for use with the <span class="term">embed</span> element (the non-standard <span class="term">embed</span> element is supported in htmLawed because of its widespread use), and the the <span class="term">xml:space</span> attribute (valid only in XHTML 1.1). A list of such 111 attributes and the elements they are allowed in is in <a href="#s5.2">section 5.2</a>.<br /> +<br /> +  When <span class="term">$config["deny_attribute"]</span> is not set, or set to <span class="term">0</span>, or empty (<span class="term">""</span>), all the 111 attributes are permitted. Otherwise, <span class="term">$config["deny_attribute"]</span> can be set as a list of comma-separated names of the denied attributes. <span class="term">on*</span> can be used to refer to the group of potentially dangerous, script-accepting attributes: <span class="term">onblur</span>, <span class="term">onchange</span>, <span class="term">onclick</span>, <span class="term">ondblclick</span>, <span class="term">onfocus</span>, <span class="term">onkeydown</span>, <span class="term">onkeypress</span>, <span class="term">onkeyup</span>, <span class="term">onmousedown</span>, <span class="term">onmousemove</span>, <span class="term">onmouseout</span>, <span class="term">onmouseover</span>, <span class="term">onmouseup</span>, <span class="term">onreset</span>, <span class="term">onselect</span> and <span class="term">onsubmit</span>.<br /> +<br /> +  Note that attributes specified in <span class="term">$config["deny_attribute"]</span> are denied globally, for all elements. To deny attributes for only specific elements, <span class="term">$spec</span> (see <a href="#s2.3">section 2.3</a>) can be used. <span class="term">$spec</span> can also be used to element-specifically permit an attribute otherwise denied through <span class="term">$config["deny_attribute"]</span>.<br /> +<br /> +  With <span class="term">$config["safe"] = 1</span> (<a href="#s3.6">section 3.6</a>), the <span class="term">on*</span> attributes are automatically disallowed.<br /> +<br /> +  <strong>Note</strong>: To deny all but a few attributes globally, a simpler way to specify <span class="term">$config["deny_attribute"]</span> would be to use the notation <span class="term">* -attribute1 -attribute2 ...</span>. Thus, a value of <span class="term">* -title -href</span> implies that except <span class="term">href</span> and <span class="term">title</span> (where allowed as per standards) all other attributes are to be removed. With this notation, the value for the parameter <span class="term">safe</span> (<a href="#s3.6">section 3.6</a>) will have no effect on <span class="term">deny_attribute</span>.<br /> +<br /> +  htmLawed (function <span class="term">hl_tag()</span>) also:<br /> +<br /> +  *  Lower-cases attribute names<br /> +  *  Removes duplicate attributes (last one stays)<br /> +  *  Gives attributes the form <span class="term">name="value"</span> and single-spaces them, removing unnecessary white-spacing<br /> +  *  Provides <em>required</em> attributes (see <a href="#s3.4.1">section 3.4.1</a>)<br /> +  *  Double-quotes values and escapes any <span class="term">"</span> inside them<br /> +  *  Replaces the possibly dangerous soft-hyphen characters (hexadecimal code-point <span class="term">ad</span>) in the values with spaces<br /> +  *  Allows custom function to additionally filter/modify attribute values (see <a href="#s3.4.9">section 3.4.9</a>)<br /> + +<div class="sub-sub-section"><h4> +<a name="s3.4.1" id="s3.4.1"></a><span class="item-no">3.4.1</span>  Auto-addition of XHTML-required attributes +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  If indicated attributes for the following elements are found missing, htmLawed (function <span class="term">hl_tag()</span>) will add them (with values same as attribute names unless indicated otherwise below):<br /> +<br /> +  *  area - alt (<span class="term">area</span>)<br /> +  *  area, img - src, alt (<span class="term">image</span>)<br /> +  *  bdo - dir (<span class="term">ltr</span>)<br /> +  *  form - action<br /> +  *  map - name<br /> +  *  optgroup - label<br /> +  *  param - name<br /> +  *  script - type (<span class="term">text/javascript</span>)<br /> +  *  textarea - rows (<span class="term">10</span>), cols (<span class="term">50</span>)<br /> +<br /> +  Additionally, with <span class="term">$config["xml:lang"]</span> set to <span class="term">1</span> or <span class="term">2</span>, if the <span class="term">lang</span> but not the <span class="term">xml:lang</span> attribute is declared, then the latter is added too, with a value copied from that of <span class="term">lang</span>. This is for better standard-compliance. With <span class="term">$config["xml:lang"]</span> set to <span class="term">2</span>, the <span class="term">lang</span> attribute is removed (XHTML 1.1 specs).<br /> +<br /> +  Note that the <span class="term">name</span> attribute for <span class="term">map</span>, invalid in XHTML 1.1, is also transformed if required -- see <a href="#s3.4.6">section 3.4.6</a>.<br /> + +</div> +<div class="sub-sub-section"><h4> +<a name="s3.4.2" id="s3.4.2"></a><span class="item-no">3.4.2</span>  Duplicate/invalid <span class="term">id</span> values +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  If <span class="term">$config["unique_ids"]</span> is <span class="term">1</span>, htmLawed (function <span class="term">hl_tag()</span>) removes <span class="term">id</span> attributes with values that are not XHTML-compliant (must begin with a letter and can contain letters, digits, <span class="term">:</span>, <span class="term">.</span>, <span class="term">-</span> and <span class="term">_</span>) or duplicate. If <span class="term">$config["unique_ids"]</span> is a word, any duplicate but otherwise valid value will be appropriately prefixed with the word to ensure its uniqueness. The word should begin with a letter and should contain only letters, numbers, <span class="term">:</span>, <span class="term">.</span>, <span class="term">_</span> and <span class="term">-</span>.<br /> +<br /> +  Even if multiple inputs need to be filtered (through multiple calls to htmLawed), htmLawed ensures uniqueness of <span class="term">id</span> values as it uses a global variable (<span class="term">$GLOBALS["hl_Ids"]</span> array). Further, an admin can restrict the use of certain <span class="term">id</span> values by presetting this variable before htmLawed is called into use. E.g.:<br /> +<br /> + +<code class="code">    $GLOBALS['hl_Ids'] = array('top'=>1, 'bottom'=>1, 'myform'=>1); // id values not allowed in input</code> +<br /> + +<code class="code">    $processed = htmLawed($text); // filter input</code> +<br /> + +</div> +<div class="sub-sub-section"><h4> +<a name="s3.4.3" id="s3.4.3"></a><span class="item-no">3.4.3</span>  URL schemes (protocols) and scripts in attribute values +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed edits attributes that take URLs as values if they are found to contain un-permitted schemes. E.g., if the <span class="term">afp</span> scheme is not permitted, then <span class="term"><a href="afp://domain.org"></span> becomes <span class="term"><a href="denied:afp://domain.org"></span>, and if Javascript is not permitted <span class="term"><a onclick="javascript:xss();"></span> becomes <span class="term"><a onclick="denied:javascript:xss();"></span>.<br /> +<br /> +  By default htmLawed permits these schemes in URLs for the <span class="term">href</span> attribute:<br /> +<br /> + +<code class="code">    aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet</code> +<br /> +<br /> +  Also, only <span class="term">file</span>, <span class="term">http</span> and <span class="term">https</span> are permitted in attributes whose names start with <span class="term">o</span> (like <span class="term">onmouseover</span>), and in these attributes that accept URLs:<br /> +<br /> + +<code class="code">    action, cite, classid, codebase, data, href, longdesc, model, pluginspage, pluginurl, src, style, usemap</code> +<br /> +<br /> +  These default sets are used when <span class="term">$config["schemes"]</span> is not set (see <a href="#s2.2">section 2.2</a>). To over-ride the defaults, <span class="term">$config["schemes"]</span> is defined as a string of semi-colon-separated sub-strings of type <span class="term">attribute: comma-separated schemes</span>. E.g., <span class="term">href: mailto, http, https; onclick: javascript; src: http, https</span>. For unspecified attributes, <span class="term">file</span>, <span class="term">http</span> and <span class="term">https</span> are permitted. This can be changed by passing schemes for <span class="term">*</span> in <span class="term">$config["schemes"]</span>. E.g., <span class="term">href: mailto, http, https; *: https, https</span>.<br /> +<br /> +  <span class="term">*</span> can be put in the list of schemes to permit all protocols. E.g., <span class="term">style: *; img: http, https</span> results in protocols not being checked in <span class="term">style</span> attribute values. However, in such cases, any relative-to-absolute URL conversion, or vice versa, (<a href="#s3.4.4">section 3.4.4</a>) is not done.<br /> +<br /> +  Thus, <em>to allow Javascript</em>, one can set <span class="term">$config["schemes"]</span> as <span class="term">href: mailto, http, https; *: http, https, javascript</span>, or <span class="term">href: mailto, http, https, javascript; *: http, https, javascript</span>, or <span class="term">*: *</span>, and so on.<br /> +<br /> +  As a side-note, one may find <span class="term">style: *</span> useful as URLs in <span class="term">style</span> attributes can be specified in a variety of ways, and the patterns that htmLawed uses to identify URLs may mistakenly identify non-URL text.<br /> +<br /> +  <strong>Note</strong>: If URL-accepting attributes other than those listed above are being allowed, then the scheme will not be checked unless the attribute name contains the string <span class="term">src</span> (e.g., <span class="term">dynsrc</span>) or starts with <span class="term">o</span> (e.g., <span class="term">onbeforecopy</span>).<br /> +<br /> +  With <span class="term">$config["safe"] = 1</span>, all URLs are disallowed in the <span class="term">style</span> attribute values.<br /> + +</div> +<div class="sub-sub-section"><h4> +<a name="s3.4.4" id="s3.4.4"></a><span class="item-no">3.4.4</span>  Absolute & relative URLs in attribute values +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed can make absolute URLs in attributes like <span class="term">href</span> relative (<span class="term">$config["abs_url"]</span> is <span class="term">-1</span>), and vice versa (<span class="term">$config["abs_url"]</span> is <span class="term">1</span>). URLs in scripts are not considered for this, and so are URLs like <span class="term">#section_6</span> (fragment), <span class="term">?name=Tim#show</span> (starting with query string), and <span class="term">;var=1?name=Tim#show</span> (starting with parameters). Further, this requires that <span class="term">$config["base_url"]</span> be set properly, with the <span class="term">://</span> and a trailing slash (<span class="term">/</span>), with no query string, etc. E.g., <span class="term">file:///D:/page/</span>, <span class="term">https://abc.com/x/y/</span>, or <span class="term">http://localhost/demo/</span> are okay, but <span class="term">file:///D:/page/?help=1</span>, <span class="term">abc.com/x/y/</span> and <span class="term">http://localhost/demo/index.htm</span> are not.<br /> +<br /> +  For making absolute URLs relative, only those URLs that have the <span class="term">$config["base_url"]</span> string at the beginning are converted. E.g., with <span class="term">$config["base_url"] = "https://abc.com/x/y/"</span>, <span class="term">https://abc.com/x/y/a.gif</span> and <span class="term">https://abc.com/x/y/z/b.gif</span> become <span class="term">a.gif</span> and <span class="term">z/b.gif</span> respectively, while <span class="term">https://abc.com/x/c.gif</span> is not changed.<br /> +<br /> +  When making relative URLs absolute, only values for scheme, network location (host-name) and path values in the base URL are inherited. See <a href="#s5.5">section 5.5</a> for more about the URL specification as per RFC <a href="http://www.ietf.org/rfc/rfc1808.txt">1808</a>.<br /> + +</div> +<div class="sub-sub-section"><h4> +<a name="s3.4.5" id="s3.4.5"></a><span class="item-no">3.4.5</span>  Lower-cased, standard attribute values +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Optionally, for standard-compliance, htmLawed (function <span class="term">hl_tag()</span>) lower-cases standard attribute values to give, e.g., <span class="term">input type="password"</span> instead of <span class="term">input type="Password"</span>, if <span class="term">$config["lc_std_val"]</span> is <span class="term">1</span>. Attribute values matching those listed below for any of the elements (plus those for the <span class="term">type</span> attribute of <span class="term">button</span> or <span class="term">input</span>) are lower-cased:<br /> +<br /> + +<code class="code">    all, baseline, bottom, button, center, char, checkbox, circle, col, colgroup, cols, data, default, file, get, groups, hidden, image, justify, left, ltr, middle, none, object, password, poly, post, preserve, radio, rect, ref, reset, right, row, rowgroup, rows, rtl, submit, text, top</code> +<br /> +<br /> + +<code class="code">    a, area, bdo, button, col, form, img, input, object, option, optgroup, param, script, select, table, td, tfoot, th, thead, tr, xml:space</code> +<br /> +<br /> +  The following <em>empty</em> (<em>minimized</em>) attributes are always assigned lower-cased values (same as the names):<br /> +<br /> + +<code class="code">    checked, compact, declare, defer, disabled, ismap, multiple, nohref, noresize, noshade, nowrap, readonly, selected</code> +<br /> + +</div> +<div class="sub-sub-section"><h4> +<a name="s3.4.6" id="s3.4.6"></a><span class="item-no">3.4.6</span>  Transformation of deprecated attributes +</h4><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  If <span class="term">$config["no_deprecated_attr"]</span> is <span class="term">0</span>, then deprecated attributes (see appendix in <a href="#s5.2">section 5.2</a>) are removed and, in most cases, their values are transformed to CSS style properties and added to the <span class="term">style</span> attributes (function <span class="term">hl_tag()</span>). Except for <span class="term">bordercolor</span> for <span class="term">table</span>, <span class="term">tr</span> and <span class="term">td</span>, the scores of proprietary attributes that were never part of any cross-browser standard are not supported.<br /> +<br /> +  <strong>Note</strong>: The attribute <span class="term">target</span> for <span class="term">a</span> is allowed even though it is not in XHTML 1.0 specs. This is because of the attribute's wide-spread use and browser-support, and because the attribute is valid in XHTML 1.1 onwards.<br /> +<br /> +  *  align - for <span class="term">img</span> with value of <span class="term">left</span> or <span class="term">right</span>, becomes, e.g., <span class="term">float: left</span>; for <span class="term">div</span> and <span class="term">table</span> with value <span class="term">center</span>, becomes <span class="term">margin: auto</span>; all others become, e.g., <span class="term">text-align: right</span><br /> +<br /> +  *  bgcolor - E.g., <span class="term">bgcolor="#ffffff"</span> becomes <span class="term">background-color: #ffffff</span><br /> +  *  border - E.g., <span class="term">height= "10"</span> becomes <span class="term">height: 10px</span><br /> +  *  bordercolor - E.g., <span class="term">bordercolor=#999999</span> becomes <span class="term">border-color: #999999;</span><br /> +  *  compact - <span class="term">font-size: 85%</span><br /> +  *  clear - E.g., 'clear="all" becomes <span class="term">clear: both</span><br /> +<br /> +  *  height - E.g., <span class="term">height= "10"</span> becomes <span class="term">height: 10px</span> and <span class="term">height="*"</span> becomes <span class="term">height: auto</span><br /> +<br /> +  *  hspace - E.g., <span class="term">hspace="10"</span> becomes <span class="term">margin-left: 10px; margin-right: 10px</span><br /> +  *  language - <span class="term">language="VBScript"</span> becomes <span class="term">type="text/vbscript"</span><br /> +  *  name - E.g., <span class="term">name="xx"</span> becomes <span class="term">id="xx"</span><br /> +  *  noshade - <span class="term">border-style: none; border: 0; background-color: gray; color: gray</span><br /> +  *  nowrap - <span class="term">white-space: nowrap</span><br /> +  *  size - E.g., <span class="term">size="10"</span> becomes <span class="term">height: 10px</span><br /> +  *  start - removed<br /> +  *  type - E.g., <span class="term">type="i"</span> becomes <span class="term">list-style-type: lower-roman</span><br /> +  *  value - removed<br /> +  *  vspace - E.g., <span class="term">vspace="10"</span> becomes <span class="term">margin-top: 10px; margin-bottom: 10px</span><br /> +  *  width - like <span class="term">height</span><br /> +<br /> +  Example input:<br /> +<br /> + +<code class="code">    <img src="j.gif" alt="image" name="dad's" /><img src="k.gif" alt="image" id="dad_off" name="dad" /></code> +<br /> + +<code class="code">    <br clear="left" /></code> +<br /> + +<code class="code">    <hr noshade size="1" /></code> +<br /> + +<code class="code">    <img name="img" src="i.gif" align="left" alt="image" hspace="10" vspace="10" width="10em" height="20" border="1" style="padding:5px;" /></code> +<br /> + +<code class="code">    <table width="50em" align="center" bgcolor="red"></code> +<br /> + +<code class="code">     <tr></code> +<br /> + +<code class="code">      <td width="20%"></code> +<br /> + +<code class="code">       <div align="center"></code> +<br /> + +<code class="code">        <h3 align="right">Section</h3></code> +<br /> + +<code class="code">        <p align="right">Para</p></code> +<br /> + +<code class="code">        <ol type="a" start="e"><li value="x">First item</li></ol></code> +<br /> + +<code class="code">       </div></code> +<br /> + +<code class="code">      </td></code> +<br /> + +<code class="code">      <td width="*"></code> +<br /> + +<code class="code">       <ol type="1"><li>First item</li></ol></code> +<br /> + +<code class="code">      </td></code> +<br /> + +<code class="code">     </tr></code> +<br /> + +<code class="code">    </table></code> +<br /> + +<code class="code">    <br clear="all" /></code> +<br /> +<br /> +  And the output with <span class="term">$config["no_deprecated_attr"] = 1</span>:<br /> +<br /> + +<code class="code">    <img src="j.gif" alt="image" /><img src="k.gif" alt="image" id="dad_off" /></code> +<br /> + +<code class="code">    <br style="clear: left;" /></code> +<br /> + +<code class="code">    <hr style="border-style: none; border: 0; background-color: gray; color: gray; size: 1px;" /></code> +<br /> + +<code class="code">    <img src="i.gif" alt="image" width="10em" height="20" style="padding:5px; float: left; margin-left: 10px; margin-right: 10px; margin-top: 10px; margin-bottom: 10px; border: 1px;" id="img" /></code> +<br /> + +<code class="code">    <table width="50em" style="margin: auto; background-color: red;"></code> +<br /> + +<code class="code">     <tr></code> +<br /> + +<code class="code">      <td style="width: 20%;"></code> +<br /> + +<code class="code">       <div style="margin: auto;"></code> +<br /> + +<code class="code">        <h3 style="text-align: right;">Section</h3></code> +<br /> + +<code class="code">        <p style="text-align: right;">Para</p></code> +<br /> + +<code class="code">        <ol style="list-style-type: lower-latin;"><li>First item</li></ol></code> +<br /> + +<code class="code">       </div></code> +<br /> + +<code class="code">      </td></code> +<br /> + +<code class="code">      <td style="width: auto;"></code> +<br /> + +<code class="code">       <ol style="list-style-type: decimal;"><li>First item</li></ol></code> +<br /> + +<code class="code">      </td></code> +<br /> + +<code class="code">     </tr></code> +<br /> + +<code class="code">    </table></code> +<br /> + +<code class="code">    <br style="clear: both;" /></code> +<br /> +<br /> +  For <span class="term">lang</span>, deprecated in XHTML 1.1, transformation is taken care of through <span class="term">$config["xml:lang"]</span>; see <a href="#s3.4.1">section 3.4.1</a>.<br /> +<br /> +  The attribute <span class="term">name</span> is deprecated in <span class="term">form</span>, <span class="term">iframe</span>, and <span class="term">img</span>, and is replaced with <span class="term">id</span> if an <span class="term">id</span> attribute doesn't exist and if the <span class="term">name</span> value is appropriate for <span class="term">id</span>. For such replacements for <span class="term">a</span> and <span class="term">map</span>, for which the <span class="term">name</span> attribute is deprecated in XHTML 1.1, <span class="term">$config["no_deprecated_attr"]</span> should be set to <span class="term">2</span> (when set to <span class="term">1</span>, for these two elements, the <span class="term">name</span> attribute is retained).<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.4.7" id="s3.4.7"></a><span class="item-no">3.4.7</span>  Anti-spam & <span class="term">href</span> +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed (function <span class="term">hl_tag()</span>) can check the <span class="term">href</span> attribute values (link addresses) as an anti-spam (email or link spam) measure.<br /> +<br /> +  If <span class="term">$config["anti_mail_spam"]</span> is not <span class="term">0</span>, the <span class="term">@</span> of email addresses in <span class="term">href</span> values like <span class="term">mailto:a@b.com</span> is replaced with text specified by <span class="term">$config["anti_mail_spam"]</span>. The text should be of a form that makes it clear to others that the address needs to be edited before a mail is sent; e.g., <span class="term"><remove_this_antispam>@</span> (makes the example address <span class="term">a<remove_this_antispam>@b.com</span>).<br /> +<br /> +  For regular links, one can choose to have a <span class="term">rel</span> attribute with <span class="term">nofollow</span> in its value (which tells some search engines to not follow a link). This can discourage link spammers. Additionally, or as an alternative, one can choose to empty the <span class="term">href</span> value altogether (disable the link).<br /> +<br /> +  For use of these options, <span class="term">$config["anti_link_spam"]</span> should be set as an array with values <span class="term">regex1</span> and <span class="term">regex2</span>, both or one of which can be empty (like <span class="term">array("", "regex2")</span>) to indicate that that option is not to be used. Otherwise, <span class="term">regex1</span> or <span class="term">regex2</span> should be PHP- and PCRE-compatible regular expression patterns: <span class="term">href</span> values will be matched against them and those matching the pattern will accordingly be treated.<br /> +<br /> +  Note that the regular expressions should have <em>delimiters</em>, and be well-formed and preferably fast. Absolute efficiency/accuracy is often not needed.<br /> +<br /> +  An example, to have a <span class="term">rel</span> attribute with <span class="term">nofollow</span> for all links, and to disable links that do not point to domains <span class="term">abc.com</span> and <span class="term">xyz.org</span>:<br /> +<br /> + +<code class="code">    $config["anti_link_spam"] = array('`.`', '`://\W*(?!(abc\.com|xyz\.org))`');</code> +<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.4.8" id="s3.4.8"></a><span class="item-no">3.4.8</span>  Inline style properties +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed can check URL schemes and dynamic expressions (to guard against Javascript, etc., script-based insecurities) in inline CSS style property values in the <span class="term">style</span> attributes. (CSS properties like <span class="term">background-image</span> that accept URLs in their values are noted in <a href="#s5.3">section 5.3</a>.) Dynamic CSS expressions that allow scripting in the IE browser, and can be a vulnerability, can be removed from property values by setting <span class="term">$config["css_expression"]</span> to <span class="term">1</span> (default setting).<br /> +<br /> +  <strong>Note</strong>: Because of the various ways of representing characters in attribute values (URL-escapement, entitification, etc.), htmLawed might alter the values of the <span class="term">style</span> attribute values, and may even falsely identify dynamic CSS expressions and URL schemes in them. If this is an important issue, checking of URLs and dynamic expressions can be turned off (<span class="term">$config["schemes"] = "...style:*..."</span>, see <a href="#s3.4.3">section 3.4.3</a>, and <span class="term">$config["css_expression"] = 0</span>). Alternately, admins can use their own custom function for finer handling of <span class="term">style</span> values through the <span class="term">hook_tag</span> parameter (see <a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  It is also possible to have htmLawed let through any <span class="term">style</span> value by setting <span class="term">$config["style_pass"]</span> to <span class="term">1</span>.<br /> +<br /> +  As such, it is better to set up a CSS file with class declarations, disallow the <span class="term">style</span> attribute, set a <span class="term">$spec</span> rule (see <a href="#s2.3">section 2.3</a>) for <span class="term">class</span> for the <span class="term">oneof</span> or <span class="term">match</span> parameter, and ask writers to make use of the <span class="term">class</span> attribute.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.4.9" id="s3.4.9"></a><span class="item-no">3.4.9</span>  Hook function for tag content +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  It is possible to utilize a custom hook function to alter the tag content htmLawed has finalized (i.e., after it has checked/corrected for required attributes, transformed attributes, lower-cased attribute names, etc.).<br /> +<br /> +  When <span class="term">$config</span> parameter <span class="term">hook_tag</span> is set to the name of a function, htmLawed (function <span class="term">hl_tag()</span>) will pass on the element name, and the <em>finalized</em> attribute name-value pairs as array elements to the function. The function is expected to return the full opening tag string like <span class="term"><element_name attribute_1_name="attribute_1_value"...></span> (for empty elements like <span class="term">img</span> and <span class="term">input</span>, the element-closing slash <span class="term">/</span> should also be included).<br /> +<br /> +  This is a <strong>powerful functionality</strong> that can be exploited for various objectives: consolidate-and-convert inline <span class="term">style</span> attributes to <span class="term">class</span>, convert <span class="term">embed</span> elements to <span class="term">object</span>, permit only one <span class="term">caption</span> element in a <span class="term">table</span> element, disallow embedding of certain types of media, <strong>inject HTML</strong>, use <a href="http://csstidy.sourceforge.net">CSSTidy</a> to sanitize <span class="term">style</span> attribute values, etc.<br /> +<br /> +  As an example, the custom hook code below can be used to force a series of specifically ordered <span class="term">id</span> attributes on all elements, and a specific <span class="term">param</span> element inside all <span class="term">object</span> elements:<br /> +<br /> + +<code class="code">    function my_tag_function($element, $attribute_array){</code> +<br /> + +<code class="code">      static $id = 0;</code> +<br /> + +<code class="code">      // Remove any duplicate element</code> +<br /> + +<code class="code">      if($element == 'param' && isset($attribute_array['allowscriptaccess'])){</code> +<br /> + +<code class="code">        return '';</code> +<br /> + +<code class="code">      }</code> +<br /> +<br /> + +<code class="code">      $new_element = '';</code> +<br /> +<br /> + +<code class="code">      // Force a serialized ID number</code> +<br /> + +<code class="code">      $attribute_array['id'] = 'my_'. $id;</code> +<br /> + +<code class="code">      ++$id;</code> +<br /> +<br /> + +<code class="code">      // Inject param for allowscriptaccess</code> +<br /> + +<code class="code">      if($element == 'object'){</code> +<br /> + +<code class="code">        $new_element = '<param id='my_'. $id; allowscriptaccess="never" />';</code> +<br /> + +<code class="code">        ++$id;</code> +<br /> + +<code class="code">      }</code> +<br /> +<br /> + +<code class="code">      $string = '';</code> +<br /> + +<code class="code">      foreach($attribute_array as $k=>$v){</code> +<br /> + +<code class="code">        $string .= " {$k}=\"{$v}\"";</code> +<br /> + +<code class="code">      }</code> +<br /> + +<code class="code">      return "<{$element}{$string}". (isset($in_array($element, $empty_elements) ? ' /' : ''). '>'. $new_element;</code> +<br /> + +<code class="code">    }</code> +<br /> +<br /> +  The <span class="term">hook_tag</span> parameter is different from the <span class="term">hook</span> parameter (<a href="#s3.7">section 3.7</a>).<br /> +<br /> +  Snippets of hook function code developed by others may be available on the <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed">htmLawed</a> website.<br /> + +</div> +</div> +<div class="sub-section"><h3> +<a name="s3.5" id="s3.5"></a><span class="item-no">3.5</span>  Simple configuration directive for most valid XHTML +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  If <span class="term">$config["valid_xhtml"]</span> is set to <span class="term">1</span>, some relevant <span class="term">$config</span> parameters (indicated by <span class="term">~</span> in <a href="#s2.2">section 2.2</a>) are auto-adjusted. This allows one to pass the <span class="term">$config</span> argument with a simpler value. If a value for a parameter auto-set through <span class="term">valid_xhtml</span> is still manually provided, then that value will over-ride the auto-set value.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.6" id="s3.6"></a><span class="item-no">3.6</span>  Simple configuration directive for most <em>safe</em> HTML +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  <em>Safe</em> HTML refers to HTML that is restricted to reduce the vulnerability for scripting attacks (such as XSS) based on HTML code which otherwise may still be legal and compliant with the HTML standard specs. When elements such as <span class="term">script</span> and <span class="term">object</span>, and attributes such as <span class="term">onmouseover</span> and <span class="term">style</span> are allowed in the input text, an input writer can introduce malevolent HTML code. Note that what is considered <span class="term">safe</span> depends on the nature of the web application and the trust-level accorded to its users.<br /> +<br /> +  htmLawed allows an admin to use <span class="term">$config["safe"]</span> to auto-adjust multiple <span class="term">$config</span> parameters (such as <span class="term">elements</span> which declares the allowed element-set), which otherwise would have to be manually set. The relevant parameters are indicated by <span class="term">"</span> in <a href="#s2.2">section 2.2</a>). Thus, one can pass the <span class="term">$config</span> argument with a simpler value.<br /> +<br /> +  With the value of <span class="term">1</span>, htmLawed considers <span class="term">CDATA</span> sections and HTML comments as plain text, and prohibits the <span class="term">applet</span>, <span class="term">embed</span>, <span class="term">iframe</span>, <span class="term">object</span> and <span class="term">script</span> elements, and the <span class="term">on*</span> attributes like <span class="term">onclick</span>. ( There are <span class="term">$config</span> parameters like <span class="term">css_expression</span> that are not affected by the value set for <span class="term">safe</span> but whose default values still contribute towards a more <em>safe</em> output.) Further, URLs with schemes (see <a href="#s3.4.3">section 3.4.3</a>) are neutralized so that, e.g., <span class="term">style="moz-binding:url(http://danger)"</span> becomes <span class="term">style="moz-binding:url(denied:http://danger)"</span> while <span class="term">style="moz-binding:url(ok)"</span> remains intact.<br /> +<br /> +  Admins, however, may still want to completely deny the <span class="term">style</span> attribute, e.g., with code like<br /> +<br /> + +<code class="code">    $processed = htmLawed($text, array('safe'=>1, 'deny_attribute'=>'style'));</code> +<br /> +<br /> +  If a value for a parameter auto-set through <span class="term">safe</span> is still manually provided, then that value can over-ride the auto-set value. E.g., with <span class="term">$config["safe"] = 1</span> and <span class="term">$config["elements"] = "*+script"</span>, <span class="term">script</span>, but not <span class="term">applet</span>, is allowed.<br /> +<br /> +  A page illustrating the efficacy of htmLawed's anti-XSS abilities with <span class="term">safe</span> set to <span class="term">1</span> against XSS vectors listed by <a href="http://ha.ckers.org/xss.html">RSnake</a> may be available <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed/rsnake/RSnakeXSSTest.htm">here</a>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.7" id="s3.7"></a><span class="item-no">3.7</span>  Using a hook function +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  If <span class="term">$config["hook"]</span> is not set to <span class="term">0</span>, then htmLawed will allow preliminarily processed input to be altered by a hook function named by <span class="term">$config["hook"]</span> before starting the main work (but after handling of characters, entities, HTML comments and <span class="term">CDATA</span> sections -- see code for function <span class="term">htmLawed()</span>).<br /> +<br /> +  The hook function also allows one to alter the <em>finalized</em> values of <span class="term">$config</span> and <span class="term">$spec</span>.<br /> +<br /> +  Note that the <span class="term">hook</span> parameter is different from the <span class="term">hook_tag</span> parameter (<a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  Snippets of hook function code developed by others may be available on the <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed">htmLawed</a> website.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.8" id="s3.8"></a><span class="item-no">3.8</span>  Obtaining <em>finalized</em> parameter values +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed can assign the <em>finalized</em> <span class="term">$config</span> and <span class="term">$spec</span> values to a variable named by <span class="term">$config["show_setting"]</span>. The variable, made global by htmLawed, is set as an array with three keys: <span class="term">config</span>, with the <span class="term">$config</span> value, <span class="term">spec</span>, with the <span class="term">$spec</span> value, and <span class="term">time</span>, with a value that is the Unix time (the output of PHP's <span class="term">microtime()</span> function) when the value was assigned. Admins should use a PHP-compliant variable name (e.g., one that does not begin with a numerical digit) that does not conflict with variable names in their non-htmLawed code.<br /> +<br /> +  The values, which are also post-hook function (if any), can be used to auto-generate information (on, e.g., the elements that are permitted) for input writers.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s3.9" id="s3.9"></a><span class="item-no">3.9</span>  Retaining non-HTML tags in input with mixed markup +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  htmLawed does not remove certain characters that though invalid are nevertheless discouraged in HTML documents as per the specs (see <a href="#s5.1">section 5.1</a>). This can be utilized to deal with input that contains mixed markup. Input that may have HTML markup as well as some other markup that is based on the <span class="term"><</span>, <span class="term">></span> and <span class="term">&</span> characters is considered to have mixed markup. The non-HTML markup can be rather proprietary (like markup for emoticons/smileys), or standard (like MathML or SVG). Or it can be programming code meant for execution/evaluation (such as embedded PHP code).<br /> +<br /> +  To deal with such mixed markup, the input text can be pre-processed to hide the non-HTML markup by specifically replacing the <span class="term"><</span>, <span class="term">></span> and <span class="term">&</span> characters with some of the HTML-discouraged characters (see <a href="#s3.1.2">section 3.1.2</a>). Post-htmLawed processing, the replacements are reverted.<br /> +<br /> +  An example (mixed HTML and PHP code in input text):<br /> +<br /> + +<code class="code">    $text = preg_replace('`<\?php(.+?)\?>`sm', "\x83?php\\1?\x84", $text);</code> +<br /> + +<code class="code">    $processed = htmLawed($text);</code> +<br /> + +<code class="code">    $processed = preg_replace('`\x83\?php(.+?)\?\x84`sm', '<?php$1?>', $processed);</code> +<br /> +<br /> +  This code will not work if <span class="term">$config["clean_ms_char"]</span> is set to <span class="term">1</span> (<a href="#s3.1">section 3.1</a>), in which case one should instead deploy a hook function (<a href="#s3.7">section 3.7</a>). (htmLawed internally uses certain control characters, code-points <span class="term">1</span> to <span class="term">7</span>, and use of these characters as markers in the logic of hook functions may cause issues.)<br /> +<br /> +  Admins may also be able to use <span class="term">$config["and_mark"]</span> to deal with such mixed markup; see <a href="#s3.2">section 3.2</a>.<br /> + +</div> +</div> +<div class="section"><h2> +<a name="s4" id="s4"></a><span class="item-no">4</span>  Other +</h2><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<div class="sub-section"><h3> +<a name="s4.1" id="s4.1"></a><span class="item-no">4.1</span>  Support +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  A careful re-reading of this documentation will very likely answer your questions.<br /> +<br /> +  Software updates and forum-based community-support may be found at <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed">http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed</a>. For general PHP issues (not htmLawed-specific), support may be found through internet searches and at <a href="http://php.net">http://php.net</a>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.2" id="s4.2"></a><span class="item-no">4.2</span>  Known issues +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  See <a href="#s2.8">section 2.8</a>.<br /> +<br /> +  Readers are advised to cross-check information given in this document.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.3" id="s4.3"></a><span class="item-no">4.3</span>  Change-log +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  (The release date for the downloadable package of files containing documentation, demo script, test-cases, etc., besides the <span class="term">htmLawed.php</span> file may be updated independently if the secondary files are revised.)<br /> +<br /> +  <em>Version number - Release date. Notes</em><br /> +<br /> +  1.1.8.1 - 16 July 2009. Minor code-change to fix a PHP error notice<br /> +<br /> +  1.1.8 - 23 April 2009. Parameter <span class="term">deny_attribute</span> now accepts the wild-card <span class="term">*</span>, making it simpler to specify its value when all but a few attributes are being denied; fixed a bug in interpreting <span class="term">$spec</span><br /> +<br /> +  1.1.7 - 11-12 March 2009. Attributes globally denied through <span class="term">deny_attribute</span> can be allowed element-specifically through <span class="term">$spec</span>; <span class="term">$config["style_pass"]</span> allowing letting through any <span class="term">style</span> value introduced; altered logic to catch certain types of dynamic crafted CSS expressions<br /> +<br /> +  1.1.3-6 - 28-31 January - 4 February 2009. Altered logic to catch certain types of dynamic crafted CSS expressions<br /> +<br /> +  1.1.2 - 22 January 2009. Fixed bug in parsing of <span class="term">font</span> attributes during tag transformation<br /> +<br /> +  1.1.1 - 27 September 2008. Better nesting correction when omitable closing tags are absent<br /> +<br /> +  1.1 - 29 June 2008. <span class="term">$config["hook_tag"]</span> and <span class="term">$config["format"]</span> introduced for custom tag/attribute check/modification/injection and output compaction/beautification; fixed a regex-in-$spec parsing bug<br /> +<br /> +  1.0.9 - 11 June 2008. Fixed bug in invalid HTML code-point entity check<br /> +<br /> +  1.0.8 - 15 May 2008. <span class="term">bordercolor</span> attribute for <span class="term">table</span>, <span class="term">td</span> and <span class="term">tr</span><br /> +<br /> +  1.0.7 - 1 May 2008. Support for <span class="term">wmode</span> attribute for <span class="term">embed</span>; <span class="term">$config["show_setting"]</span> introduced; improved <span class="term">$config["elements"]</span> evaluation<br /> +<br /> +  1.0.6 - 20 April 2008. <span class="term">$config["and_mark"]</span> introduced<br /> +<br /> +  1.0.5 - 12 March 2008. <span class="term">style</span> URL schemes essentially disallowed when $config <span class="term">safe</span> is on; improved regex for CSS expression search<br /> +<br /> +  1.0.4 - 10 March 2008. Improved corrections for <span class="term">blockquote</span>, <span class="term">form</span>, <span class="term">map</span> and <span class="term">noscript</span><br /> +<br /> +  1.0.3 - 3 March 2008. Character entities for soft-hyphens are now replaced with spaces (instead of being removed); a bug allowing <span class="term">td</span> directly inside <span class="term">table</span> fixed; <span class="term">safe</span> <span class="term">$config</span> parameter added<br /> +<br /> +  1.0.2 - 13 February 2008. Improved implementation of <span class="term">$config["keep_bad"]</span><br /> +<br /> +  1.0.1 - 7 November 2007. Improved regex for identifying URLs, protocols and dynamic expressions (<span class="term">hl_tag()</span> and <span class="term">hl_prot()</span>); no error display with <span class="term">hl_regex()</span><br /> +<br /> +  1.0 - 2 November 2007. First release<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.4" id="s4.4"></a><span class="item-no">4.4</span>  Testing +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  To test htmLawed using a form interface, a <a href="htmLawedTest.php">demo</a> web-page is provided with the htmLawed distribution (<span class="term">htmLawed.php</span> and <span class="term">htmLawedTest.php</span> should be in the same directory on the web-server). A file with <a href="htmLawed_TESTCASE.txt">test-cases</a> is also provided.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.5" id="s4.5"></a><span class="item-no">4.5</span>  Upgrade, & old versions +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Upgrading is as simple as replacing the previous version of <span class="term">htmLawed.php</span> (assuming it was not modified for customized features). As htmLawed output is almost always used in static documents, upgrading should not affect old, finalized content.<br /> +<br /> +  Old versions of htmLawed may be available online. E.g., for version 1.0, check <a href="http://www.bioinformatics.org/phplabware/downloads/htmLawed1.zip">http://www.bioinformatics.org/phplabware/downloads/htmLawed1.zip</a>, for 1.1.1, htmLawed111.zip, and for 1.1.10, htmLawed1110.zip.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.6" id="s4.6"></a><span class="item-no">4.6</span>  Comparison with <span class="term">HTMLPurifier</span> +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  The HTMLPurifier PHP library by Edward Yang is a very good HTML filtering script that uses object oriented PHP code. Compared to htmLawed, it:<br /> +<br /> +  *  does not support PHP versions older than 5.0 (HTMLPurifier dropped PHP 4 support after version 2)<br /> +<br /> +  *  is 15-20 times bigger (scores of files totalling more than 750 kb)<br /> +<br /> +  *  consumes 10-15 times more RAM memory (just including the HTMLPurifier files without calling the filter requires a few MBs of memory)<br /> +<br /> +  *  is expectedly slower<br /> +<br /> +  *  does not allow admins to fully allow all valid HTML (because of incomplete HTML support, it always considers elements like <span class="term">script</span> illegal)<br /> +<br /> +  *  lacks many of the extra features of htmLawed (like entity conversions and code compaction/beautification)<br /> +<br /> +  *  has poor documentation<br /> +<br /> +  However, HTMLPurifier has finer checks for character encodings and attribute values, and can log warnings and errors. Visit the HTMLPurifier <a href="http://htmlpurifier.org">website</a> for updated information.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.7" id="s4.7"></a><span class="item-no">4.7</span>  Use through application plug-ins/modules +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Plug-ins/modules to implement htmLawed in applications such as Drupal and DokuWiki may have been developed. Please check the application websites and the forum on the htmLawed <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed">site</a>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.8" id="s4.8"></a><span class="item-no">4.8</span>  Use in non-PHP applications +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Non-PHP applications written in Python, Ruby, etc., may be able to use htmLawed through system calls to the PHP engine. Such code may have been documented on the internet. Also check the forum on the htmLawed <a href="http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed">site</a>.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.9" id="s4.9"></a><span class="item-no">4.9</span>  Donate +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  A donation in any currency and amount to appreciate or support this software can be sent by <a href="http://paypal.com">PayPal</a> to this email address: drpatnaik at yahoo dot com.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s4.10" id="s4.10"></a><span class="item-no">4.10</span>  Acknowledgements +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Bryan Blakey, Ulf Harnhammer, Gareth Heyes, Lukasz Pilorz, Shelley Powers, Edward Yang, and many anonymous users.<br /> +<br /> +  Thank you!<br /> + +</div> +</div> +<div class="section"><h2> +<a name="s5" id="s5"></a><span class="item-no">5</span>  Appendices +</h2><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<div class="sub-section"><h3> +<a name="s5.1" id="s5.1"></a><span class="item-no">5.1</span>  Characters discouraged in XHTML +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Characters represented by the following hexadecimal code-points are <em>not</em> invalid, even though some validators may issue messages stating otherwise.<br /> +<br /> +  <span class="term">7f</span> to <span class="term">84</span>, <span class="term">86</span> to <span class="term">9f</span>, <span class="term">fdd0</span> to <span class="term">fddf</span>, <span class="term">1fffe</span>, <span class="term">1ffff</span>, <span class="term">2fffe</span>, <span class="term">2ffff</span>, <span class="term">3fffe</span>, <span class="term">3ffff</span>, <span class="term">4fffe</span>, <span class="term">4ffff</span>, <span class="term">5fffe</span>, <span class="term">5ffff</span>, <span class="term">6fffe</span>, <span class="term">6ffff</span>, <span class="term">7fffe</span>, <span class="term">7ffff</span>, <span class="term">8fffe</span>, <span class="term">8ffff</span>, <span class="term">9fffe</span>, <span class="term">9ffff</span>, <span class="term">afffe</span>, <span class="term">affff</span>, <span class="term">bfffe</span>, <span class="term">bffff</span>, <span class="term">cfffe</span>, <span class="term">cffff</span>, <span class="term">dfffe</span>, <span class="term">dffff</span>, <span class="term">efffe</span>, <span class="term">effff</span>, <span class="term">ffffe</span>, <span class="term">fffff</span>, <span class="term">10fffe</span> and <span class="term">10ffff</span><br /> + +</div> +<div class="sub-section"><h3> +<a name="s5.2" id="s5.2"></a><span class="item-no">5.2</span>  Valid attribute-element combinations +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Valid attribute-element combinations as per W3C specs.<br /> +<br /> +  *  includes deprecated attributes (marked <span class="term">^</span>), attributes for the non-standard <span class="term">embed</span> element (marked <span class="term">*</span>), and the proprietary <span class="term">bordercolor</span> (marked <span class="term">~</span>)<br /> +  *  only non-frameset, HTML body elements<br /> +  *  <span class="term">name</span> for <span class="term">a</span> and <span class="term">map</span>, and <span class="term">lang</span> are invalid in XHTML 1.1<br /> +  *  <span class="term">target</span> is valid for <span class="term">a</span> in XHTML 1.1 and higher<br /> +  *  <span class="term">xml:space</span> is only for XHTML 1.1<br /> +<br /> +  abbr - td, th<br /> +  accept - form, input<br /> +  accept-charset - form<br /> +  accesskey - a, area, button, input, label, legend, textarea<br /> +  action - form<br /> +  align - caption^, embed, applet, iframe, img^, input^, object^, legend^, table^, hr^, div^, h1^, h2^, h3^, h4^, h5^, h6^, p^, col, colgroup, tbody, td, tfoot, th, thead, tr<br /> +  alt - applet, area, img, input<br /> +  archive - applet, object<br /> +  axis - td, th<br /> +  bgcolor - embed, table^, tr^, td^, th^<br /> +  border - table, img^, object^<br /> +  bordercolor~ - table, td, tr<br /> +  cellpadding - table<br /> +  cellspacing - table<br /> +  char - col, colgroup, tbody, td, tfoot, th, thead, tr<br /> +  charoff - col, colgroup, tbody, td, tfoot, th, thead, tr<br /> +  charset - a, script<br /> +  checked - input<br /> +  cite - blockquote, q, del, ins<br /> +  classid - object<br /> +  clear - br^<br /> +  code - applet<br /> +  codebase - object, applet<br /> +  codetype - object<br /> +  color - font<br /> +  cols - textarea<br /> +  colspan - td, th<br /> +  compact - dir, dl^, menu, ol^, ul^<br /> +  coords - area, a<br /> +  data - object<br /> +  datetime - del, ins<br /> +  declare - object<br /> +  defer - script<br /> +  dir - bdo<br /> +  disabled - button, input, optgroup, option, select, textarea<br /> +  enctype - form<br /> +  face - font<br /> +  for - label<br /> +  frame - table<br /> +  frameborder - iframe<br /> +  headers - td, th<br /> +  height - embed, iframe, td^, th^, img, object, applet<br /> +  href - a, area<br /> +  hreflang - a<br /> +  hspace - applet, img^, object^<br /> +  ismap - img, input<br /> +  label - option, optgroup<br /> +  language - script^<br /> +  longdesc - img, iframe<br /> +  marginheight - iframe<br /> +  marginwidth - iframe<br /> +  maxlength - input<br /> +  method - form<br /> +  model* - embed<br /> +  multiple - select<br /> +  name - button, embed, textarea, applet^, select, form^, iframe^, img^, a^, input, object, map^, param<br /> +  nohref - area<br /> +  noshade - hr^<br /> +  nowrap - td^, th^<br /> +  object - applet<br /> +  onblur - a, area, button, input, label, select, textarea<br /> +  onchange - input, select, textarea<br /> +  onfocus - a, area, button, input, label, select, textarea<br /> +  onreset - form<br /> +  onselect - input, textarea<br /> +  onsubmit - form<br /> +  pluginspage* - embed<br /> +  pluginurl* - embed<br /> +  prompt - isindex<br /> +  readonly - textarea, input<br /> +  rel - a<br /> +  rev - a<br /> +  rows - textarea<br /> +  rowspan - td, th<br /> +  rules - table<br /> +  scope - td, th<br /> +  scrolling - iframe<br /> +  selected - option<br /> +  shape - area, a<br /> +  size - hr^, font, input, select<br /> +  span - col, colgroup<br /> +  src - embed, script, input, iframe, img<br /> +  standby - object<br /> +  start - ol^<br /> +  summary - table<br /> +  tabindex - a, area, button, input, object, select, textarea<br /> +  target - a^, area, form<br /> +  type - a, embed, object, param, script, input, li^, ol^, ul^, button<br /> +  usemap - img, input, object<br /> +  valign - col, colgroup, tbody, td, tfoot, th, thead, tr<br /> +  value - input, option, param, button, li^<br /> +  valuetype - param<br /> +  vspace - applet, img^, object^<br /> +  width - embed, hr^, iframe, img, object, table, td^, th^, applet, col, colgroup, pre^<br /> +  wmode - embed<br /> +  xml:space - pre, script, style<br /> +<br /> +  These are allowed in all but the shown elements:<br /> +<br /> +  class - param, script<br /> +  dir - applet, bdo, br, iframe, param, script<br /> +  id - script<br /> +  lang - applet, br, iframe, param, script<br /> +  onclick - applet, bdo, br, font, iframe, isindex, param, script<br /> +  ondblclick - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onkeydown - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onkeypress - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onkeyup - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onmousedown - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onmousemove - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onmouseout - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onmouseover - applet, bdo, br, font, iframe, isindex, param, script<br /> +  onmouseup - applet, bdo, br, font, iframe, isindex, param, script<br /> +  style - param, script<br /> +  title - param, script<br /> +  xml:lang - applet, br, iframe, param, script<br /> + +</div> +<div class="sub-section"><h3> +<a name="s5.3" id="s5.3"></a><span class="item-no">5.3</span>  CSS 2.1 properties accepting URLs +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  background<br /> +  background-image<br /> +  content<br /> +  cue-after<br /> +  cue-before<br /> +  cursor<br /> +  list-style<br /> +  list-style-image<br /> +  play-during<br /> + +</div> +<div class="sub-section"><h3> +<a name="s5.4" id="s5.4"></a><span class="item-no">5.4</span>  Microsoft Windows 1252 character replacements +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Key: <span class="term">d</span> double, <span class="term">l</span> left, <span class="term">q</span> quote, <span class="term">r</span> right, <span class="term">s.</span> single<br /> +<br /> +  Code-point (decimal) - hexadecimal value - replacement entity - represented character<br /> +<br /> +  127 - 7f - (removed) - (not used)<br /> +  128 - 80 - &#8364; - euro<br /> +  129 - 81 - (removed) - (not used)<br /> +  130 - 82 - &#8218; - baseline s. q<br /> +  131 - 83 - &#402; - florin<br /> +  132 - 84 - &#8222; - baseline d q<br /> +  133 - 85 - &#8230; - ellipsis<br /> +  134 - 86 - &#8224; - dagger<br /> +  135 - 87 - &#8225; - d dagger<br /> +  136 - 88 - &#710; - circumflex accent<br /> +  137 - 89 - &#8240; - permile<br /> +  138 - 8a - &#352; - S Hacek<br /> +  139 - 8b - &#8249; - l s. guillemet<br /> +  140 - 8c - &#338; - OE ligature<br /> +  141 - 8d - (removed) - (not used)<br /> +  142 - 8e - &#381; - Z dieresis<br /> +  143 - 8f - (removed) - (not used)<br /> +  144 - 90 - (removed) - (not used)<br /> +  145 - 91 - &#8216; - l s. q<br /> +  146 - 92 - &#8217; - r s. q<br /> +  147 - 93 - &#8220; - l d q<br /> +  148 - 94 - &#8221; - r d q<br /> +  149 - 95 - &#8226; - bullet<br /> +  150 - 96 - &#8211; - en dash<br /> +  151 - 97 - &#8212; - em dash<br /> +  152 - 98 - &#732; - tilde accent<br /> +  153 - 99 - &#8482; - trademark<br /> +  154 - 9a - &#353; - s Hacek<br /> +  155 - 9b - &#8250; - r s. guillemet<br /> +  156 - 9c - &#339; - oe ligature<br /> +  157 - 9d - (removed) - (not used)<br /> +  158 - 9e - &#382; - z dieresis<br /> +  159 - 9f - &#376; - Y dieresis<br /> + +</div> +<div class="sub-section"><h3> +<a name="s5.5" id="s5.5"></a><span class="item-no">5.5</span>  URL format +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  An <em>absolute</em> URL has a <span class="term">protocol</span> or <span class="term">scheme</span>, a <span class="term">network location</span> or <span class="term">hostname</span>, and, optional <span class="term">path</span>, <span class="term">parameters</span>, <span class="term">query</span> and <span class="term">fragment</span> segments. Thus, an absolute URL has this generic structure:<br /> +<br /> + +<code class="code">    (scheme) : (//network location) /(path) ;(parameters) ?(query) #(fragment)</code> +<br /> +<br /> +  The schemes can only contain letters, digits, <span class="term">+</span>, <span class="term">.</span> and <span class="term">-</span>. Hostname is the portion after the <span class="term">//</span> and up to the first <span class="term">/</span> (if any; else, up to the end) when <span class="term">:</span> is followed by a <span class="term">//</span> (e.g., <span class="term">abc.com</span> in <span class="term">ftp://abc.com/def</span>); otherwise, it consists of everything after the <span class="term">:</span> (e.g., <span class="term">def@abc.com</span> in mailto:def@abc.com').<br /> +<br /> +  <em>Relative</em> URLs do not have explicit schemes and network locations; such values are inherited from a <em>base</em> URL.<br /> + +</div> +<div class="sub-section"><h3> +<a name="s5.6" id="s5.6"></a><span class="item-no">5.6</span>  Brief on htmLawed code +</h3><span class="totop"><a href="#peak">(to top)</a></span><br style="clear: both;" /> +<br /> +  Much of the code's logic and reasoning can be understood from the documentation above.<br /> +<br /> +  The <strong>output</strong> of htmLawed is a text string containing the processed input. There is no custom error tracking.<br /> +<br /> +  <strong>Function arguments</strong> for htmLawed are:<br /> +<br /> +  *  <span class="term">$in</span> - 1st argument; a text string; the <strong>input text</strong> to be processed. Any extraneous slashes added by PHP when <em>magic quotes</em> are enabled should be removed beforehand using PHP's <span class="term">stripslashes()</span> function.<br /> +<br /> +  *  <span class="term">$config</span> - 2nd argument; an associative array; optional (named <span class="term">$C</span> in htmLawed code). The array has keys with names like <span class="term">balance</span> and <span class="term">keep_bad</span>, and the values, which can be boolean, string, or array, depending on the key, are read to accordingly set the <strong>configurable parameters</strong> (indicated by the keys). All configurable parameters receive some default value if the value to be used is not specified by the user through <span class="term">$config</span>. <em>Finalized</em> <span class="term">$config</span> is thus a filtered and possibly larger array.<br /> +<br /> +  *  <span class="term">$spec</span> - 3rd argument; a text string; optional. The string has rules, written in an htmLawed-designated format, <strong>specifying</strong> element-specific attribute and attribute value restrictions. Function <span class="term">hl_spec()</span> is used to convert the string to an associative-array for internal use. <em>Finalized</em> <span class="term">$spec</span> is thus an array.<br /> +<br /> +  <em>Finalized</em> <span class="term">$config</span> and <span class="term">$spec</span> are made <strong>global variables</strong> while htmLawed is at work. Values of any pre-existing global variables with same names are noted, and their values are restored after htmLawed finishes processing the input (to capture the <em>finalized</em> values, the <span class="term">show_settings</span> parameter of <span class="term">$config</span> should be used). Depending on <span class="term">$config</span>, another global variable <span class="term">hl_Ids</span>, to track <span class="term">id</span> attribute values for uniqueness, may be set. Unlike the other two variables, this one is not reset (or unset) post-processing.<br /> +<br /> +  Except for the main function <span class="term">htmLawed()</span> and the functions <span class="term">kses()</span> and <span class="term">kses_hook()</span>, htmLawed's functions are <strong>name-spaced</strong> using the <span class="term">hl_</span> prefix. The <strong>functions</strong> and their roles are:<br /> +<br /> +  *  <span class="term">hl_attrval</span> - checking attribute values against $spec<br /> +  *  <span class="term">hl_bal</span> - tag balancing<br /> +  *  <span class="term">hl_cmtcd</span> - handling CDATA sections and HTML comments<br /> +  *  <span class="term">hl_ent</span> - entity handling<br /> +  *  <span class="term">hl_prot</span> - checking a URL scheme/protocol<br /> +  *  <span class="term">hl_regex</span> - checking syntax of a regular expression<br /> +  *  <span class="term">hl_spec</span> - converting user-supplied $spec value to one used by htmLawed internally<br /> +  *  <span class="term">hl_tag</span> - handling tags<br /> +  *  <span class="term">hl_tag2</span> - transforming tags<br /> +  *  <span class="term">hl_tidy</span> - compact/beautify HTML<br /> +  *  <span class="term">hl_version</span> - reporting htmLawed version<br /> +  *  <span class="term">htmLawed</span> - main function<br /> +  *  <span class="term">kses</span> - main function of <span class="term">kses</span><br /> +  *  <span class="term">kses_hook</span> - hook function of <span class="term">kses</span><br /> +<br /> +  The last two are for compatibility with pre-existing code using the <span class="term">kses</span> script. htmLawed's <span class="term">kses()</span> basically passes on the filtering task to <span class="term">htmLawed()</span> function after deciphering <span class="term">$config</span> and <span class="term">$spec</span> from the argument values supplied to it. <span class="term">kses_hook()</span> is an empty function and is meant for being filled with custom code if the <span class="term">kses</span> script users were using one.<br /> +<br /> +  <span class="term">htmLawed()</span> finalizes <span class="term">$spec</span> (with the help of <span class="term">hl_spec()</span>) and <span class="term">$config</span>, and globalizes them. Finalization of <span class="term">$config</span> involves setting default values if an inappropriate or invalid one is supplied. This includes calling <span class="term">hl_regex()</span> to check well-formedness of regular expression patterns if such expressions are user-supplied through <span class="term">$config</span>. <span class="term">htmLawed()</span> then removes invalid characters like nulls and <span class="term">x01</span> and appropriately handles entities using <span class="term">hl_ent()</span>. HTML comments and CDATA sections are identified and treated as per <span class="term">$config</span> with the help of <span class="term">hl_cmtcd()</span>. When retained, the <span class="term"><</span> and <span class="term">></span> characters identifying them, and the <span class="term"><</span>, <span class="term">></span> and <span class="term">&</span> characters inside them, are replaced with control characters (code-points <span class="term">1</span> to <span class="term">5</span>) till any tag balancing is completed.<br /> +<br /> +  After this <em>initial processing</em> <span class="term">htmLawed()</span> identifies tags using regex and processes them with the help of <span class="term">hl_tag()</span> --  a large function that analyzes tag content, filtering it as per HTML standards, <span class="term">$config</span> and <span class="term">$spec</span>. Among other things, <span class="term">hl_tag()</span> transforms deprecated elements using <span class="term">hl_tag2()</span>, removes attributes from closing tags, checks attribute values as per <span class="term">$spec</span> rules using <span class="term">hl_attrval()</span>, and checks URL protocols using <span class="term">hl_prot()</span>. <span class="term">htmLawed()</span> performs tag balancing and nesting checks with a call to <span class="term">hl_bal()</span>, and optionally compacts/beautifies the output with proper white-spacing with a call to <span class="term">hl_tidy()</span>. The latter temporarily replaces white-space, and <span class="term"><</span>, <span class="term">></span> and <span class="term">&</span> characters inside <span class="term">pre</span>, <span class="term">script</span> and <span class="term">textarea</span> elements, and HTML comments and CDATA sections with control characters (code-points <span class="term">1</span> to <span class="term">5</span>, and <span class="term">7</span>).<br /> +<br /> +  htmLawed permits the use of custom code or <strong>hook functions</strong> at two stages. The first, called inside <span class="term">htmLawed()</span>, allows the input text as well as the finalized $config and $spec values to be altered right after the initial processing (see <a href="#s3.7">section 3.7</a>). The second is called by <span class="term">hl_tag()</span> once the tag content is finalized (see <a href="#s3.4.9">section 3.4.9</a>).<br /> +<br /> +  Being dictated by the external and stable HTML standard, htmLawed's objective is very clear-cut and less concerned with tweakability. The code is only minimally annotated with comments -- it is not meant to instruct; PHP developers familiar with the HTML specs will see the logic, and others can always refer to the htmLawed documentation. The compact structuring of the statements is meant to aid in quickly grasping the logic, at least when viewed with code syntax highlighted. +</div> +</div> +<br /> +<hr /><br /><br /><span class="subtle"><small>HTM version of <em><a href="htmLawed_README.txt">htmLawed_README.txt</a></em> generated on 23 Apr, 2009 using <a href="http://www.bioinformatics.org/phplabware/internal_utilities">rTxt2htm</a> from PHP Labware</small></span> +</div><!-- ended div body --> +</div><!-- ended div top --> +</body> +</html>
\ No newline at end of file diff --git a/extlib/htmLawed/htmLawed_README.txt b/extlib/htmLawed/htmLawed_README.txt new file mode 100644 index 000000000..3ce4b9ac1 --- /dev/null +++ b/extlib/htmLawed/htmLawed_README.txt @@ -0,0 +1,1600 @@ +/* +htmLawed_README.txt, 16 July 2009 +htmLawed 1.1.8.1, 16 July 2009 +Copyright Santosh Patnaik +GPL v3 license +A PHP Labware internal utility - http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed +*/ + + +== Content ========================================================== + + +1 About htmLawed + 1.1 Example uses + 1.2 Features + 1.3 History + 1.4 License & copyright + 1.5 Terms used here +2 Usage + 2.1 Simple + 2.2 Configuring htmLawed using the '$config' parameter + 2.3 Extra HTML specifications using the '$spec' parameter + 2.4 Performance time & memory usage + 2.5 Some security risks to keep in mind + 2.6 Use without modifying old 'kses()' code + 2.7 Tolerance for ill-written HTML + 2.8 Limitations & work-arounds + 2.9 Examples +3 Details + 3.1 Invalid/dangerous characters + 3.2 Character references/entities + 3.3 HTML elements + 3.3.1 HTML comments and 'CDATA' sections + 3.3.2 Tag-transformation for better XHTML-Strict + 3.3.3 Tag balancing and proper nesting + 3.3.4 Elements requiring child elements + 3.3.5 Beautify or compact HTML + 3.4 Attributes + 3.4.1 Auto-addition of XHTML-required attributes + 3.4.2 Duplicate/invalid 'id' values + 3.4.3 URL schemes (protocols) and scripts in attribute values + 3.4.4 Absolute & relative URLs + 3.4.5 Lower-cased, standard attribute values + 3.4.6 Transformation of deprecated attributes + 3.4.7 Anti-spam & 'href' + 3.4.8 Inline style properties + 3.4.9 Hook function for tag content + 3.5 Simple configuration directive for most valid XHTML + 3.6 Simple configuration directive for most `safe` HTML + 3.7 Using a hook function + 3.8 Obtaining `finalized` parameter values + 3.9 Retaining non-HTML tags in input with mixed markup +4 Other + 4.1 Support + 4.2 Known issues + 4.3 Change-log + 4.4 Testing + 4.5 Upgrade, & old versions + 4.6 Comparison with 'HTMLPurifier' + 4.7 Use through application plug-ins/modules + 4.8 Use in non-PHP applications + 4.9 Donate + 4.10 Acknowledgements +5 Appendices + 5.1 Characters discouraged in HTML + 5.2 Valid attribute-element combinations + 5.3 CSS 2.1 properties accepting URLs + 5.4 Microsoft Windows 1252 character replacements + 5.5 URL format + 5.6 Brief on htmLawed code + + +== 1 About htmLawed ================================================ + + + htmLawed is a highly customizable single-file PHP script to make text secure, and standard- and admin policy-compliant for use in the body of HTML 4, XHTML 1 or 1.1, or generic XML documents. It is thus a configurable input (X)HTML filter, processor, purifier, sanitizer, beautifier, etc., and an alternative to the HTMLTidy:- http://tidy.sourceforge.net application. + + The `lawing in` of input text is needed to ensure that HTML code in the text is standard-compliant, does not introduce security vulnerabilities, and does not break the aesthetics, design or layout of web-pages. htmLawed tries to do this by, for example, making HTML well-formed with balanced and properly nested tags, neutralizing code that may be used for cross-site scripting ('XSS') attacks, and allowing only specified HTML elements/tags and attributes. + + +-- 1.1 Example uses ------------------------------------------------ + + + * Filtering of text submitted as comments on blogs to allow only certain HTML elements + + * Making RSS/Atom newsfeed item-content standard-compliant: often one uses an excerpt from an HTML document for the content, and with unbalanced tags, non-numerical entities, etc., such excerpts may not be XML-compliant + + * Text processing for stricter XML standard-compliance: e.g., to have lowercased 'x' in hexadecimal numeric entities becomes necessary if an XHTML document with MathML content needs to be served as 'application/xml' + + * Scraping text or data from web-pages + + * Pretty-printing HTML code + + +-- 1.2 Features ---------------------------------------------------o + + + Key: '*' security feature, '^' standard compliance, '~' requires setting right options, '`' different from 'Kses' + + * make input more *secure* and *standard-compliant* + * use for HTML 4, XHTML 1.0 or 1.1, or even generic *XML* documents ^~` + + * *beautify* or *compact* HTML ^~` + + * *restrict elements* ^~` + * proper closure of empty elements like 'img' ^` + * *transform deprecated elements* like 'u' ^~` + * HTML *comments* and 'CDATA' sections can be permitted ^~` + * elements like 'script', 'object' and 'form' can be permitted ~ + + * *restrict attributes*, including *element-specifically* ^~` + * remove *invalid attributes* ^` + * element and attribute names are *lower-cased* ^ + * provide *required attributes*, like 'alt' for 'image' ^` + * *transform deprecated attributes* ^~` + * attributes *declared only once* ^` + + * *restrict attribute values*, including *element-specifically* ^~` + * a value is declared for `empty` (`minimized`) attributes like 'checked' ^ + * check for potentially dangerous attribute values *~ + * ensure *unique* 'id' attribute values ^~` + * *double-quote* attribute values ^ + * lower-case *standard attribute values* like 'password' ^` + + * *attribute-specific URL protocol/scheme restriction* *~` + * disable *dynamic expressions* in 'style' values *~` + + * neutralize invalid named character entities ^` + * *convert* hexadecimal numeric entities to decimal ones, or vice versa ^~` + * convert named entities to numeric ones for generic XML use ^~` + + * remove *null* characters * + * neutralize potentially dangerous proprietary Netscape *Javascript entities* * + * replace potentially dangerous *soft-hyphen* character in attribute values with spaces * + + * remove common *invalid characters* not allowed in HTML or XML ^` + * replace *characters from Microsoft applications* like 'Word' that are discouraged in HTML or XML ^~` + * neutralize entities for characters invalid or discouraged in HTML or XML ^` + * appropriately neutralize '<', '&', '"', and '>' characters ^*` + + * understands improperly spaced tag content (like, spread over more than a line) and properly spaces them ` + * attempts to *balance tags* for well-formedness ^~` + * understands when *omitable closing tags* like '</p>' (allowed in HTML 4, transitional, e.g.) are missing ^~` + * attempts to permit only *validly nested tags* ^~` + * option to *remove or neutralize bad content* ^~` + * attempts to *rectify common errors of plain-text misplacement* (e.g., directly inside 'blockquote') ^~` + + * fast, *non-OOP* code of ~45 kb incurring peak basal memory usage of ~0.5 MB + * *compatible* with pre-existing code using 'Kses' (the filter used by 'WordPress') + + * optional *anti-spam* measures such as addition of 'rel="nofollow"' and link-disabling ~` + * optionally makes *relative URLs absolute*, and vice versa ~` + + * optionally mark '&' to identify the entities for '&', '<' and '>' introduced by htmLawed ~` + + * allows deployment of powerful *hook functions* to *inject* HTML, *consolidate* 'style' attributes to 'class', finely check attribute values, etc. ~` + + * *independent of character encoding* of input and does not affect it + + * *tolerance for ill-written HTML* to a certain degree + + +-- 1.3 History ----------------------------------------------------o + + + htmLawed was developed for use with 'LabWiki', a wiki software developed at PHP Labware, as a suitable software could not be found. Existing PHP software like 'Kses' and 'HTMLPurifier' were deemed inadequate, slow, resource-intensive, or dependent on external applications like 'HTML Tidy'. + + htmLawed started as a modification of Ulf Harnhammar's 'Kses' (version 0.2.2) software, and is compatible with code that uses 'Kses'; see section:- #2.6. + + +-- 1.4 License & copyright ----------------------------------------o + + + htmLawed is free and open-source software licensed under GPL license version 3:- http://www.gnu.org/licenses/gpl-3.0.txt, and copyrighted by Santosh Patnaik, MD, PhD. + + +-- 1.5 Terms used here --------------------------------------------o + + + * `administrator` - or admin; person setting up the code to pass input through htmLawed; also, `user` + * `attributes` - name-value pairs like 'href="http://x.com"' in opening tags + * `author` - `writer` + * `character` - atomic unit of text; internally represented by a numeric `code-point` as specified by the `encoding` or `charset` in use + * `entity` - markup like '>' and ' ' used to refer to a character + * `element` - HTML element like 'a' and 'img' + * `element content` - content between the opening and closing tags of an element, like 'click' of '<a href="x">click</a>' + * `HTML` - implies XHTML unless specified otherwise + * `input` - text string given to htmLawed to process + * `processing` - involves filtering, correction, etc., of input + * `safe` - absence or reduction of certain characters and HTML elements and attributes in the input that can otherwise potentially and circumstantially expose web-site users to security vulnerabilities like cross-site scripting attacks (XSS) + * `scheme` - URL protocol like 'http' and 'ftp' + * `specs` - standard specifications + * `style property` - terms like 'border' and 'height' for which declarations are made in values for the 'style' attribute of elements + * `tag` - markers like '<a href="x">' and '</a>' delineating element content; the opening tag can contain attributes + * `tag content` - consists of tag markers '<' and '>', element names like 'div', and possibly attributes + * `user` - administrator + * `writer` - end-user like a blog commenter providing the input that is to be processed; also, `author` + + +== 2 Usage ========================================================oo + + + htmLawed should work with PHP 4.3 and higher. Either 'include()' the 'htmLawed.php' file or copy-paste the entire code. + + To easily *test* htmLawed using a form-based interface, use the provided demo:- htmLawedTest.php ('htmLawed.php' and 'htmLawedTest.php' should be in the same directory on the web-server). + + +-- 2.1 Simple ------------------------------------------------------ + + + The input text to be processed, '$text', is passed as an argument of type string; 'htmLawed()' returns the processed string: + + $processed = htmLawed($text); + + *Note*: If input is from a '$_GET' or '$_POST' value, and 'magic quotes' are enabled on the PHP setup, run 'stripslashes()' on the input before passing to htmLawed. + + By default, htmLawed will process the text allowing all valid HTML elements/tags, secure URL scheme/CSS style properties, etc. It will allow 'CDATA' sections and HTML comments, balance tags, and ensure proper nesting of elements. Such actions can be configured using two other optional arguments -- '$config' and '$spec': + + $processed = htmLawed($text, $config, $spec); + + These extra parameters are detailed below. Some examples are shown in section:- #2.9. + + *Note*: For maximum protection against 'XSS' and other scripting attacks (e.g., by disallowing Javascript code), consider using the 'safe' parameter; see section:- #3.6. + + +-- 2.2 Configuring htmLawed using the '$config' parameter ---------o + + + '$config' instructs htmLawed on how to tackle certain tasks. When '$config' is not specified, or not set as an array (e.g., '$config = 1'), htmLawed will take default actions. One or many of the task-action or value-specification pairs can be specified in '$config' as array key-value pairs. If a parameter is not specified, htmLawed will use the default value/action indicated further below. + + $config = array('comment'=>0, 'cdata'=>1); + $processed = htmLawed($text, $config); + + Or, + + $processed = htmLawed($text, array('comment'=>0, 'cdata'=>1)); + + Below are the possible value-specification combinations. In PHP code, values that are integers should not be quoted and should be used as numeric types (unless meant as string/text). + + Key: '*' default, '^' different default when htmLawed is used in the Kses-compatible mode (see section:- #2.6), '~' different default when 'valid_xhtml' is set to '1' (see section:- #3.5), '"' different default when 'safe' is set to '1' (see section:- #3.6) + + *abs_url* + Make URLs absolute or relative; '$config["base_url"]' needs to be set; see section:- #3.4.4 + + '-1' - make relative + '0' - no action * + '1' - make absolute + + *and_mark* + Mark '&' characters in the original input; see section:- #3.2 + + *anti_link_spam* + Anti-link-spam measure; see section:- #3.4.7 + + '0' - no measure taken * + 'array("regex1", "regex2")' - will ensure a 'rel' attribute with 'nofollow' in its value in case the 'href' attribute value matches the regular expression pattern 'regex1', and/or will remove 'href' if its value matches the regular expression pattern 'regex2'. E.g., 'array("/./", "/://\W*(?!(abc\.com|xyz\.org))/")'; see section:- #3.4.7 for more. + + *anti_mail_spam* + Anti-mail-spam measure; see section:- #3.4.7 + + '0' - no measure taken * + 'word' - '@' in mail address in 'href' attribute value is replaced with specified 'word' + + *balance* + Balance tags for well-formedness and proper nesting; see section:- #3.3.3 + + '0' - no + '1' - yes * + + *base_url* + Base URL value that needs to be set if '$config["abs_url"]' is not '0'; see section:- #3.4.4 + + *cdata* + Handling of 'CDATA' sections; see section:- #3.3.1 + + '0' - don't consider 'CDATA' sections as markup and proceed as if plain text ^" + '1' - remove + '2' - allow, but neutralize any '<', '>', and '&' inside by converting them to named entities + '3' - allow * + + *clean_ms_char* + Replace discouraged characters introduced by Microsoft Word, etc.; see section:- #3.1 + + '0' - no * + '1' - yes + '2' - yes, but replace special single & double quotes with ordinary ones + + *comment* + Handling of HTML comments; see section:- #3.3.1 + + '0' - don't consider comments as markup and proceed as if plain text ^" + '1' - remove + '2' - allow, but neutralize any '<', '>', and '&' inside by converting to named entities + '3' - allow * + + *css_expression* + Allow dynamic CSS expression by not removing the expression from CSS property values in 'style' attributes; see section:- #3.4.8 + + '0' - remove * + '1' - allow + + *deny_attribute* + Denied HTML attributes; see section:- #3.4 + + '0' - none * + 'string' - dictated by values in 'string' + 'on*' (like 'onfocus') attributes not allowed - " + + *elements* + Allowed HTML elements; see section:- #3.3 + + '* -center -dir -font -isindex -menu -s -strike -u' - ~ + 'applet, embed, iframe, object, script' not allowed - " + + *hexdec_entity* + Allow hexadecimal numeric entities and do not convert to the more widely accepted decimal ones, or convert decimal to hexadecimal ones; see section:- #3.2 + + '0' - no + '1' - yes * + '2' - convert decimal to hexadecimal ones + + *hook* + Name of an optional hook function to alter the input string, '$config' or '$spec' before htmLawed starts its main work; see section:- #3.7 + + '0' - no hook function * + 'name' - 'name' is name of the hook function ('kses_hook' ^) + + *hook_tag* + Name of an optional hook function to alter tag content finalized by htmLawed; see section:- #3.4.9 + + '0' - no hook function * + 'name' - 'name' is name of the hook function + + *keep_bad* + Neutralize bad tags by converting '<' and '>' to entities, or remove them; see section:- #3.3.3 + + '0' - remove ^ + '1' - neutralize both tags and element content + '2' - remove tags but neutralize element content + '3' and '4' - like '1' and '2' but remove if text ('pcdata') is invalid in parent element + '5' and '6' * - like '3' and '4' but line-breaks, tabs and spaces are left + + *lc_std_val* + For XHTML compliance, predefined, standard attribute values, like 'get' for the 'method' attribute of 'form', must be lowercased; see section:- #3.4.5 + + '0' - no + '1' - yes * + + *make_tag_strict* + Transform/remove these non-strict XHTML elements, even if they are allowed by the admin: 'applet' 'center' 'dir' 'embed' 'font' 'isindex' 'menu' 's' 'strike' 'u'; see section:- #3.3.2 + + '0' - no ^ + '1' - yes, but leave 'applet', 'embed' and 'isindex' elements that currently can't be transformed * + '2' - yes, removing 'applet', 'embed' and 'isindex' elements and their contents (nested elements remain) ~ + + *named_entity* + Allow non-universal named HTML entities, or convert to numeric ones; see section:- #3.2 + + '0' - convert + '1' - allow * + + *no_deprecated_attr* + Allow deprecated attributes or transform them; see section:- #3.4.6 + + '0' - allow ^ + '1' - transform, but 'name' attributes for 'a' and 'map' are retained * + '2' - transform + + *parent* + Name of the parent element, possibly imagined, that will hold the input; see section:- #3.3 + + *safe* + Magic parameter to make input the most secure against XSS without needing to specify other relevant '$config' parameters; see section:- #3.6 + + '0' - no * + '1' - will auto-adjust other relevant '$config' parameters (indicated by '"' in this list) + + *schemes* + Array of attribute-specific, comma-separated, lower-cased list of schemes (protocols) allowed in attributes accepting URLs; '*' covers all unspecified attributes; see section:- #3.4.3 + + 'href: aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet; *:file, http, https' * + '*: ftp, gopher, http, https, mailto, news, nntp, telnet' ^ + 'href: aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet; style: nil; *:file, http, https' " + + *show_setting* + Name of a PHP variable to assign the `finalized` '$config' and '$spec' values; see section:- #3.8 + + *style_pass* + Do not look at 'style' attribute values, letting them through without any alteration + + '0' - no * + '1' - htmLawed will let through any 'style' value; see section:- #3.4.8 + + *tidy* + Beautify or compact HTML code; see section:- #3.3.5 + + '-1' - compact + '0' - no * + '1' or 'string' - beautify (custom format specified by 'string') + + *unique_ids* + 'id' attribute value checks; see section:- #3.4.2 + + '0' - no ^ + '1' - remove duplicate and/or invalid ones * + 'word' - remove invalid ones and replace duplicate ones with new and unique ones based on the 'word'; the admin-specified 'word', like 'my_', should begin with a letter (a-z) and can contain letters, digits, '.', '_', '-', and ':'. + + *valid_xhtml* + Magic parameter to make input the most valid XHTML without needing to specify other relevant '$config' parameters; see section:- #3.5 + + '0' - no * + '1' - will auto-adjust other relevant '$config' parameters (indicated by '~' in this list) + + *xml:lang* + Auto-adding 'xml:lang' attribute; see section:- #3.4.1 + + '0' - no * + '1' - add if 'lang' attribute is present + '2' - add if 'lang' attribute is present, and remove 'lang' ~ + + +-- 2.3 Extra HTML specifications using the $spec parameter --------o + + + The '$spec' argument can be used to disallow an otherwise legal attribute for an element, or to restrict the attribute's values. This can also be helpful as a security measure (e.g., in certain versions of browsers, certain values can cause buffer overflows and denial of service attacks), or in enforcing admin policy compliance. '$spec' is specified as a string of text containing one or more `rules`, with multiple rules separated from each other by a semi-colon (';'). E.g., + + $spec = 'i=-*; td, tr=style, id, -*; a=id(match="/[a-z][a-z\d.:\-`"]*/i"/minval=2), href(maxlen=100/minlen=34); img=-width,-alt'; + $processed = htmLawed($text, $config, $spec); + + Or, + + $processed = htmLawed($text, $config, 'i=-*; td, tr=style, id, -*; a=id(match="/[a-z][a-z\d.:\-`"]*/i"/minval=2), href(maxlen=100/minlen=34); img=-width,-alt'); + + A rule begins with an HTML *element* name(s) (`rule-element`), for which the rule applies, followed by an equal ('=') sign. A rule-element may represent multiple elements if comma (,)-separated element names are used. E.g., 'th,td,tr='. + + Rest of the rule consists of comma-separated HTML *attribute names*. A minus ('-') character before an attribute means that the attribute is not permitted inside the rule-element. E.g., '-width'. To deny all attributes, '-*' can be used. + + Following shows examples of rule excerpts with rule-element 'a' and the attributes that are being permitted: + + * 'a=' - all + * 'a=id' - all + * 'a=href, title, -id, -onclick' - all except 'id' and 'onclick' + * 'a=*, id, -id' - all except 'id' + * 'a=-*' - none + * 'a=-*, href, title' - none except 'href' and 'title' + * 'a=-*, -id, href, title' - none except 'href' and 'title' + + Rules regarding *attribute values* are optionally specified inside round brackets after attribute names in slash ('/')-separated `parameter = value` pairs. E.g., 'title(maxlen=30/minlen=5)'. None, or one or more of the following parameters may be specified: + + * 'oneof' - one or more choices separated by '|' that the value should match; if only one choice is provided, then the value must match that choice + + * 'noneof' - one or more choices separated by '|' that the value should not match + + * 'maxlen' and 'minlen' - upper and lower limits for the number of characters in the attribute value; specified in numbers + + * 'maxval' and 'minval' - upper and lower limits for the numerical value specified in the attribute value; specified in numbers + + * 'match' and 'nomatch' - pattern that the attribute value should or should not match; specified as PHP/PCRE-compatible regular expressions with delimiters and possibly modifiers + + * 'default' - a value to force on the attribute if the value provided by the writer does not fit any of the specified parameters + + If 'default' is not set and the attribute value does not satisfy any of the specified parameters, then the attribute is removed. The 'default' value can also be used to force all attribute declarations to take the same value (by getting the values declared illegal by setting, e.g., 'maxlen' to '-1'). + + Examples with `input` '<input title="WIDTH" value="10em" /><input title="length" value="5" />' are shown below. + + `Rule`: 'input=title(maxlen=60/minlen=6), value' + `Output`: '<input value="10em" /><input title="length" value="5" />' + + `Rule`: 'input=title(), value(maxval=8/default=6)' + `Output`: '<input title="WIDTH" value="6" /><input title="length" value="5" />' + + `Rule`: 'input=title(nomatch=$w.d$i), value(match=$em$/default=6em)' + `Output`: '<input value="10em" /><input title="length" value="6em" />' + + `Rule`: 'input=title(oneof=height|depth/default=depth), value(noneof=5|6)' + `Output`: '<input title="depth" value="10em" /><input title="depth" />' + + *Special characters*: The characters ';', ',', '/', '(', ')', '|', '~' and space have special meanings in the rules. Words in the rules that use such characters, or the characters themselves, should be `escaped` by enclosing in pairs of double-quotes ('"'). A back-tick ('`') can be used to escape a literal '"'. An example rule illustrating this is 'input=value(maxlen=30/match="/^\w/"/default="your `"ID`"")'. + + *Note*: To deny an attribute for all elements for which it is legal, '$config["deny_attribute"]' (see section:- #3.4) can be used instead of '$spec'. Also, attributes can be allowed element-specifically through '$spec' while being denied globally through '$config["deny_attribute"]'. The 'hook_tag' parameter (section:- #3.4.9) can also be used to implement the '$spec' functionality. + + +-- 2.4 Performance time & memory usage ----------------------------o + + + The time and memory used by htmLawed depends on its configuration and the size of the input, and the amount, nestedness and well-formedness of the HTML markup within it. In particular, tag balancing and beautification each can increase the processing time by about a quarter. + + The htmLawed demo:- htmLawedTest.php can be used to evaluate the performance and effects of different types of input and '$config'. + + +-- 2.5 Some security risks to keep in mind ------------------------o + + + When setting the parameters/arguments (like those to allow certain HTML elements) for use with htmLawed, one should bear in mind that the setting may let through potentially `dangerous` HTML code. (This may not be a problem if the authors are trusted.) + + For example, following increase security risks: + + * Allowing 'script', 'applet', 'embed', 'iframe' or 'object' elements, or certain of their attributes like 'allowscriptaccess' + + * Allowing HTML comments (some Internet Explorer versions are vulnerable with, e.g., '<!--[if gte IE 4]><script>alert("xss");</script><![endif]-->' + + * Allowing dynamic CSS expressions (a feature of the IE browser) + + `Unsafe` HTML can be removed by setting '$config' appropriately. E.g., '$config["elements"] = "* -script"' (section:- #3.3), '$config["safe"] = 1' (section:- #3.6), etc. + + +-- 2.6 Use without modifying old 'kses()' code --------------------o + + + The 'Kses' PHP script is used by many applications (like 'WordPress'). It is possible to have such applications use htmLawed instead, since it is compatible with code that calls the 'kses()' function declared in the 'Kses' file (usually named 'kses.php'). E.g., application code like this will continue to work after replacing 'Kses' with htmLawed: + + $comment_filtered = kses($comment_input, array('a'=>array(), 'b'=>array(), 'i'=>array())); + + For some of the '$config' parameters, htmLawed will use values other than the default ones. These are indicated by '^' in section:- #2.2. To force htmLawed to use other values, function 'kses()' in the htmLawed code should be edited -- a few configurable parameters/variables need to be changed. + + If the application uses a 'Kses' file that has the 'kses()' function declared, then, to have the application use htmLawed instead of 'Kses', simply rename 'htmLawed.php' (to 'kses.php', e.g.) and replace the 'Kses' file (or just replace the code in the 'Kses' file with the htmLawed code). If the 'kses()' function in the 'Kses' file had been renamed by the application developer (e.g., in 'WordPress', it is named 'wp_kses()'), then appropriately rename the 'kses()' function in the htmLawed code. + + If the 'Kses' file used by the application has been highly altered by the application developers, then one may need a different approach. E.g., with 'WordPress', it is best to copy the htmLawed code to 'wp_includes/kses.php', rename the newly added function 'kses()' to 'wp_kses()', and delete the code for the original 'wp_kses()' function. + + If the 'Kses' code has a non-empty hook function (e.g., 'wp_kses_hook()' in case of 'WordPress'), then the code for htmLawed's 'kses_hook()' function should be appropriately edited. However, the requirement of the hook function should be re-evaluated considering that htmLawed has extra capabilities. With 'WordPress', the hook function is an essential one. The following code is suggested for the htmLawed 'kses_hook()' in case of 'WordPress': + + function kses_hook($string, &$cf, &$spec){ + // kses compatibility + $allowed_html = $spec; + $allowed_protocols = array(); + foreach($cf['schemes'] as $v){ + foreach($v as $k2=>$v2){ + if(!in_array($k2, $allowed_protocols)){ + $allowed_protocols[] = $k2; + } + } + } + return wp_kses_hook($string, $allowed_html, $allowed_protocols); + // eof + } + + +-- 2.7 Tolerance for ill-written HTML -----------------------------o + + + htmLawed can work with ill-written HTML code in the input. However, HTML that is too ill-written may not be `read` as HTML, and be considered mere plain text instead. Following statements indicate the degree of `looseness` that htmLawed can work with, and can be provided in instructions to writers: + + * Tags must be flanked by '<' and '>' with no '>' inside -- any needed '>' should be put in as '>'. It is possible for tag content (element name and attributes) to be spread over many lines instead of being on one. A space may be present between the tag content and '>', like '<div >' and '<img / >', but not after the '<'. + + * Element and attribute names need not be lower-cased. + + * Attribute string of elements may be liberally spaced with tabs, line-breaks, etc. + + * Attribute values may not be double-quoted, or may be single-quoted. + + * Left-padding of numeric entities (like, ' ', '&x07ff;') with '0' is okay as long as the number of characters between between the '&' and the ';' does not exceed 8. All entities must end with ';' though. + + * Named character entities must be properly cased. E.g., '≪' or '&TILDE;' will not be let through without modification. + + * HTML comments should not be inside element tags (okay between tags), and should begin with '<!--' and end with '-->'. Characters like '<', '>', and '&' may be allowed inside depending on '$config', but any '-->' inside should be put in as '-->'. Any '--' inside will be automatically converted to '-', and a space will be added before the comment delimiter '-->'. + + * 'CDATA' sections should not be inside element tags, and can be in element content only if plain text is allowed for that element. They should begin with '<[CDATA[' and end with ']]>'. Characters like '<', '>', and '&' may be allowed inside depending on '$config', but any ']]>' inside should be put in as ']]>'. + + * For attribute values, character entities '<', '>' and '&' should be used instead of characters '<' and '>', and '&' (when '&' is not part of a character entity). This applies even for Javascript code in values of attributes like 'onclick'. + + * Characters '<', '>', '&' and '"' that are part of actual Javascript, etc., code in 'script' elements should be used as such and not be put in as entities like '>'. Otherwise, though the HTML will be valid, the code may fail to work. Further, if such characters have to be used, then they should be put inside 'CDATA' sections. + + * Simple instructions like "an opening tag cannot be present between two closing tags" and "nested elements should be closed in the reverse order of how they were opened" can help authors write balanced HTML. If tags are imbalanced, htmLawed will try to balance them, but in the process, depending on '$config["keep_bad"]', some code/text may be lost. + + * Input authors should be notified of admin-specified allowed elements, attributes, configuration values (like conversion of named entities to numeric ones), etc. + + * With '$config["unique_ids"]' not '0' and the 'id' attribute being permitted, writers should carefully avoid using duplicate or invalid 'id' values as even though htmLawed will correct/remove the values, the final output may not be the one desired. E.g., when '<a id="home"></a><input id="home" /><label for="home"></label>' is processed into +'<a id="home"></a><input id="prefix_home" /><label for="home"></label>'. + + * Note that even if intended HTML is lost in a highly ill-written input, the processed output will be more secure and standard-compliant. + + * For URLs, unless '$config["scheme"]' is appropriately set, writers should avoid using escape characters or entities in schemes. E.g., 'http' (which many browsers will read as the harmless 'http') may be considered bad by htmLawed. + + * htmLawed will attempt to put plain text present directly inside 'blockquote', 'form', 'map' and 'noscript' elements (illegal as per the specs) inside auto-generated 'div' elements. + + +-- 2.8 Limitations & work-arounds ---------------------------------o + + + htmLawed's main objective is to make the input text `more` standard-compliant, secure for web-page readers, and free of HTML elements and attributes considered undesirable by the administrator. Some of its current limitations, regardless of this objective, are noted below along with work-arounds. + + It should be borne in mind that no browser application is 100% standard-compliant, and that some of the standard specs (like asking for normalization of white-spacing within 'textarea' elements) are clearly wrong. Regarding security, note that `unsafe` HTML code is not necessarily legally invalid. + + * htmLawed is meant for input that goes into the 'body' of HTML documents. HTML's head-level elements are not supported, nor are the frameset elements 'frameset', 'frame' and 'noframes'. + + * It cannot transform the non-standard 'embed' elements to the standard-compliant 'object' elements. Yet, it can allow 'embed' elements if permitted ('embed' is widely used and supported). Admins can certainly use the 'hook_tag' parameter (section:- #3.4.9) to deploy a custom embed-to-object converter function. + + * The only non-standard element that may be permitted is 'embed'; others like 'noembed' and 'nobr' cannot be permitted without modifying the htmLawed code. + + * It cannot handle input that has non-HTML code like 'SVG' and 'MathML'. One way around is to break the input into pieces and passing only those without non-HTML code to htmLawed. Another is described in section:- #3.9. A third way may be to some how take advantage of the '$config["and_mark"]' parameter (see section:- #3.2). + + * By default, htmLawed won't check many attribute values for standard compliance. E.g., 'width="20m"' with the dimension in non-standard 'm' is let through. Implementing universal and strict attribute value checks can make htmLawed slow and resource-intensive. Admins should look at the 'hook_tag' parameter (section:- #3.4.9) or '$spec' to enforce finer checks. + + * The attributes, deprecated (which can be transformed too) or not, that it supports are largely those that are in the specs. Only a few of the proprietary attributes are supported. + + * Except for contained URLs and dynamic expressions (also optional), htmLawed does not check CSS style property values. Admins should look at using the 'hook_tag' parameter (section:- #3.4.9) or '$spec' for finer checks. Perhaps the best option is to disallow 'style' but allow 'class' attributes with the right 'oneof' or 'match' values for 'class', and have the various class style properties in '.css' CSS stylesheet files. + + * htmLawed does not parse emoticons, decode `BBcode`, or `wikify`, auto-converting text to proper HTML. Similarly, it won't convert line-breaks to 'br' elements. Such functions are beyond its purview. Admins should use other code to pre- or post-process the input for such purposes. + + * htmLawed cannot be used to have links force-opened in new windows (by auto-adding appropriate 'target' and 'onclick' attributes to 'a'). Admins should look at Javascript-based DOM-modifying solutions for this. Admins may also be able to use a custom hook function to enforce such checks ('hook_tag' parameter; see section:- #3.4.9). + + * Nesting-based checks are not possible. E.g., one cannot disallow 'p' elements specifically inside 'td' while permitting it elsewhere. Admins may be able to use a custom hook function to enforce such checks ('hook_tag' parameter; see section:- #3.4.9). + + * Except for optionally converting absolute or relative URLs to the other type, htmLawed will not alter URLs (e.g., to change the value of query strings or to convert 'http' to 'https'. Having absolute URLs may be a standard-requirement, e.g., when HTML is embedded in email messages, whereas altering URLs for other purposes is beyond htmLawed's goals. Admins may be able to use a custom hook function to enforce such checks ('hook_tag' parameter; see section:- #3.4.9). + + * Pairs of opening and closing tags that do not enclose any content (like '<em></em>') are not removed. This may be against the standard specs for certain elements (e.g., 'table'). However, presence of such standard-incompliant code will not break the display or layout of content. Admins can also use simple regex-based code to filter out such code. + + * htmLawed does not check for certain element orderings described in the standard specs (e.g., in a 'table', 'tbody' is allowed before 'tfoot'). Admins may be able to use a custom hook function to enforce such checks ('hook_tag' parameter; see section:- #3.4.9). + + * htmLawed does not check the number of nested elements. E.g., it will allow two 'caption' elements in a 'table' element, illegal as per the specs. Admins may be able to use a custom hook function to enforce such checks ('hook_tag' parameter; see section:- #3.4.9). + + * htmLawed might convert certain entities to actual characters and remove backslashes and CSS comment-markers ('/*') in 'style' attribute values in order to detect malicious HTML like crafted IE-specific dynamic expressions like 'expression...'. If this is too harsh, admins can allow CSS expressions through htmLawed core but then use a custom function through the 'hook_tag' parameter (section:- #3.4.9) to more specifically identify CSS expressions in the 'style' attribute values. Also, using '$config["style_pass"]', it is possible to have htmLawed pass 'style' attribute values without even looking at them (section:- #3.4.8). + + * htmLawed does not correct certain possible attribute-based security vulnerabilities (e.g., '<a href="http://x%22+style=%22background-image:xss">x</a>'). These arise when browsers mis-identify markup in `escaped` text, defeating the very purpose of escaping text (a bad browser will read the given example as '<a href="http://x" style="background-image:xss">x</a>'). + + * Because of poor Unicode support in PHP, htmLawed does not remove the `high value` HTML-invalid characters with multi-byte code-points. Such characters however are extremely unlikely to be in the input. (see section:- #3.1). + + * Like any script using PHP's PCRE regex functions, PHP setup-specific low PCRE limit values can cause htmLawed to at least partially fail with very long input texts. + + +-- 2.9 Examples ---------------------------------------------------o + + + *1.* A blog administrator wants to allow only 'a', 'em', 'strike', 'strong' and 'u' in comments, but needs 'strike' and 'u' transformed to 'span' for better XHTML 1-strict compliance, and, he wants the 'a' links to be to 'http' or 'https' resources: + + $processed = htmLawed($in, array('elements'=>'a, em, strike, strong, u', 'make_tag_strict'=>1, 'safe'=>1, 'schemes'=>'*:http, https'), 'a=href'); + + *2.* An author uses a custom-made web application to load content on his web-site. He is the only one using that application and the content he generates has all types of HTML, including scripts. The web application uses htmLawed primarily as a tool to correct errors that creep in while writing HTML and to take care of the occasional `bad` characters in copy-paste text introduced by Microsoft Office. The web application provides a preview before submitted input is added to the content. For the previewing process, htmLawed is set up as follows: + + $processed = htmLawed($in, array('css_expression'=>1, 'keep_bad'=>1, 'make_tag_strict'=>1, 'schemes'=>'*:*', 'valid_xhtml'=>1)); + + For the final submission process, 'keep_bad' is set to '6'. A value of '1' for the preview process allows the author to note and correct any HTML mistake without losing any of the typed text. + + *3.* A data-miner is scraping information in a specific table of similar web-pages and is collating the data rows, and uses htmLawed to reduce unnecessary markup and white-spaces: + + $processed = htmLawed($in, array('elements'=>'tr, td', 'tidy'=>-1), 'tr, td ='); + + +== 3 Details =====================================================oo + + +-- 3.1 Invalid/dangerous characters -------------------------------- + + + Valid characters (more correctly, their code-points) in HTML or XML are, hexadecimally, '9', 'a', 'd', '20' to 'd7ff', and 'e000' to '10ffff', except 'fffe' and 'ffff' (decimally, '9', '10', '13', '32' to '55295', and '57344' to '1114111', except '65534' and '65535'). htmLawed removes the invalid characters '0' to '8', 'b', 'c', and 'e' to '1f'. + + Because of PHP's poor native support for multi-byte characters, htmLawed cannot check for the remaining invalid code-points. However, for various reasons, it is very unlikely for any of those characters to be in the input. + + Characters that are discouraged (see section:- #5.1) but not invalid are not removed by htmLawed. + + It (function 'hl_tag()') also replaces the potentially dangerous (in some Mozilla [Firefox] and Opera browsers) soft-hyphen character (code-point, hexadecimally, 'ad', or decimally, '173') in attribute values with spaces. Where required, the characters '<', '>', '&', and '"' are converted to entities. + + With '$config["clean_ms_char"]' set as '1' or '2', many of the discouraged characters (decimal code-points '127' to '159' except '133') that many Microsoft applications incorrectly use (as per the 'Windows 1252' ['Cp-1252'] or a similar encoding system), and the character for decimal code-point '133', are converted to appropriate decimal numerical entities (or removed for a few cases)-- see appendix in section:- #5.4. This can help avoid some display issues arising from copying-pasting of content. + + With '$config["clean_ms_char"]' set as '2', characters for the hexadecimal code-points '82', '91', and '92' (for special single-quotes), and '84', '93', and '94' (for special double-quotes) are converted to ordinary single and double quotes respectively and not to entities. + + The character values are replaced with entities/characters and not character values referred to by the entities/characters to keep this task independent of the character-encoding of input text. + + The '$config["clean_ms_char"]' parameter need not be used if authors do not copy-paste Microsoft-created text or if the input text is not believed to use the 'Windows 1252' or a similar encoding. Further, the input form and the web-pages displaying it or its content should have the character encoding appropriately marked-up. + + +-- 3.2 Character references/entities ------------------------------o + + + Valid character entities take the form '&*;' where '*' is '#x' followed by a hexadecimal number (hexadecimal numeric entity; like ' ' for non-breaking space), or alphanumeric like 'gt' (external or named entity; like ' ' for non-breaking space), or '#' followed by a number (decimal numeric entity; like ' ' for non-breaking space). Character entities referring to the soft-hyphen character (the '­' or '\xad' character; hexadecimal code-point 'ad' [decimal '173']) in attribute values are always replaced with spaces; soft-hyphens in attribute values introduce vulnerabilities in some older versions of the Opera and Mozilla [Firefox] browsers. + + htmLawed (function 'hl_ent()'): + + * Neutralizes entities with multiple leading zeroes or missing semi-colons (potentially dangerous) + + * Lowercases the 'X' (for XML-compliance) and 'A-F' of hexadecimal numeric entities + + * Neutralizes entities referring to characters that are HTML-invalid (see section:- #3.1) + + * Neutralizes entities referring to characters that are HTML-discouraged (code-points, hexadecimally, '7f' to '84', '86' to '9f', and 'fdd0' to 'fddf', or decimally, '127' to '132', '134' to '159', and '64991' to '64976'). Entities referring to the remaining discouraged characters (see section:- #5.1 for a full list) are let through. + + * Neutralizes named entities that are not in the specs. + + * Optionally converts valid HTML-specific named entities except '>', '<', '"', and '&' to decimal numeric ones (hexadecimal if $config["hexdec_entity"] is '2') for generic XML-compliance. For this, '$config["named_entity"]' should be '1'. + + * Optionally converts hexadecimal numeric entities to the more widely supported decimal ones. For this, '$config["hexdec_entity"]' should be '0'. + + * Optionally converts decimal numeric entities to the hexadecimal ones. For this, '$config["hexdec_entity"]' should be '2'. + + `Neutralization` refers to the `entitification` of '&' to '&'. + + *Note*: htmLawed does not convert entities to the actual characters represented by them; one can pass the htmLawed output through PHP's 'html_entity_decode' function:- http://www.php.net/html_entity_decode for that. + + *Note*: If '$config["and_mark"]' is set, and set to a value other than '0', then the '&' characters in the original input are replaced with the control character for the hexadecimal code-point '6' ('\x06'; '&' characters introduced by htmLawed, e.g., after converting '<' to '<', are not affected). This allows one to distinguish, say, an '>' introduced by htmLawed and an '>' put in by the input writer, and can be helpful in further processing of the htmLawed-processed text (e.g., to identify the character sequence 'o(><)o' to generate an emoticon image). When this feature is active, admins should ensure that the htmLawed output is not directly used in web pages or XML documents as the presence of the '\x06' can break documents. Before use in such documents, and preferably before any storage, any remaining '\x06' should be changed back to '&', e.g., with: + + $final = str_replace("\x06", '&', $prelim); + + Also, see section:- #3.9. + + +-- 3.3 HTML elements ----------------------------------------------o + + + htmLawed can be configured to allow only certain HTML elements (tags) in the input. Disallowed elements (just tag-content, and not element-content), based on '$config["keep_bad"]', are either `neutralized` (converted to plain text by entitification of '<' and '>') or removed. + + E.g., with only 'em' permitted: + + Input: + + <em>My</em> website is <a href="http://a.com>a.com</a>. + + Output, with '$config["keep_bad"] = 0': + + <em>My</em> website is a.com. + + Output, with '$config["keep_bad"]' not '0': + + <em>My</em> website is <a href="">a.com</a>. + + See section:- #3.3.3 for differences between the various non-zero '$config["keep_bad"]' values. + + htmLawed by default permits these 86 elements: + + a, abbr, acronym, address, applet, area, b, bdo, big, blockquote, br, button, caption, center, cite, code, col, colgroup, dd, del, dfn, dir, div, dl, dt, em, embed, fieldset, font, form, h1, h2, h3, h4, h5, h6, hr, i, iframe, img, input, ins, isindex, kbd, label, legend, li, map, menu, noscript, object, ol, optgroup, option, p, param, pre, q, rb, rbc, rp, rt, rtc, ruby, s, samp, script, select, small, span, strike, strong, sub, sup, table, tbody, td, textarea, tfoot, th, thead, tr, tt, u, ul, var + + Except for 'embed' (included because of its wide-spread use) and the Ruby elements ('rb', 'rbc', 'rp', 'rt', 'rtc', 'ruby'; part of XHTML 1.1), these are all the elements in the HTML 4/XHTML 1 specs. Strict-specific specs. exclude 'center', 'dir', 'font', 'isindex', 'menu', 's', 'strike', and 'u'. + + With '$config["safe"] = 1', the default set will exclude 'applet', 'embed', 'iframe', 'object' and 'script'; see section:- #3.6. + + When '$config["elements"]', which specifies allowed elements, is `properly` defined, and neither empty nor set to '0' or '*', the default set is not used. To have elements added to or removed from the default set, a '+/-' notation is used. E.g., '*-script-object' implies that only 'script' and 'object' are disallowed, whereas '*+embed' means that 'noembed' is also allowed. Elements can also be specified as comma separated names. E.g., 'a, b, i' means only 'a', 'b' and 'i' are permitted. In this notation, '*', '+' and '-' have no significance and can actually cause a mis-reading. + + Some more examples of '$config["elements"]' values indicating permitted elements (note that empty spaces are liberally allowed for clarity): + + * 'a, blockquote, code, em, strong' -- only 'a', 'blockquote', 'code', 'em', and 'strong' + * '*-script' -- all excluding 'script' + * '* -center -dir -font -isindex -menu -s -strike -u' -- only XHTML-Strict elements + * '*+noembed-script' -- all including 'noembed' excluding 'script' + + Some mis-usages (and the resulting permitted elements) that can be avoided: + + * '-*' -- none; instead of htmLawed, one might just use, e.g., the 'htmlspecialchars()' PHP function + * '*, -script' -- all except 'script'; admin probably meant '*-script' + * '-*, a, em, strong' -- all; admin probably meant 'a, em, strong' + * '*' -- all; admin need not have set 'elements' + * '*-form+form' -- all; a '+' will always over-ride any '-' + * '*, noembed' -- only 'noembed'; admin probably meant '*+noembed' + * 'a, +b, i' -- only 'a' and 'i'; admin probably meant 'a, b, i' + + Basically, when using the '+/-' notation, commas (',') should not be used, and vice versa, and '*' should be used with the former but not the latter. + + *Note*: Even if an element that is not in the default set is allowed through '$config["elements"]', like 'noembed' in the last example, it will eventually be removed during tag balancing unless such balancing is turned off ('$config["balance"]' set to '0'). Currently, the only way around this, which actually is simple, is to edit the various arrays in the function 'hl_bal()' to accommodate the element and its nesting properties. + + *A possibly second way to specify allowed elements* is to set '$config["parent"]' to an element name that supposedly will hold the input, and to set '$config["balance"]' to '1'. During tag balancing (see section:- #3.3.3), all elements that cannot legally nest inside the parent element will be removed. The parent element is auto-reset to 'div' if '$config["parent"]' is empty, 'body', or an element not in htmLawed's default set of 86 elements. + + `Tag transformation` is possible for improving XHTML-Strict compliance -- most of the deprecated elements are removed or converted to valid XHTML-Strict ones; see section:- #3.3.2. + + +.. 3.3.1 Handling of comments and CDATA sections ................... + + + 'CDATA' sections have the format '<![CDATA[...anything but not "]]>"...]]>', and HTML comments, '<!--...anything but not "-->"... -->'. Neither HTML comments nor 'CDATA' sections can reside inside tags. HTML comments can exist anywhere else, but 'CDATA' sections can exist only where plain text is allowed (e.g., immediately inside 'td' element content but not immediately inside 'tr' element content). + + htmLawed (function 'hl_cmtcd()') handles HTML comments or 'CDATA' sections depending on the values of '$config["comment"]' or '$config["cdata"]'. If '0', such markup is not looked for and the text is processed like plain text. If '1', it is removed completely. If '2', it is preserved but any '<', '>' and '&' inside are changed to entities. If '3', they are left as such. + + Note that for the last two cases, HTML comments and 'CDATA' sections will always be removed from tag content (function 'hl_tag()'). + + Examples: + + Input: + <!-- home link --><a href="home.htm"><![CDATA[x=&y]]>Home</a> + Output ('$config["comment"] = 0, $config["cdata"] = 2'): + <-- home link --><a href="home.htm"><![CDATA[x=&y]]>Home</a> + Output ('$config["comment"] = 1, $config["cdata"] = 2'): + <a href="home.htm"><![CDATA[x=&y]]>Home</a> + Output ('$config["comment"] = 2, $config["cdata"] = 2'): + <!-- home link --><a href="home.htm"><![CDATA[x=&y]]>Home</a> + Output ('$config["comment"] = 2, $config["cdata"] = 1'): + <!-- home link --><a href="home.htm">Home</a> + Output ('$config["comment"] = 3, $config["cdata"] = 3'): + <!-- home link --><a href="home.htm"><![CDATA[x=&y]]>Home</a> + + For standard-compliance, comments are given the form '<!--comment -->', and any '--' in the content is made '-'. + + When '$config["safe"] = 1', CDATA sections and comments are considered plain text unless '$config["comment"]' or '$config["cdata"]' is explicitly specified; see section:- #3.6. + + +.. 3.3.2 Tag-transformation for better XHTML-Strict ................o + + + If '$config["make_tag_strict"]' is set and not '0', following non-XHTML-Strict elements (and attributes), even if admin-permitted, are mutated as indicated (element content remains intact; function 'hl_tag2()'): + + * applet - (based on '$config["make_tag_strict"]', unchanged ('1') or removed ('2')) + * center - 'div style="text-align: center;"' + * dir - 'ul' + * embed - (based on '$config["make_tag_strict"]', unchanged ('1') or removed ('2')) + * font (face, size, color) - 'span style="font-family: ; font-size: ; color: ;"' (size transformation reference:- http://style.cleverchimp.com/font_size_intervals/altintervals.html) + * isindex - (based on '$config["make_tag_strict"]', unchanged ('1') or removed ('2')) + * menu - 'ul' + * s - 'span style="text-decoration: line-through;"' + * strike - 'span style="text-decoration: line-through;"' + * u - 'span style="text-decoration: underline;"' + + For an element with a pre-existing 'style' attribute value, the extra style properties are appended. + + Example input: + + <center> + The PHP <s>software</s> script used for this <strike>web-page</strike> web-page is <font style="font-weight: bold " face=arial size='+3' color = "red ">htmLawedTest.php</font>, from <u style= 'color:green'>PHP Labware</u>. + </center> + + The output: + + <div style="text-align: center;"> + The PHP <span style="text-decoration: line-through;">software</span> script used for this <span style="text-decoration: line-through;">web-page</span> web-page is <span style="font-weight: bold; font-family: arial; color: red; font-size: 200%;">htmLawedTest.php</span>, from <span style="color:green; text-decoration: underline;">PHP Labware</span>. + </div> + + +-- 3.3.3 Tag balancing and proper nesting -------------------------o + + + If '$config["balance"]' is set to '1', htmLawed (function 'hl_bal()') checks and corrects the input to have properly balanced tags and legal element content (i.e., any element nesting should be valid, and plain text may be present only in the content of elements that allow them). + + Depending on the value of '$config["keep_bad"]' (see section:- #2.2 and section:- #3.3), illegal content may be removed or neutralized to plain text by converting < and > to entities: + + '0' - remove; this option is available only to maintain Kses-compatibility and should not be used otherwise (see section:- #2.6) + '1' - neutralize tags and keep element content + '2' - remove tags but keep element content + '3' and '4' - like '1' and '2', but keep element content only if text ('pcdata') is valid in parent element as per specs + '5' and '6' - like '3' and '4', but line-breaks, tabs and spaces are left + + Example input (disallowing the 'p' element): + + <*> Pseudo-tags <*> + <xml>Non-HTML tag xml</xml> + <p> + Disallowed tag p + </p> + <ul>Bad<li>OK</li></ul> + + The output with '$config["keep_bad"] = 1': + + <*> Pseudo-tags <*> + <xml>Non-HTML tag xml</xml> + <p> + Disallowed tag p + </p> + <ul>Bad<li>OK</li></ul> + + The output with '$config["keep_bad"] = 3': + + <*> Pseudo-tags <*> + <xml>Non-HTML tag xml</xml> + <p> + Disallowed tag p + </p> + <ul><li>OK</li></ul> + + The output with '$config["keep_bad"] = 6': + + <*> Pseudo-tags <*> + Non-HTML tag xml + + Disallowed tag p + + <ul><li>OK</li></ul> + + An option like '1' is useful, e.g., when a writer previews his submission, whereas one like '3' is useful before content is finalized and made available to all. + + *Note:* In the example above, unlike '<*>', '<xml>' gets considered as a tag (even though there is no HTML element named 'xml'). In general, text matching the regular expression pattern '<(/?)([a-zA-Z][a-zA-Z1-6]*)([^>]*?)\s?>' is considered a tag (phrase enclosed by the angled brackets '<' and '>', and starting [with an optional slash preceding] with an alphanumeric word that starts with an alphabet...). + + Nesting/content rules for each of the 86 elements in htmLawed's default set (see section:- #3.3) are defined in function 'hl_bal()'. This means that if a non-standard element besides 'embed' is being permitted through '$config["elements"]', the element's tag content will end up getting removed if '$config["balance"]' is set to '1'. + + Plain text and/or certain elements nested inside 'blockquote', 'form', 'map' and 'noscript' need to be in block-level elements. This point is often missed during manual writing of HTML code. htmLawed attempts to address this during balancing. E.g., if the parent container is set as 'form', the input 'B:<input type="text" value="b" />C:<input type="text" value="c" />' is converted to '<div>B:<input type="text" value="b" />C:<input type="text" value="c" /></div>'. + + +-- 3.3.4 Elements requiring child elements ------------------------o + + + As per specs, the following elements require legal child elements nested inside them: + + blockquote, dir, dl, form, map, menu, noscript, ol, optgroup, rbc, rtc, ruby, select, table, tbody, tfoot, thead, tr, ul + + In some cases, the specs stipulate the number and/or the ordering of the child elements. A 'table' can have 0 or 1 'caption', 'tbody', 'tfoot', and 'thead', but they must be in this order: 'caption', 'thead', 'tfoot', 'tbody'. + + htmLawed currently does not check for conformance to these rules. Note that any non-compliance in this regard will not introduce security vulnerabilities, crash browser applications, or affect the rendering of web-pages. + + +-- 3.3.5 Beautify or compact HTML ---------------------------------o + + + By default, htmLawed will neither `beautify` HTML code by formatting it with indentations, etc., nor will it make it compact by removing un-needed white-space.(It does always properly white-space tag content.) + + As per the HTML standards, spaces, tabs and line-breaks in web-pages (except those inside 'pre' elements) are all considered equivalent, and referred to as `white-spaces`. Browser applications are supposed to consider contiguous white-spaces as just a single space, and to disregard white-spaces trailing opening tags or preceding closing tags. This white-space `normalization` allows the use of text/code beautifully formatted with indentations and line-spacings for readability. Such `pretty` HTML can, however, increase the size of web-pages, or make the extraction or scraping of plain text cumbersome. + + With the '$config' parameter 'tidy', htmLawed can be used to beautify or compact the input text. Input with just plain text and no HTML markup is also subject to this. Besides 'pre', the 'script' and 'textarea' elements, CDATA sections, and HTML comments are not subjected to the tidying process. + + To `compact`, use '$config["tidy"] = -1'; single instances or runs of white-spaces are replaced with a single space, and white-spaces trailing and leading open and closing tags, respectively, are removed. + + To `beautify`, '$config["tidy"]' is set as '1', or for customized tidying, as a string like '2s2n'. The 's' or 't' character specifies the use of spaces or tabs for indentation. The first and third characters, any of the digits 0-9, specify the number of spaces or tabs per indentation, and any parental lead spacing (extra indenting of the whole block of input text). The 'r' and 'n' characters are used to specify line-break characters: 'n' for '\n' (Unix/Mac OS X line-breaks), 'rn' or 'nr' for '\r\n' (Windows/DOS line-breaks), or 'r' for '\r'. + + The '$config["tidy"]' value of '1' is equivalent to '2s0n'. Other '$config["tidy"]' values are read loosely: a value of '4' is equivalent to '4s0n'; 't2', to '1t2n'; 's', to '2s0n'; '2TR', to '2t0r'; 'T1', to '1t1n'; 'nr3', to '3s0nr', and so on. Except in the indentations and line-spacings, runs of white-spaces are replaced with a single space during beautification. + + Input formatting using '$config["tidy"]' is not recommended when input text has mixed markup (like HTML + PHP). + + +-- 3.4 Attributes ------------------------------------------------oo + + + htmLawed will only permit attributes described in the HTML specs (including deprecated ones). It also permits some attributes for use with the 'embed' element (the non-standard 'embed' element is supported in htmLawed because of its widespread use), and the the 'xml:space' attribute (valid only in XHTML 1.1). A list of such 111 attributes and the elements they are allowed in is in section:- #5.2. + + When '$config["deny_attribute"]' is not set, or set to '0', or empty ('""'), all the 111 attributes are permitted. Otherwise, '$config["deny_attribute"]' can be set as a list of comma-separated names of the denied attributes. 'on*' can be used to refer to the group of potentially dangerous, script-accepting attributes: 'onblur', 'onchange', 'onclick', 'ondblclick', 'onfocus', 'onkeydown', 'onkeypress', 'onkeyup', 'onmousedown', 'onmousemove', 'onmouseout', 'onmouseover', 'onmouseup', 'onreset', 'onselect' and 'onsubmit'. + + Note that attributes specified in '$config["deny_attribute"]' are denied globally, for all elements. To deny attributes for only specific elements, '$spec' (see section:- #2.3) can be used. '$spec' can also be used to element-specifically permit an attribute otherwise denied through '$config["deny_attribute"]'. + + With '$config["safe"] = 1' (section:- #3.6), the 'on*' attributes are automatically disallowed. + + *Note*: To deny all but a few attributes globally, a simpler way to specify '$config["deny_attribute"]' would be to use the notation '* -attribute1 -attribute2 ...'. Thus, a value of '* -title -href' implies that except 'href' and 'title' (where allowed as per standards) all other attributes are to be removed. With this notation, the value for the parameter 'safe' (section:- #3.6) will have no effect on 'deny_attribute'. + + htmLawed (function 'hl_tag()') also: + + * Lower-cases attribute names + * Removes duplicate attributes (last one stays) + * Gives attributes the form 'name="value"' and single-spaces them, removing unnecessary white-spacing + * Provides `required` attributes (see section:- #3.4.1) + * Double-quotes values and escapes any '"' inside them + * Replaces the possibly dangerous soft-hyphen characters (hexadecimal code-point 'ad') in the values with spaces + * Allows custom function to additionally filter/modify attribute values (see section:- #3.4.9) + + +.. 3.4.1 Auto-addition of XHTML-required attributes ................ + + + If indicated attributes for the following elements are found missing, htmLawed (function 'hl_tag()') will add them (with values same as attribute names unless indicated otherwise below): + + * area - alt ('area') + * area, img - src, alt ('image') + * bdo - dir ('ltr') + * form - action + * map - name + * optgroup - label + * param - name + * script - type ('text/javascript') + * textarea - rows ('10'), cols ('50') + + Additionally, with '$config["xml:lang"]' set to '1' or '2', if the 'lang' but not the 'xml:lang' attribute is declared, then the latter is added too, with a value copied from that of 'lang'. This is for better standard-compliance. With '$config["xml:lang"]' set to '2', the 'lang' attribute is removed (XHTML 1.1 specs). + + Note that the 'name' attribute for 'map', invalid in XHTML 1.1, is also transformed if required -- see section:- #3.4.6. + + +.. 3.4.2 Duplicate/invalid 'id' values ............................o + + + If '$config["unique_ids"]' is '1', htmLawed (function 'hl_tag()') removes 'id' attributes with values that are not XHTML-compliant (must begin with a letter and can contain letters, digits, ':', '.', '-' and '_') or duplicate. If '$config["unique_ids"]' is a word, any duplicate but otherwise valid value will be appropriately prefixed with the word to ensure its uniqueness. The word should begin with a letter and should contain only letters, numbers, ':', '.', '_' and '-'. + + Even if multiple inputs need to be filtered (through multiple calls to htmLawed), htmLawed ensures uniqueness of 'id' values as it uses a global variable ('$GLOBALS["hl_Ids"]' array). Further, an admin can restrict the use of certain 'id' values by presetting this variable before htmLawed is called into use. E.g.: + + $GLOBALS['hl_Ids'] = array('top'=>1, 'bottom'=>1, 'myform'=>1); // id values not allowed in input + $processed = htmLawed($text); // filter input + + +.. 3.4.3 URL schemes (protocols) and scripts in attribute values ............o + + + htmLawed edits attributes that take URLs as values if they are found to contain un-permitted schemes. E.g., if the 'afp' scheme is not permitted, then '<a href="afp://domain.org">' becomes '<a href="denied:afp://domain.org">', and if Javascript is not permitted '<a onclick="javascript:xss();">' becomes '<a onclick="denied:javascript:xss();">'. + + By default htmLawed permits these schemes in URLs for the 'href' attribute: + + aim, feed, file, ftp, gopher, http, https, irc, mailto, news, nntp, sftp, ssh, telnet + + Also, only 'file', 'http' and 'https' are permitted in attributes whose names start with 'o' (like 'onmouseover'), and in these attributes that accept URLs: + + action, cite, classid, codebase, data, href, longdesc, model, pluginspage, pluginurl, src, style, usemap + + These default sets are used when '$config["schemes"]' is not set (see section:- #2.2). To over-ride the defaults, '$config["schemes"]' is defined as a string of semi-colon-separated sub-strings of type 'attribute: comma-separated schemes'. E.g., 'href: mailto, http, https; onclick: javascript; src: http, https'. For unspecified attributes, 'file', 'http' and 'https' are permitted. This can be changed by passing schemes for '*' in '$config["schemes"]'. E.g., 'href: mailto, http, https; *: https, https'. + + '*' can be put in the list of schemes to permit all protocols. E.g., 'style: *; img: http, https' results in protocols not being checked in 'style' attribute values. However, in such cases, any relative-to-absolute URL conversion, or vice versa, (section:- #3.4.4) is not done. + + Thus, `to allow Javascript`, one can set '$config["schemes"]' as 'href: mailto, http, https; *: http, https, javascript', or 'href: mailto, http, https, javascript; *: http, https, javascript', or '*: *', and so on. + + As a side-note, one may find 'style: *' useful as URLs in 'style' attributes can be specified in a variety of ways, and the patterns that htmLawed uses to identify URLs may mistakenly identify non-URL text. + + *Note*: If URL-accepting attributes other than those listed above are being allowed, then the scheme will not be checked unless the attribute name contains the string 'src' (e.g., 'dynsrc') or starts with 'o' (e.g., 'onbeforecopy'). + + With '$config["safe"] = 1', all URLs are disallowed in the 'style' attribute values. + + +.. 3.4.4 Absolute & relative URLs in attribute values .............o + + + htmLawed can make absolute URLs in attributes like 'href' relative ('$config["abs_url"]' is '-1'), and vice versa ('$config["abs_url"]' is '1'). URLs in scripts are not considered for this, and so are URLs like '#section_6' (fragment), '?name=Tim#show' (starting with query string), and ';var=1?name=Tim#show' (starting with parameters). Further, this requires that '$config["base_url"]' be set properly, with the '://' and a trailing slash ('/'), with no query string, etc. E.g., 'file:///D:/page/', 'https://abc.com/x/y/', or 'http://localhost/demo/' are okay, but 'file:///D:/page/?help=1', 'abc.com/x/y/' and 'http://localhost/demo/index.htm' are not. + + For making absolute URLs relative, only those URLs that have the '$config["base_url"]' string at the beginning are converted. E.g., with '$config["base_url"] = "https://abc.com/x/y/"', 'https://abc.com/x/y/a.gif' and 'https://abc.com/x/y/z/b.gif' become 'a.gif' and 'z/b.gif' respectively, while 'https://abc.com/x/c.gif' is not changed. + + When making relative URLs absolute, only values for scheme, network location (host-name) and path values in the base URL are inherited. See section:- #5.5 for more about the URL specification as per RFC 1808:- http://www.ietf.org/rfc/rfc1808.txt. + + +.. 3.4.5 Lower-cased, standard attribute values ....................o + + + Optionally, for standard-compliance, htmLawed (function 'hl_tag()') lower-cases standard attribute values to give, e.g., 'input type="password"' instead of 'input type="Password"', if '$config["lc_std_val"]' is '1'. Attribute values matching those listed below for any of the elements (plus those for the 'type' attribute of 'button' or 'input') are lower-cased: + + all, baseline, bottom, button, center, char, checkbox, circle, col, colgroup, cols, data, default, file, get, groups, hidden, image, justify, left, ltr, middle, none, object, password, poly, post, preserve, radio, rect, ref, reset, right, row, rowgroup, rows, rtl, submit, text, top + + a, area, bdo, button, col, form, img, input, object, option, optgroup, param, script, select, table, td, tfoot, th, thead, tr, xml:space + + The following `empty` (`minimized`) attributes are always assigned lower-cased values (same as the names): + + checked, compact, declare, defer, disabled, ismap, multiple, nohref, noresize, noshade, nowrap, readonly, selected + + +.. 3.4.6 Transformation of deprecated attributes ..................o + + + If '$config["no_deprecated_attr"]' is '0', then deprecated attributes (see appendix in section:- #5.2) are removed and, in most cases, their values are transformed to CSS style properties and added to the 'style' attributes (function 'hl_tag()'). Except for 'bordercolor' for 'table', 'tr' and 'td', the scores of proprietary attributes that were never part of any cross-browser standard are not supported. + + *Note*: The attribute 'target' for 'a' is allowed even though it is not in XHTML 1.0 specs. This is because of the attribute's wide-spread use and browser-support, and because the attribute is valid in XHTML 1.1 onwards. + + * align - for 'img' with value of 'left' or 'right', becomes, e.g., 'float: left'; for 'div' and 'table' with value 'center', becomes 'margin: auto'; all others become, e.g., 'text-align: right' + + * bgcolor - E.g., 'bgcolor="#ffffff"' becomes 'background-color: #ffffff' + * border - E.g., 'height= "10"' becomes 'height: 10px' + * bordercolor - E.g., 'bordercolor=#999999' becomes 'border-color: #999999;' + * compact - 'font-size: 85%' + * clear - E.g., 'clear="all" becomes 'clear: both' + + * height - E.g., 'height= "10"' becomes 'height: 10px' and 'height="*"' becomes 'height: auto' + + * hspace - E.g., 'hspace="10"' becomes 'margin-left: 10px; margin-right: 10px' + * language - 'language="VBScript"' becomes 'type="text/vbscript"' + * name - E.g., 'name="xx"' becomes 'id="xx"' + * noshade - 'border-style: none; border: 0; background-color: gray; color: gray' + * nowrap - 'white-space: nowrap' + * size - E.g., 'size="10"' becomes 'height: 10px' + * start - removed + * type - E.g., 'type="i"' becomes 'list-style-type: lower-roman' + * value - removed + * vspace - E.g., 'vspace="10"' becomes 'margin-top: 10px; margin-bottom: 10px' + * width - like 'height' + + Example input: + + <img src="j.gif" alt="image" name="dad's" /><img src="k.gif" alt="image" id="dad_off" name="dad" /> + <br clear="left" /> + <hr noshade size="1" /> + <img name="img" src="i.gif" align="left" alt="image" hspace="10" vspace="10" width="10em" height="20" border="1" style="padding:5px;" /> + <table width="50em" align="center" bgcolor="red"> + <tr> + <td width="20%"> + <div align="center"> + <h3 align="right">Section</h3> + <p align="right">Para</p> + <ol type="a" start="e"><li value="x">First item</li></ol> + </div> + </td> + <td width="*"> + <ol type="1"><li>First item</li></ol> + </td> + </tr> + </table> + <br clear="all" /> + + And the output with '$config["no_deprecated_attr"] = 1': + + <img src="j.gif" alt="image" /><img src="k.gif" alt="image" id="dad_off" /> + <br style="clear: left;" /> + <hr style="border-style: none; border: 0; background-color: gray; color: gray; size: 1px;" /> + <img src="i.gif" alt="image" width="10em" height="20" style="padding:5px; float: left; margin-left: 10px; margin-right: 10px; margin-top: 10px; margin-bottom: 10px; border: 1px;" id="img" /> + <table width="50em" style="margin: auto; background-color: red;"> + <tr> + <td style="width: 20%;"> + <div style="margin: auto;"> + <h3 style="text-align: right;">Section</h3> + <p style="text-align: right;">Para</p> + <ol style="list-style-type: lower-latin;"><li>First item</li></ol> + </div> + </td> + <td style="width: auto;"> + <ol style="list-style-type: decimal;"><li>First item</li></ol> + </td> + </tr> + </table> + <br style="clear: both;" /> + + For 'lang', deprecated in XHTML 1.1, transformation is taken care of through '$config["xml:lang"]'; see section:- #3.4.1. + + The attribute 'name' is deprecated in 'form', 'iframe', and 'img', and is replaced with 'id' if an 'id' attribute doesn't exist and if the 'name' value is appropriate for 'id'. For such replacements for 'a' and 'map', for which the 'name' attribute is deprecated in XHTML 1.1, '$config["no_deprecated_attr"]' should be set to '2' (when set to '1', for these two elements, the 'name' attribute is retained). + + +-- 3.4.7 Anti-spam & 'href' ---------------------------------------o + + + htmLawed (function 'hl_tag()') can check the 'href' attribute values (link addresses) as an anti-spam (email or link spam) measure. + + If '$config["anti_mail_spam"]' is not '0', the '@' of email addresses in 'href' values like 'mailto:a@b.com' is replaced with text specified by '$config["anti_mail_spam"]'. The text should be of a form that makes it clear to others that the address needs to be edited before a mail is sent; e.g., '<remove_this_antispam>@' (makes the example address 'a<remove_this_antispam>@b.com'). + + For regular links, one can choose to have a 'rel' attribute with 'nofollow' in its value (which tells some search engines to not follow a link). This can discourage link spammers. Additionally, or as an alternative, one can choose to empty the 'href' value altogether (disable the link). + + For use of these options, '$config["anti_link_spam"]' should be set as an array with values 'regex1' and 'regex2', both or one of which can be empty (like 'array("", "regex2")') to indicate that that option is not to be used. Otherwise, 'regex1' or 'regex2' should be PHP- and PCRE-compatible regular expression patterns: 'href' values will be matched against them and those matching the pattern will accordingly be treated. + + Note that the regular expressions should have `delimiters`, and be well-formed and preferably fast. Absolute efficiency/accuracy is often not needed. + + An example, to have a 'rel' attribute with 'nofollow' for all links, and to disable links that do not point to domains 'abc.com' and 'xyz.org': + + $config["anti_link_spam"] = array('`.`', '`://\W*(?!(abc\.com|xyz\.org))`'); + + +-- 3.4.8 Inline style properties ----------------------------------o + + + htmLawed can check URL schemes and dynamic expressions (to guard against Javascript, etc., script-based insecurities) in inline CSS style property values in the 'style' attributes. (CSS properties like 'background-image' that accept URLs in their values are noted in section:- #5.3.) Dynamic CSS expressions that allow scripting in the IE browser, and can be a vulnerability, can be removed from property values by setting '$config["css_expression"]' to '1' (default setting). + + *Note*: Because of the various ways of representing characters in attribute values (URL-escapement, entitification, etc.), htmLawed might alter the values of the 'style' attribute values, and may even falsely identify dynamic CSS expressions and URL schemes in them. If this is an important issue, checking of URLs and dynamic expressions can be turned off ('$config["schemes"] = "...style:*..."', see section:- #3.4.3, and '$config["css_expression"] = 0'). Alternately, admins can use their own custom function for finer handling of 'style' values through the 'hook_tag' parameter (see section:- #3.4.9). + + It is also possible to have htmLawed let through any 'style' value by setting '$config["style_pass"]' to '1'. + + As such, it is better to set up a CSS file with class declarations, disallow the 'style' attribute, set a '$spec' rule (see section:- #2.3) for 'class' for the 'oneof' or 'match' parameter, and ask writers to make use of the 'class' attribute. + + +-- 3.4.9 Hook function for tag content ----------------------------o + + + It is possible to utilize a custom hook function to alter the tag content htmLawed has finalized (i.e., after it has checked/corrected for required attributes, transformed attributes, lower-cased attribute names, etc.). + + When '$config' parameter 'hook_tag' is set to the name of a function, htmLawed (function 'hl_tag()') will pass on the element name, and the `finalized` attribute name-value pairs as array elements to the function. The function is expected to return the full opening tag string like '<element_name attribute_1_name="attribute_1_value"...>' (for empty elements like 'img' and 'input', the element-closing slash '/' should also be included). + + This is a *powerful functionality* that can be exploited for various objectives: consolidate-and-convert inline 'style' attributes to 'class', convert 'embed' elements to 'object', permit only one 'caption' element in a 'table' element, disallow embedding of certain types of media, *inject HTML*, use CSSTidy:- http://csstidy.sourceforge.net to sanitize 'style' attribute values, etc. + + As an example, the custom hook code below can be used to force a series of specifically ordered 'id' attributes on all elements, and a specific 'param' element inside all 'object' elements: + + function my_tag_function($element, $attribute_array){ + static $id = 0; + // Remove any duplicate element + if($element == 'param' && isset($attribute_array['allowscriptaccess'])){ + return ''; + } + + $new_element = ''; + + // Force a serialized ID number + $attribute_array['id'] = 'my_'. $id; + ++$id; + + // Inject param for allowscriptaccess + if($element == 'object'){ + $new_element = '<param id='my_'. $id; allowscriptaccess="never" />'; + ++$id; + } + + $string = ''; + foreach($attribute_array as $k=>$v){ + $string .= " {$k}=\"{$v}\""; + } + return "<{$element}{$string}". (isset($in_array($element, $empty_elements) ? ' /' : ''). '>'. $new_element; + } + + The 'hook_tag' parameter is different from the 'hook' parameter (section:- #3.7). + + Snippets of hook function code developed by others may be available on the htmLawed:- http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed website. + + +-- 3.5 Simple configuration directive for most valid XHTML -------oo + + + If '$config["valid_xhtml"]' is set to '1', some relevant '$config' parameters (indicated by '~' in section:- #2.2) are auto-adjusted. This allows one to pass the '$config' argument with a simpler value. If a value for a parameter auto-set through 'valid_xhtml' is still manually provided, then that value will over-ride the auto-set value. + + +-- 3.6 Simple configuration directive for most `safe` HTML --------o + + + `Safe` HTML refers to HTML that is restricted to reduce the vulnerability for scripting attacks (such as XSS) based on HTML code which otherwise may still be legal and compliant with the HTML standard specs. When elements such as 'script' and 'object', and attributes such as 'onmouseover' and 'style' are allowed in the input text, an input writer can introduce malevolent HTML code. Note that what is considered 'safe' depends on the nature of the web application and the trust-level accorded to its users. + + htmLawed allows an admin to use '$config["safe"]' to auto-adjust multiple '$config' parameters (such as 'elements' which declares the allowed element-set), which otherwise would have to be manually set. The relevant parameters are indicated by '"' in section:- #2.2). Thus, one can pass the '$config' argument with a simpler value. + + With the value of '1', htmLawed considers 'CDATA' sections and HTML comments as plain text, and prohibits the 'applet', 'embed', 'iframe', 'object' and 'script' elements, and the 'on*' attributes like 'onclick'. ( There are '$config' parameters like 'css_expression' that are not affected by the value set for 'safe' but whose default values still contribute towards a more `safe` output.) Further, URLs with schemes (see section:- #3.4.3) are neutralized so that, e.g., 'style="moz-binding:url(http://danger)"' becomes 'style="moz-binding:url(denied:http://danger)"' while 'style="moz-binding:url(ok)"' remains intact. + + Admins, however, may still want to completely deny the 'style' attribute, e.g., with code like + + $processed = htmLawed($text, array('safe'=>1, 'deny_attribute'=>'style')); + + If a value for a parameter auto-set through 'safe' is still manually provided, then that value can over-ride the auto-set value. E.g., with '$config["safe"] = 1' and '$config["elements"] = "*+script"', 'script', but not 'applet', is allowed. + + A page illustrating the efficacy of htmLawed's anti-XSS abilities with 'safe' set to '1' against XSS vectors listed by RSnake:- http://ha.ckers.org/xss.html may be available here:- http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed/rsnake/RSnakeXSSTest.htm. + + +-- 3.7 Using a hook function --------------------------------------o + + + If '$config["hook"]' is not set to '0', then htmLawed will allow preliminarily processed input to be altered by a hook function named by '$config["hook"]' before starting the main work (but after handling of characters, entities, HTML comments and 'CDATA' sections -- see code for function 'htmLawed()'). + + The hook function also allows one to alter the `finalized` values of '$config' and '$spec'. + + Note that the 'hook' parameter is different from the 'hook_tag' parameter (section:- #3.4.9). + + Snippets of hook function code developed by others may be available on the htmLawed:- http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed website. + + +-- 3.8 Obtaining `finalized` parameter values ---------------------o + + + htmLawed can assign the `finalized` '$config' and '$spec' values to a variable named by '$config["show_setting"]'. The variable, made global by htmLawed, is set as an array with three keys: 'config', with the '$config' value, 'spec', with the '$spec' value, and 'time', with a value that is the Unix time (the output of PHP's 'microtime()' function) when the value was assigned. Admins should use a PHP-compliant variable name (e.g., one that does not begin with a numerical digit) that does not conflict with variable names in their non-htmLawed code. + + The values, which are also post-hook function (if any), can be used to auto-generate information (on, e.g., the elements that are permitted) for input writers. + + +-- 3.9 Retaining non-HTML tags in input with mixed markup ---------o + + + htmLawed does not remove certain characters that though invalid are nevertheless discouraged in HTML documents as per the specs (see section:- #5.1). This can be utilized to deal with input that contains mixed markup. Input that may have HTML markup as well as some other markup that is based on the '<', '>' and '&' characters is considered to have mixed markup. The non-HTML markup can be rather proprietary (like markup for emoticons/smileys), or standard (like MathML or SVG). Or it can be programming code meant for execution/evaluation (such as embedded PHP code). + + To deal with such mixed markup, the input text can be pre-processed to hide the non-HTML markup by specifically replacing the '<', '>' and '&' characters with some of the HTML-discouraged characters (see section:- #3.1.2). Post-htmLawed processing, the replacements are reverted. + + An example (mixed HTML and PHP code in input text): + + $text = preg_replace('`<\?php(.+?)\?>`sm', "\x83?php\\1?\x84", $text); + $processed = htmLawed($text); + $processed = preg_replace('`\x83\?php(.+?)\?\x84`sm', '<?php$1?>', $processed); + + This code will not work if '$config["clean_ms_char"]' is set to '1' (section:- #3.1), in which case one should instead deploy a hook function (section:- #3.7). (htmLawed internally uses certain control characters, code-points '1' to '7', and use of these characters as markers in the logic of hook functions may cause issues.) + + Admins may also be able to use '$config["and_mark"]' to deal with such mixed markup; see section:- #3.2. + + +== 4 Other =======================================================oo + + +-- 4.1 Support ----------------------------------------------------- + + + A careful re-reading of this documentation will very likely answer your questions. + + Software updates and forum-based community-support may be found at http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed. For general PHP issues (not htmLawed-specific), support may be found through internet searches and at http://php.net. + + +-- 4.2 Known issues -----------------------------------------------o + + + See section:- #2.8. + + Readers are advised to cross-check information given in this document. + + +-- 4.3 Change-log -------------------------------------------------o + + + (The release date for the downloadable package of files containing documentation, demo script, test-cases, etc., besides the 'htmLawed.php' file may be updated independently if the secondary files are revised.) + + `Version number - Release date. Notes` + + 1.1.8.1 - 16 July 2009. Minor code-change to fix a PHP error notice + + 1.1.8 - 23 April 2009. Parameter 'deny_attribute' now accepts the wild-card '*', making it simpler to specify its value when all but a few attributes are being denied; fixed a bug in interpreting '$spec' + + 1.1.7 - 11-12 March 2009. Attributes globally denied through 'deny_attribute' can be allowed element-specifically through '$spec'; '$config["style_pass"]' allowing letting through any 'style' value introduced; altered logic to catch certain types of dynamic crafted CSS expressions + + 1.1.3-6 - 28-31 January - 4 February 2009. Altered logic to catch certain types of dynamic crafted CSS expressions + + 1.1.2 - 22 January 2009. Fixed bug in parsing of 'font' attributes during tag transformation + + 1.1.1 - 27 September 2008. Better nesting correction when omitable closing tags are absent + + 1.1 - 29 June 2008. '$config["hook_tag"]' and '$config["format"]' introduced for custom tag/attribute check/modification/injection and output compaction/beautification; fixed a regex-in-$spec parsing bug + + 1.0.9 - 11 June 2008. Fixed bug in invalid HTML code-point entity check + + 1.0.8 - 15 May 2008. 'bordercolor' attribute for 'table', 'td' and 'tr' + + 1.0.7 - 1 May 2008. Support for 'wmode' attribute for 'embed'; '$config["show_setting"]' introduced; improved '$config["elements"]' evaluation + + 1.0.6 - 20 April 2008. '$config["and_mark"]' introduced + + 1.0.5 - 12 March 2008. 'style' URL schemes essentially disallowed when $config 'safe' is on; improved regex for CSS expression search + + 1.0.4 - 10 March 2008. Improved corrections for 'blockquote', 'form', 'map' and 'noscript' + + 1.0.3 - 3 March 2008. Character entities for soft-hyphens are now replaced with spaces (instead of being removed); a bug allowing 'td' directly inside 'table' fixed; 'safe' '$config' parameter added + + 1.0.2 - 13 February 2008. Improved implementation of '$config["keep_bad"]' + + 1.0.1 - 7 November 2007. Improved regex for identifying URLs, protocols and dynamic expressions ('hl_tag()' and 'hl_prot()'); no error display with 'hl_regex()' + + 1.0 - 2 November 2007. First release + + +-- 4.4 Testing ----------------------------------------------------o + + + To test htmLawed using a form interface, a demo:- htmLawedTest.php web-page is provided with the htmLawed distribution ('htmLawed.php' and 'htmLawedTest.php' should be in the same directory on the web-server). A file with test-cases:- htmLawed_TESTCASE.txt is also provided. + + +-- 4.5 Upgrade, & old versions ------------------------------------o + + + Upgrading is as simple as replacing the previous version of 'htmLawed.php' (assuming it was not modified for customized features). As htmLawed output is almost always used in static documents, upgrading should not affect old, finalized content. + + Old versions of htmLawed may be available online. E.g., for version 1.0, check http://www.bioinformatics.org/phplabware/downloads/htmLawed1.zip, for 1.1.1, htmLawed111.zip, and for 1.1.10, htmLawed1110.zip. + + +-- 4.6 Comparison with 'HTMLPurifier' -----------------------------o + + + The HTMLPurifier PHP library by Edward Yang is a very good HTML filtering script that uses object oriented PHP code. Compared to htmLawed, it: + + * does not support PHP versions older than 5.0 (HTMLPurifier dropped PHP 4 support after version 2) + + * is 15-20 times bigger (scores of files totalling more than 750 kb) + + * consumes 10-15 times more RAM memory (just including the HTMLPurifier files without calling the filter requires a few MBs of memory) + + * is expectedly slower + + * does not allow admins to fully allow all valid HTML (because of incomplete HTML support, it always considers elements like 'script' illegal) + + * lacks many of the extra features of htmLawed (like entity conversions and code compaction/beautification) + + * has poor documentation + + However, HTMLPurifier has finer checks for character encodings and attribute values, and can log warnings and errors. Visit the HTMLPurifier website:- http://htmlpurifier.org for updated information. + + +-- 4.7 Use through application plug-ins/modules -------------------o + + + Plug-ins/modules to implement htmLawed in applications such as Drupal and DokuWiki may have been developed. Please check the application websites and the forum on the htmLawed site:- http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed. + + +-- 4.8 Use in non-PHP applications --------------------------------o + + + Non-PHP applications written in Python, Ruby, etc., may be able to use htmLawed through system calls to the PHP engine. Such code may have been documented on the internet. Also check the forum on the htmLawed site:- http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed. + + +-- 4.9 Donate -----------------------------------------------------o + + + A donation in any currency and amount to appreciate or support this software can be sent by PayPal:- http://paypal.com to this email address: drpatnaik at yahoo dot com. + + +-- 4.10 Acknowledgements ------------------------------------------o + + + Bryan Blakey, Ulf Harnhammer, Gareth Heyes, Lukasz Pilorz, Shelley Powers, Edward Yang, and many anonymous users. + + Thank you! + + +== 5 Appendices ==================================================oo + + +-- 5.1 Characters discouraged in XHTML ----------------------------- + + + Characters represented by the following hexadecimal code-points are `not` invalid, even though some validators may issue messages stating otherwise. + + '7f' to '84', '86' to '9f', 'fdd0' to 'fddf', '1fffe', '1ffff', '2fffe', '2ffff', '3fffe', '3ffff', '4fffe', '4ffff', '5fffe', '5ffff', '6fffe', '6ffff', '7fffe', '7ffff', '8fffe', '8ffff', '9fffe', '9ffff', 'afffe', 'affff', 'bfffe', 'bffff', 'cfffe', 'cffff', 'dfffe', 'dffff', 'efffe', 'effff', 'ffffe', 'fffff', '10fffe' and '10ffff' + + +-- 5.2 Valid attribute-element combinations -----------------------o + + + Valid attribute-element combinations as per W3C specs. + + * includes deprecated attributes (marked '^'), attributes for the non-standard 'embed' element (marked '*'), and the proprietary 'bordercolor' (marked '~') + * only non-frameset, HTML body elements + * 'name' for 'a' and 'map', and 'lang' are invalid in XHTML 1.1 + * 'target' is valid for 'a' in XHTML 1.1 and higher + * 'xml:space' is only for XHTML 1.1 + + abbr - td, th + accept - form, input + accept-charset - form + accesskey - a, area, button, input, label, legend, textarea + action - form + align - caption^, embed, applet, iframe, img^, input^, object^, legend^, table^, hr^, div^, h1^, h2^, h3^, h4^, h5^, h6^, p^, col, colgroup, tbody, td, tfoot, th, thead, tr + alt - applet, area, img, input + archive - applet, object + axis - td, th + bgcolor - embed, table^, tr^, td^, th^ + border - table, img^, object^ + bordercolor~ - table, td, tr + cellpadding - table + cellspacing - table + char - col, colgroup, tbody, td, tfoot, th, thead, tr + charoff - col, colgroup, tbody, td, tfoot, th, thead, tr + charset - a, script + checked - input + cite - blockquote, q, del, ins + classid - object + clear - br^ + code - applet + codebase - object, applet + codetype - object + color - font + cols - textarea + colspan - td, th + compact - dir, dl^, menu, ol^, ul^ + coords - area, a + data - object + datetime - del, ins + declare - object + defer - script + dir - bdo + disabled - button, input, optgroup, option, select, textarea + enctype - form + face - font + for - label + frame - table + frameborder - iframe + headers - td, th + height - embed, iframe, td^, th^, img, object, applet + href - a, area + hreflang - a + hspace - applet, img^, object^ + ismap - img, input + label - option, optgroup + language - script^ + longdesc - img, iframe + marginheight - iframe + marginwidth - iframe + maxlength - input + method - form + model* - embed + multiple - select + name - button, embed, textarea, applet^, select, form^, iframe^, img^, a^, input, object, map^, param + nohref - area + noshade - hr^ + nowrap - td^, th^ + object - applet + onblur - a, area, button, input, label, select, textarea + onchange - input, select, textarea + onfocus - a, area, button, input, label, select, textarea + onreset - form + onselect - input, textarea + onsubmit - form + pluginspage* - embed + pluginurl* - embed + prompt - isindex + readonly - textarea, input + rel - a + rev - a + rows - textarea + rowspan - td, th + rules - table + scope - td, th + scrolling - iframe + selected - option + shape - area, a + size - hr^, font, input, select + span - col, colgroup + src - embed, script, input, iframe, img + standby - object + start - ol^ + summary - table + tabindex - a, area, button, input, object, select, textarea + target - a^, area, form + type - a, embed, object, param, script, input, li^, ol^, ul^, button + usemap - img, input, object + valign - col, colgroup, tbody, td, tfoot, th, thead, tr + value - input, option, param, button, li^ + valuetype - param + vspace - applet, img^, object^ + width - embed, hr^, iframe, img, object, table, td^, th^, applet, col, colgroup, pre^ + wmode - embed + xml:space - pre, script, style + + These are allowed in all but the shown elements: + + class - param, script + dir - applet, bdo, br, iframe, param, script + id - script + lang - applet, br, iframe, param, script + onclick - applet, bdo, br, font, iframe, isindex, param, script + ondblclick - applet, bdo, br, font, iframe, isindex, param, script + onkeydown - applet, bdo, br, font, iframe, isindex, param, script + onkeypress - applet, bdo, br, font, iframe, isindex, param, script + onkeyup - applet, bdo, br, font, iframe, isindex, param, script + onmousedown - applet, bdo, br, font, iframe, isindex, param, script + onmousemove - applet, bdo, br, font, iframe, isindex, param, script + onmouseout - applet, bdo, br, font, iframe, isindex, param, script + onmouseover - applet, bdo, br, font, iframe, isindex, param, script + onmouseup - applet, bdo, br, font, iframe, isindex, param, script + style - param, script + title - param, script + xml:lang - applet, br, iframe, param, script + + +-- 5.3 CSS 2.1 properties accepting URLs ------------------------o + + + background + background-image + content + cue-after + cue-before + cursor + list-style + list-style-image + play-during + + +-- 5.4 Microsoft Windows 1252 character replacements --------------o + + + Key: 'd' double, 'l' left, 'q' quote, 'r' right, 's.' single + + Code-point (decimal) - hexadecimal value - replacement entity - represented character + + 127 - 7f - (removed) - (not used) + 128 - 80 - € - euro + 129 - 81 - (removed) - (not used) + 130 - 82 - ‚ - baseline s. q + 131 - 83 - ƒ - florin + 132 - 84 - „ - baseline d q + 133 - 85 - … - ellipsis + 134 - 86 - † - dagger + 135 - 87 - ‡ - d dagger + 136 - 88 - ˆ - circumflex accent + 137 - 89 - ‰ - permile + 138 - 8a - Š - S Hacek + 139 - 8b - ‹ - l s. guillemet + 140 - 8c - Œ - OE ligature + 141 - 8d - (removed) - (not used) + 142 - 8e - Ž - Z dieresis + 143 - 8f - (removed) - (not used) + 144 - 90 - (removed) - (not used) + 145 - 91 - ‘ - l s. q + 146 - 92 - ’ - r s. q + 147 - 93 - “ - l d q + 148 - 94 - ” - r d q + 149 - 95 - • - bullet + 150 - 96 - – - en dash + 151 - 97 - — - em dash + 152 - 98 - ˜ - tilde accent + 153 - 99 - ™ - trademark + 154 - 9a - š - s Hacek + 155 - 9b - › - r s. guillemet + 156 - 9c - œ - oe ligature + 157 - 9d - (removed) - (not used) + 158 - 9e - ž - z dieresis + 159 - 9f - Ÿ - Y dieresis + + +-- 5.5 URL format -------------------------------------------------o + + + An `absolute` URL has a 'protocol' or 'scheme', a 'network location' or 'hostname', and, optional 'path', 'parameters', 'query' and 'fragment' segments. Thus, an absolute URL has this generic structure: + + (scheme) : (//network location) /(path) ;(parameters) ?(query) #(fragment) + + The schemes can only contain letters, digits, '+', '.' and '-'. Hostname is the portion after the '//' and up to the first '/' (if any; else, up to the end) when ':' is followed by a '//' (e.g., 'abc.com' in 'ftp://abc.com/def'); otherwise, it consists of everything after the ':' (e.g., 'def@abc.com' in mailto:def@abc.com'). + + `Relative` URLs do not have explicit schemes and network locations; such values are inherited from a `base` URL. + + +-- 5.6 Brief on htmLawed code -------------------------------------o + + + Much of the code's logic and reasoning can be understood from the documentation above. + + The *output* of htmLawed is a text string containing the processed input. There is no custom error tracking. + + *Function arguments* for htmLawed are: + + * '$in' - 1st argument; a text string; the *input text* to be processed. Any extraneous slashes added by PHP when `magic quotes` are enabled should be removed beforehand using PHP's 'stripslashes()' function. + + * '$config' - 2nd argument; an associative array; optional (named '$C' in htmLawed code). The array has keys with names like 'balance' and 'keep_bad', and the values, which can be boolean, string, or array, depending on the key, are read to accordingly set the *configurable parameters* (indicated by the keys). All configurable parameters receive some default value if the value to be used is not specified by the user through '$config'. `Finalized` '$config' is thus a filtered and possibly larger array. + + * '$spec' - 3rd argument; a text string; optional. The string has rules, written in an htmLawed-designated format, *specifying* element-specific attribute and attribute value restrictions. Function 'hl_spec()' is used to convert the string to an associative-array for internal use. `Finalized` '$spec' is thus an array. + + `Finalized` '$config' and '$spec' are made *global variables* while htmLawed is at work. Values of any pre-existing global variables with same names are noted, and their values are restored after htmLawed finishes processing the input (to capture the `finalized` values, the 'show_settings' parameter of '$config' should be used). Depending on '$config', another global variable 'hl_Ids', to track 'id' attribute values for uniqueness, may be set. Unlike the other two variables, this one is not reset (or unset) post-processing. + + Except for the main function 'htmLawed()' and the functions 'kses()' and 'kses_hook()', htmLawed's functions are *name-spaced* using the 'hl_' prefix. The *functions* and their roles are: + + * 'hl_attrval' - checking attribute values against $spec + * 'hl_bal' - tag balancing + * 'hl_cmtcd' - handling CDATA sections and HTML comments + * 'hl_ent' - entity handling + * 'hl_prot' - checking a URL scheme/protocol + * 'hl_regex' - checking syntax of a regular expression + * 'hl_spec' - converting user-supplied $spec value to one used by htmLawed internally + * 'hl_tag' - handling tags + * 'hl_tag2' - transforming tags + * 'hl_tidy' - compact/beautify HTML + * 'hl_version' - reporting htmLawed version + * 'htmLawed' - main function + * 'kses' - main function of 'kses' + * 'kses_hook' - hook function of 'kses' + + The last two are for compatibility with pre-existing code using the 'kses' script. htmLawed's 'kses()' basically passes on the filtering task to 'htmLawed()' function after deciphering '$config' and '$spec' from the argument values supplied to it. 'kses_hook()' is an empty function and is meant for being filled with custom code if the 'kses' script users were using one. + + 'htmLawed()' finalizes '$spec' (with the help of 'hl_spec()') and '$config', and globalizes them. Finalization of '$config' involves setting default values if an inappropriate or invalid one is supplied. This includes calling 'hl_regex()' to check well-formedness of regular expression patterns if such expressions are user-supplied through '$config'. 'htmLawed()' then removes invalid characters like nulls and 'x01' and appropriately handles entities using 'hl_ent()'. HTML comments and CDATA sections are identified and treated as per '$config' with the help of 'hl_cmtcd()'. When retained, the '<' and '>' characters identifying them, and the '<', '>' and '&' characters inside them, are replaced with control characters (code-points '1' to '5') till any tag balancing is completed. + + After this `initial processing` 'htmLawed()' identifies tags using regex and processes them with the help of 'hl_tag()' -- a large function that analyzes tag content, filtering it as per HTML standards, '$config' and '$spec'. Among other things, 'hl_tag()' transforms deprecated elements using 'hl_tag2()', removes attributes from closing tags, checks attribute values as per '$spec' rules using 'hl_attrval()', and checks URL protocols using 'hl_prot()'. 'htmLawed()' performs tag balancing and nesting checks with a call to 'hl_bal()', and optionally compacts/beautifies the output with proper white-spacing with a call to 'hl_tidy()'. The latter temporarily replaces white-space, and '<', '>' and '&' characters inside 'pre', 'script' and 'textarea' elements, and HTML comments and CDATA sections with control characters (code-points '1' to '5', and '7'). + + htmLawed permits the use of custom code or *hook functions* at two stages. The first, called inside 'htmLawed()', allows the input text as well as the finalized $config and $spec values to be altered right after the initial processing (see section:- #3.7). The second is called by 'hl_tag()' once the tag content is finalized (see section:- #3.4.9). + + Being dictated by the external and stable HTML standard, htmLawed's objective is very clear-cut and less concerned with tweakability. The code is only minimally annotated with comments -- it is not meant to instruct; PHP developers familiar with the HTML specs will see the logic, and others can always refer to the htmLawed documentation. The compact structuring of the statements is meant to aid in quickly grasping the logic, at least when viewed with code syntax highlighted. + +___________________________________________________________________oo + + +@@description: htmLawed PHP software is a free, open-source, customizable HTML input purifier and filter +@@encoding: utf-8 +@@keywords: htmLawed, HTM, HTML, HTML Tidy, converter, filter, formatter, purifier, sanitizer, XSS, input, PHP, software, code, script, security, cross-site scripting, hack, sanitize, remove, standards, tags, attributes, elements +@@language: en +@@title: htmLawed documentation
\ No newline at end of file diff --git a/extlib/htmLawed/htmLawed_TESTCASE.txt b/extlib/htmLawed/htmLawed_TESTCASE.txt new file mode 100644 index 000000000..366465ce3 --- /dev/null +++ b/extlib/htmLawed/htmLawed_TESTCASE.txt @@ -0,0 +1,370 @@ +/* +htmLawed_TESTCASE.txt, 23 April 2009 +htmLawed 1.1.8.1, 16 July 2009 +Copyright Santosh Patnaik +GPL v3 license +A PHP Labware internal utility - http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed +*/ + +This file has UTF-8-encoded text with both correct and incorrect/malformed HTML/XHTML code snippets to test htmLawed (test cases/samples). The entire text may also be used as a unit. + +************************************************ +when viewing this file in a web browser, set the +character encoding to Unicode/UTF-8 +************************************************ + +--------------------- start -------------------- + +<em>Try different $config and $spec values. Some text even when filtered in will not be displayed in a rendered web-page</em><br /> + +<h6>Attributes</h6> + +<strong>Xml:lang:</strong><a lang="en" xml:lang="en"></a>, <a lang="en"></a>, <a xml:lang="en"></a><br /> +<strong>Standard, predefined value, or empty attribute:</strong> <input type="text" disabled />, <input type="text" disabled="DISABLED" />, <input type="text" disabled="1" /><br /> +<strong>Required:</strong> <img />, <img alt="image" /><br /> +<strong>Quote & space variation:</strong> <a id=id1 name=xy>a</a>, <a id='id2' name="xy">a</a>, <a id=' id3 ' name = "n" >a</a><br /> +<strong>Invalid:</strong> <a id="id4" src="s">a</a><br /> +<strong>Duplicated:</strong> <a id="id5" id="id6">a</a><br /> +<strong>Deprecated:</strong> <a id="id7" target="self" name="n">a</a>, <hr noshade="noshade" /><br /> +<strong>Casing:</strong> <a HREF=""></a><br /> +<strong>Admin-restricted?:</strong> <a href="x" onclick="alert();"></a> + +<h6>Attribute values</h6> + +<strong>Duplicate ID value:</strong><a id="id8"></a>, <a id="my_id8"></a>, <a id="id8"></a><br /> +(try 'my_' for prefix)<br /> +<strong>Double-quotes in value:</strong><a title=ab"c"></a>, <a title="ab"c"></a>, <a title='ab"c'></a><br /> +(try filter for CSS expression)<br /> +<strong>CSS expression</strong>: <div style="prop:expression();"></div><div style="prop:expression()"></div><div style="prop: expression();"></div><div style="prop : expression()"></div><div style="prop:expression(js);"></div><div style="prop:expression(js;)"></div><div style="prop: expression('js');"></div><div style="prop : expr ession('js':)"></div><div style="prop:expression( 'js@ );"></div><br /> +<strong>Other:</strong> <input size="50" class="my" value="an input an input an input" />, <input size="5" class="your" value="an input" /><br /> +(try 'maxlen', 'maxval', etc., for 'input' in '$spec') + +<h6>Blockquotes</h6> + +<blockquote>abc</blockquote><br /> +<blockquote>abc<div>def</div></blockquote><br /> +<blockquote><div>abc</div>def</blockquote><br /> +<blockquote>abc<div>def</div>ghi</blockquote><br /> +abc<div>def</div>ghi<br /> +(try with blockquote parent) + +<h6>CDATA sections</h6> + +<strong>Special characters inside:</strong> <![CDATA[ ]]> ]]>, <![CDATA[ 3 < 4 > 3.5, & 4 > 4 ]]><br /> +<strong>Normal:</strong> <![CDATA[ check ]]>, <em>CDATA follows:<![CDATA[ check ]]></em><br /> +<strong>Malformed:</strong> <![cdata check ]]>, < ![CDATA check ]]>, <![CDATA check ]]>, < ![CDATA check ] ]><br /> +<strong>Invalid:</strong> <em <![CDATA[ check ]]>>CDATA in tag content</em>, <table><![CDATA[ check ]]><tr><td>text not allowed</td></tr></table> + +<h6>Complex-1: deprecated elements</h6> + +<center> +The PHP <s>software</s> script used for this <strike>web-page</strike> webpage is <font style="font-weight: bold " face=arial size='+3' color = "red ">htmLawedTest.php</font>, from <u style= 'color:green'>PHP Labware</u>. +</center> + +<h6>Complex-2: deprecated attributes</h6> + +<img src="s" alt="a" name="n" /><img src="s" alt="a" id="id9" name="n" /> +<br clear="left" /> +<hr noshade size="1" /> +<img name="id10" src="s" align="left" alt="image" hspace="10" vspace="10" width="10em" height="20" border="1" style="padding:5px;" /> +<table width="50em" align="center" bgcolor="red"> + <tr> + <td width="20%"> + <div align="center"> + <h3 align="right">Section</h3> + <p align="right">Para</p> + <ol type="a" start="e"><li value="x"><a name="x">First</a> <a name="x" id="id11">item</a></li></ol> + </div> + </td> + <td width="*"> + <ol type="1"><li>First item</li></ol> + </td> + </tr> + </table> +<br clear="all" /> + +<h6>Complex-3: embed, object, area</h6> + +<object width="425" height="350"><param name="movie" value="http://www.youtube.com/v/ls7gi1VwdIQ"></param><embed src="http://www.youtube.com/v/ls7gi1VwdIQ" type="application/x-shockwave-flash" width="425" height="350"></embed></object><br /> + +<embed src="http://www.youtube.com/v/ls7gi1VwdIQ" type="application/x-shockwave-flash" width="425" height="350"></embed><br /> + +<object data="1.gif" type="image/gif" usemap="#map1"><map name="map1"> +<p>navigate the site: <a href="1" shape="REct" coOrds="0,0,118,28">1</a> | <a href="3" shape="circle" coords="184,200,60">3</a> | <a href="4" shape="poly" coords="276,0,276,28,100,200,50,50,276,0">4</a></p> +<area href="5" shape="Rect" coords="0,0,118,28"> +</map></object> + +<h6>Complex-4: nested and other tables</h6> + +<table border="1" bgcolor="red"> <tr> <td> Cell </td> <td colspan="2" rowspan="2"> <table border="1" bgcolor="green"> <tr> <td> Cell </td> <td colspan="2" rowspan="2"> </td> </tr> <tr> <td> Cell </td> </tr> <tr> <td> Cell </td> <td> Cell </td> <td> Cell </td> </tr> </table> </td> </tr> <tr> <td> Cell </td> </tr> <tr> <td> Cell </td> <td> Cell </td> <td> Cell </td> </tr> </table><br /> +<strong>PCDATA wrong:</strong> <table>Well<caption>Hello</caption></table><br /> +<strong>Missing tr:</strong> <table><td>Well</td></table><br /> + +<h6>Complex-5: pseudo, disallowed or non-HTML tags</h6> + +(Try different 'keep_bad' values) +<*> Pseudotags <*> +<xml>Non-HTML tag xml</xml> +<p> +Disallowed tag p +</p> +<ul>Bad<li>OK</li></ul> + +<h6>Elements</h6> + +<strong>Unbalanced:</strong> <a href="h"><em>check</a></em><br /> +<strong>Non-XHTML:</strong> <div><center><dir></dir></center></div><br /> +<strong>Malformed:</strong> < a href=""></a>, <a href="" ></a>, <a href="" ></a>, <a href="" +></a>, <a href="">< /a>, < a href=""></a >, <img src="s" alt="a" />, <img src="s" alt="a"/ >, <imgsrc="s" alt="a" /><br /> +<strong>Invalid:</strong> <image src="s" alt="a" /><br /> +<strong>Empty:</strong> <img src="s" alt="a" />, <img src="s" alt="a"></img>, <img src="s" alt="a">text</img><br /> +<strong>Content invalid:</strong> <a href="h">1<a>2</a></a><br /> +<strong>Content invalid?:</strong> <form></form><br /> (try setting 'form' as parent) +<strong>Casing:</strong> <A href=""></a> + +<h6>Entities</h6> + +<strong>Special:</strong> & 3 < 2 & 5>4 and j >i >a & i<j>a<br /> +<strong>Padding:</strong> B B f f  <br /> +<strong>Malformed:</strong> & #x27;, &x27;, ' &TILDE;, &tilde<br /> +<strong>Invalid:</strong> , �, , �, , &bad;<br /> +<strong>Discouraged characters:</strong> , „, , <br /> +<strong>Context:</strong> '>', <?<br /> +<strong>Casing:</strong> ', ', &TILDE;, ˜ +<br /> +(also check named-to-numeric and hexdec-to-decimal, and vice versa, conversions) + +<h6>Format</h6> + +<strong>Valid but ill-formatted:</strong> text <!-- comment --> +text <!-- +A c o m m e n t --> +<script> + <![CDATA[ + code + ]]> +</script><!-- comment --><![CDATA[ cdata ]]> <a>text</b> text<pre id="none">p r e</pre> +<textarea>text</textarea> <textarea> + text text +</textarea> text text <br /><hr /> +text <img src="none" alt="none" /> t<em class="none">e<strong>x</strong>t</em> +text <img src="none" alt="none" /> <b>t<em> e <strong> x </strong> t</em></b> + <a href="a"> text <img src="none" alt="none" /> <b>t <em> e <strong> x </strong> t</em></b> + </a> +<span style="background-color: yellow;">text <img src="none" alt="none" /> <b> <em> t e <strong> x </strong> t</em></b></span> +<script>script</script> +<div> + <pre id="none">p <a>r</a> e <!-- comment --> </pre> + <pre> + pre + </pre> +</div> +<div><div><table border="1" style="background-color: red;"><tr><td>Cell</td><td colspan="2" rowspan="2"><table border="1" style="background-color: green;"><tr><td>Cell</td><td colspan="2" rowspan="2"></td></tr><tr><td>Cell</td></tr><tr><td>Cell</td><td>Cell</td><td>Cell</td></tr></table></td></tr><tr><td>Cell</td></tr><tr><td>Cell</td><td>Cell</td><td>Cell</td></tr></table></div></div> +(try to compact or beautify) + +<h6>Forms</h6> + +(note nesting of 'form', missing required attributes, etc.)<br /> +<form> +<script type="text/javascript">s</script> +<fieldset><legend>p</legend>l <input name="personal_lastname" type="text" tabindex="1"></fieldset> +<input name="h" type="checkbox" value="h" tabindex="20"> h +<textarea name="t">t</textarea> +<form action="a" method="get"></form></form><br /> +<form action="b" method="get"><p><input type="text" value="i" /></form><br /> +<form>B:<input type="text" value="b" />C:<input type="text" value="c" /></form><br /> +(try each of these lines separately)<br /> +<form action="a">what<br /> +<form action="a">what +(try with container as div and as form)<br /> +<form>c <a>a</a> <b>b</b><input /><script>s</script> + +<h6>HTML comments (also CDATA)</h6> + +Special characters inside: <!-- <![CDATA check ]]> -->, <!-- 3 < 4 > 3.5, & 4 > 4 -->, <!-- che--ck -->, <!--[if !IE]> <--><a>c</a><!--> <![endif]--><br /> +Normal: <!-- check -->, <!--check -->, <em>comment:<!-- check --></em><!-- check -->, <table><!-- check --><tr><td>text not allowed</td></tr></table><br /> +Malformed: <![cdata check ]]>, < ![CDATA check ]]>, < ![CDATA check ] ]><br /> +Invalid: <em <!-- check -->>comment in tag content</em>, <!--check--> + +<h6>Ins-Del</h6> + +(depending on context, these elements can be of either block or inline type)<br /> +<p><ins datetime="d" cite="c"><div>block</div></ins></p><br /> +<p><del>d</del></p><br /> +<p><ins><del>d</del></ins></p><div><ins><p><del><div>d</div></del></p></ins></div><ins><div>d</div></ins> + +<h6>Lists</h6> + +<strong>Invalid character data</strong>: <ul><li>(item</li>)</ul><br /> +<strong>Definition list</strong>: <dl><dt>a</dt>bad<dd>first <em>one</em></dd><dt>b</dt><dd>second</dd></dl><br /> +<strong>Definition list, close-tags omitted</strong>: <dl><dt>a</dt>bad<dd>first <em>one</em></dd><dt>b<dd>second</dl><br /> +<strong>Definition lists, nested</strong>: <dl> + <dt>T1</dt> + <dd>D1</dd> + <dt>T2</dt> + <dd>D2<dl><dt>t1</dt><dd>d1</dd><dt>t2</dt><dd>d2</dd></dl></dd> + <dt>T3</dt> + <dd>D3</dd> + <dt>T4</dt> + <dd>D4<dl><dt>t1</dt><dd>d1</dd></dl></dd> +</dl><br /> +<strong>Definition lists, nested, close-tags omitted</strong>: <dl> + <dt>T1 + <dd>D1</dd> + <dt>T2</dt> + <dd>D2<dl><dt>t1<dd>d1<dt>t2</dt><dd>d2</dd></dl></dd> + <dt>T3 + <dd>D3 + <dt>T4 + <dd>D4<dl><dt>t1<dd>d1</dl></dd> +</dl><br /> +<strong>Nested</strong>: <ul> + <li>l1</li> + <li>l2<ol><li>lo1</li><li>lo2</li></ol></li> + <li>l3</li> + <li>l4<ol><li>lo3</li><li>lo4<ol><li>lo5</li></ol></li></ol></li> +</ul><br /> +<strong>Nested, close-tags omitted</strong>: <ul> + <li>l1</li> + <li>l2<ol><li>lo1<li>lo2</ol> + <li>l3 + <li>l4<ol><li>lo3<li>lo4<ol><li>lo5</ol></ol> +</ul><br /> +<strong>Complex</strong>: +<ol><script></script><li><table><tr><td> +<ul><li id="search" class="widget widget_search"> <form id="searchform" method="get" action="http://kohei.us"> + <div> + + <input type="text" name="s" id="s" size="15" /><br /> + <input type="submit" value="Search" /> + </div> + </form> + </li></ul> +</td></tr></table></li></ol> + +<h6>Non-English text-1</h6> + +Inscrieţi-vă acum la a Zecea Conferinţă Internaţională<br /> +გთხოვთ ახლავე გაიაროთ რეგისტრაცია<br /> +večjezično računalništvo<br /> +<a title="הירשמו +כעת לכנס ">Зарегистрируйтесь сейчас +на Десятую Международную Конференцию по</a><br /> +(this file should have utf-8 encoding; some characters may not be displayed because of missing fonts, etc.) + +<h6>Non-English text-2: entities</h6> + +用统一码<br /> +გთხოვთ<br /> +Inscreva-se agora para a Décima Conferência Internacional Sobre O Unicode, realizada entre os dias 10 e 12 de março de 1997 em Mainz +na Alemanha. + +<h6>Ruby</h6> + +(need compatible browser)<br /> +<ruby xml:lang="ja"> + <rbc> + <rb>斎</rb> + <rb>藤</rb> + <rb>信</rb> + <rb>男</rb> + </rbc> + <rtc class="reading"> + <rt>さい</rt> + <rt>とう</rt> + <rt>のぶ</rt> + <rt>お</rt> + </rtc> + <rtc class="annotation"> + <rt rbspan="4" xml:lang="en">W3C Associate Chairman</rt> + </rtc> +</ruby><br /> +<ruby> + <rb>WWW</rb> + <rp>(</rp><rt>World Wide Web</rt><rp>)</rp> +</ruby><br /> +<ruby> + A + <rp>(</rp><rt>aaa</rt><rp>)</rp> +</ruby> + +<h6>Tables</h6> + +<strong>Omitted closing tags:</strong> <table> +<colgroup><col style="x" /><col style="y" /> +<thead> +<tr><th>h1c1<th>h1c2 +<tbody> +<tr><td>r1c1<td>r1c2 +<tr><td>r2c1<td>r2c2 +</table><br /> +<strong>Nested, omitted closing tags:</strong> <table> +<colgroup><col style="x" /><col style="y" /> +<thead> +<tr><th>h1c1<th>h1c2 +<tbody> +<tr><td>r1c1<td>r1c2<table> +<colgroup><col style="x" /><col style="y" /> +<thead> +<tr><th>h1c1<th>h1c2 +<tbody> +<tr><td>r1c1<td>r1c2 +<tr><td>r2c1<td>r2c2 +</table> +<tr><td>r2c1<td>r2c2 +</table><br /> + +<h6>URLs</h6> + +<strong>Relative and absolute:</strong> <a href="mailto:x"></a>, <a href="http://a.com/b/c/d.f"></a>, <a href="./../d.f"></a>, <a href="./d.f"></a>, <a href="d.f"></a>, <a href="#s"></a>, <a href="./../../d.f#s"></a><br /> +(try base URL value of 'http://a.com/b/')<br /> +<strong>CSS URLs:</strong> <div style="background-image: url('a.gif');"></div>, <div style="background-image: URL("a.gif");"></div>, <div style="background-image: url('http://a.com/a.gif');"></div>, <div style="background-image: url('./../a.gif');"></div>, <div style="background-image: url('js:xss')"></div><br /> +<strong>Anti-spam:</strong> (try regex for 'http://a.com', etc.) <a href="mailto:x@y.com"></a>, <a href="http://a.com/b@d.f"></a>, <a href="a.com/d.f" rel="nofollow"></a>, <a href="a.com/d.f" rel="1, 2"></a>, <a href="a.com/d.f"></a>, <a href="b.com/d.f"></a>, <a href="c.com/d.f"></a><br /> + +<h6>XSS</h6> + +'';!--"<xss>=&{()}<br /> +<img src="javascript%3Aalert('xss');" /><br /> +<img src="javascript:alert('xss');" /><br /> +<img src="java script:alert('xss');" /><br /> +<img +src=javascript:alert('XSS') /><br /> +<div style="javascript:alert('xss');"></div><br /> +<div style="background-image:url(javascript:alert('xss'));"></div><br /> +<div style="background-image:url("javascript:alert('xss')" );"></div><br /> +<!--[if gte IE 4]><script>alert('xss');</script><![endif]--><br /> +<script a=">" src="http://ha.ckers.org/xss.js"></script><br /> +<div style="background-image: url('js:xss')"></div><br /> +<a style=";-moz-binding:url(http://lukasz.pilorz.net/xss/xss.xml#xss)" href="http://example.com">test</a><br /> +<strong>Bad IE7:</strong> <a href="http://x&x=%22+style%3d%22background-image%3a+expression%28alert +%28%27xss%3f%29%29">x</a><br /> +<strong>Bad IE7:</strong> <a style=color:expr/*comment*/ession(alert(document.domain))>xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background: expression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background: expression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background: %45xpression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background:/**/expression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background:/**/Expression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background:/**/Expression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background: expr%45ssion(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background: exp/* */ression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background: exp /* */ression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background: exp/ * * /ression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background:/* x */expression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="xxx" style="background:/* */ */expression(alert('xss'));">xxx</a><br /> +<strong>Bad IE7:</strong> <a href="x" style="width: /****/**;;;;;;*/expression/**/(alert('xss'));">x</a><br /> +<strong>Bad IE7:</strong> <a href="x" style="padding:10px; background:/**/expression(alert('xss'));">x</a><br /> +<strong>Bad IE7:</strong> <a href="x" style="background: huh /* */ */expression(alert('xss'));">x</a><br /> +<strong>Bad IE7:</strong> <a href="x" style="background:/**/expression(alert('xss'));background:/**/expression(alert('xss'));">x</a><br /> +<strong>Bad IE7:</strong> exp/*<a style='no\xss:noxss("*//*");xss:ex/*XSS*//*/*/pression(alert("XSS"))'>x</a><br /> +<strong>Bad IE7:</strong> <a style="background:Expre\ssion(alert('xss'));">hi</a><br /> +<strong>Bad IE7:</strong> <a style="background:expre\ssion(alert('xss'));">hi</a><br /> +<strong>Bad IE7:</strong> <a style="color: \0065 \0078 \0070 \0072 \0065 \0073 \0073 \0069 \006f \006e \0028 \0061 \006c \0065 \0072 \0074 \0028 \0031 \0029 \0029">test</a><br /> +<strong>Bad IE7:</strong> <a style="xss:e\0078pression(window.x?0:(alert(/XSS/),window.x=1));">hi</a><br /> +<strong>Bad IE7:</strong> <a style="background:url('java +script:eval(document.all.mycode.expr)')">hi</a><br /> + +<h6>Other</h6> + +3 < 4 <br /> +3 > 4 <br /> + > 3 <br />
\ No newline at end of file diff --git a/htaccess.sample b/htaccess.sample index 634900dbf..942e98334 100644 --- a/htaccess.sample +++ b/htaccess.sample @@ -1,12 +1,14 @@ -RewriteEngine On +<IfModule mod_rewrite.c> + RewriteEngine On -# NOTE: change this to your actual Laconica path; may be "/". + # NOTE: change this to your actual Laconica path; may be "/". -RewriteBase /mublog/ + RewriteBase /mublog/ -RewriteCond %{REQUEST_FILENAME} !-f -RewriteCond %{REQUEST_FILENAME} !-d -RewriteRule (.*) index.php?p=$1 [L,QSA] + RewriteCond %{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_FILENAME} !-d + RewriteRule (.*) index.php?p=$1 [L,QSA] +</IfModule> <FilesMatch "\.(ini)"> Order allow,deny @@ -107,8 +107,27 @@ function checkMirror($action_obj, $args) function main() { + // fake HTTP redirects using lighttpd's 404 redirects + if (strpos($_SERVER['SERVER_SOFTWARE'], 'lighttpd') !== false) { + $_lighty_url = $base_url.$_SERVER['REQUEST_URI']; + $_lighty_url = @parse_url($_lighty_url); + + if ($_lighty_url['path'] != '/index.php' && $_lighty_url['path'] != '/') { + $_lighty_path = preg_replace('/^'.preg_quote(common_config('site','path')).'\//', '', substr($_lighty_url['path'], 1)); + $_SERVER['QUERY_STRING'] = 'p='.$_lighty_path; + if ($_lighty_url['query']) + $_SERVER['QUERY_STRING'] .= '&'.$_lighty_url['query']; + parse_str($_lighty_url['query'], $_lighty_query); + foreach ($_lighty_query as $key => $val) { + $_GET[$key] = $_REQUEST[$key] = $val; + } + $_GET['p'] = $_REQUEST['p'] = $_lighty_path; + } + } + $_SERVER['REDIRECT_URL'] = preg_replace("/\?.+$/", "", $_SERVER['REQUEST_URI']); + // quick check for fancy URL auto-detection support in installer. - if (isset($_SERVER['REDIRECT_URL']) && ((dirname($_SERVER['REQUEST_URI']) . '/check-fancy') === $_SERVER['REDIRECT_URL'])) { + if (isset($_SERVER['REDIRECT_URL']) && (preg_replace("/^\/$/","",(dirname($_SERVER['REQUEST_URI']))) . '/check-fancy') === $_SERVER['REDIRECT_URL']) { die("Fancy URL support detection succeeded. We suggest you enable this to get fancy (pretty) URLs."); } global $user, $action; @@ -165,7 +184,8 @@ function main() if (!$user && common_config('site', 'private') && !in_array($action, array('login', 'openidlogin', 'finishopenidlogin', - 'recoverpassword', 'api', 'doc', 'register'))) { + 'recoverpassword', 'api', 'doc', 'register')) && + !preg_match('/rss$/', $action)) { common_redirect(common_local_url('login')); return; } diff --git a/install.php b/install.php index 570b08edf..227f99789 100644 --- a/install.php +++ b/install.php @@ -43,12 +43,12 @@ function checkPrereqs() $pass = false; } - if (version_compare(PHP_VERSION, '5.0.0', '<')) { - ?><p class="error">Require PHP version 5 or greater.</p><?php + if (version_compare(PHP_VERSION, '5.2.3', '<')) { + ?><p class="error">Require PHP version 5.2.3 or greater.</p><?php $pass = false; } - $reqs = array('gd', 'mysql', 'curl', + $reqs = array('gd', 'curl', 'xmlwriter', 'mbstring', 'gettext'); @@ -58,6 +58,10 @@ function checkPrereqs() $pass = false; } } + if (!checkExtension('pgsql') && !checkExtension('mysql')) { + ?><p class="error">Cannot find mysql or pgsql extension. You need one or the other: <code><?php echo $req; ?></code></p><?php + $pass = false; + } if (!is_writable(INSTALLDIR)) { ?><p class="error">Cannot write config file to: <code><?php echo INSTALLDIR; ?></code></p> @@ -66,17 +70,16 @@ function checkPrereqs() $pass = false; } - if (!is_writable(INSTALLDIR.'/avatar/')) { - ?><p class="error">Cannot write avatar directory: <code><?php echo INSTALLDIR; ?>/avatar/</code></p> - <p>On your server, try this command: <code>chmod a+w <?php echo INSTALLDIR; ?>/avatar/</code></p> - <? - $pass = false; - } - if (!is_writable(INSTALLDIR.'/background/')) { - ?><p class="error">Cannot write background directory: <code><?php echo INSTALLDIR; ?>/background/</code></p> - <p>On your server, try this command: <code>chmod a+w <?php echo INSTALLDIR; ?>/background/</code></p> - <? - $pass = false; + // Check the subdirs used for file uploads + $fileSubdirs = array('avatar', 'background', 'file'); + foreach ($fileSubdirs as $fileSubdir) { + $fileFullPath = INSTALLDIR."/$fileSubdir/"; + if (!is_writable($fileFullPath)) { + ?><p class="error">Cannot write <?php echo $fileSubdir; ?> directory: <code><?php echo $fileFullPath; ?></code></p> + <p>On your server, try this command: <code>chmod a+w <?php echo $fileFullPath; ?></code></p> + <?php + $pass = false; + } } return $pass; @@ -127,7 +130,15 @@ function showForm() <p class="form_guide">Database hostname</p> </li> <li> - <label for="host">Database</label> + + <label for="dbtype">Type</label> + <input type="radio" name="dbtype" id="fancy-mysql" value="mysql" checked='checked' /> MySQL<br /> + <input type="radio" name="dbtype" id="dbtype-pgsql" value="pgsql" /> PostgreSQL<br /> + <p class="form_guide">Database type</p> + </li> + + <li> + <label for="database">Name</label> <input type="text" id="database" name="database" /> <p class="form_guide">Database name</p> </li> @@ -139,7 +150,7 @@ function showForm() <li> <label for="password">Password</label> <input type="password" id="password" name="password" /> - <p class="form_guide">Database password</p> + <p class="form_guide">Database password (optional)</p> </li> </ul> <input type="submit" name="submit" class="submit" value="Submit" /> @@ -152,7 +163,7 @@ E_O_T; function updateStatus($status, $error=false) { ?> - <li <?php echo ($error) ? 'class="error"': ''; ?>><?print $status;?></li> + <li <?php echo ($error) ? 'class="error"': ''; ?>><?php echo $status;?></li> <?php } @@ -163,6 +174,7 @@ function handlePost() <?php $host = $_POST['host']; + $dbtype = $_POST['dbtype']; $database = $_POST['database']; $username = $_POST['username']; $password = $_POST['password']; @@ -191,64 +203,28 @@ function handlePost() $fail = true; } - if (empty($password)) { - updateStatus("No password specified.", true); - $fail = true; - } +// if (empty($password)) { +// updateStatus("No password specified.", true); +// $fail = true; +// } if (empty($sitename)) { updateStatus("No sitename specified.", true); $fail = true; } - if($fail){ - showForm(); - return; - } - - updateStatus("Starting installation..."); - updateStatus("Checking database..."); - $conn = mysql_connect($host, $username, $password); - if (!$conn) { - updateStatus("Can't connect to server '$host' as '$username'.", true); - showForm(); - return; - } - updateStatus("Changing to database..."); - $res = mysql_select_db($database, $conn); - if (!$res) { - updateStatus("Can't change to database.", true); - showForm(); - return; - } - updateStatus("Running database script..."); - $res = runDbScript(INSTALLDIR.'/db/laconica.sql', $conn); - if ($res === false) { - updateStatus("Can't run database script.", true); - showForm(); - return; - } - foreach (array('sms_carrier' => 'SMS carrier', - 'notice_source' => 'notice source', - 'foreign_services' => 'foreign service') - as $scr => $name) { - updateStatus(sprintf("Adding %s data to database...", $name)); - $res = runDbScript(INSTALLDIR.'/db/'.$scr.'.sql', $conn); - if ($res === false) { - updateStatus(sprintf("Can't run %d script.", $name), true); + if($fail){ showForm(); - return; - } - } - updateStatus("Writing config file..."); - $sqlUrl = "mysqli://$username:$password@$host/$database"; - $res = writeConf($sitename, $sqlUrl, $fancy); - if (!$res) { - updateStatus("Can't write config file.", true); - showForm(); return; } - updateStatus("Done!"); + + switch($dbtype) { + case 'mysql': mysql_db_installer($host, $database, $username, $password, $sitename, $fancy); + break; + case 'pgsql': pgsql_db_installer($host, $database, $username, $password, $sitename, $fancy); + break; + default: + } if ($path) $path .= '/'; updateStatus("You can visit your <a href='/$path'>new Laconica site</a>."); ?> @@ -256,7 +232,120 @@ function handlePost() <?php } -function writeConf($sitename, $sqlUrl, $fancy) +function pgsql_db_installer($host, $database, $username, $password, $sitename, $fancy) { + $connstring = "dbname=$database host=$host user=$username"; + + //No password would mean trust authentication used. + if (!empty($password)) { + $connstring .= " password=$password"; + } + updateStatus("Starting installation..."); + updateStatus("Checking database..."); + $conn = pg_connect($connstring); + + if ($conn ===false) { + updateStatus("Failed to connect to database: $connstring"); + showForm(); + return false; + } + + //ensure database encoding is UTF8 + $record = pg_fetch_object(pg_query($conn, 'SHOW server_encoding')); + if ($record->server_encoding != 'UTF8') { + updateStatus("Laconica requires UTF8 character encoding. Your database is ". htmlentities($record->server_encoding)); + showForm(); + return false; + } + + updateStatus("Running database script..."); + //wrap in transaction; + pg_query($conn, 'BEGIN'); + $res = runDbScript(INSTALLDIR.'/db/laconica_pg.sql', $conn, 'pgsql'); + + if ($res === false) { + updateStatus("Can't run database script.", true); + showForm(); + return; + } + foreach (array('sms_carrier' => 'SMS carrier', + 'notice_source' => 'notice source', + 'foreign_services' => 'foreign service') + as $scr => $name) { + updateStatus(sprintf("Adding %s data to database...", $name)); + $res = runDbScript(INSTALLDIR.'/db/'.$scr.'.sql', $conn, 'pgsql'); + if ($res === false) { + updateStatus(sprintf("Can't run %d script.", $name), true); + showForm(); + return; + } + } + pg_query($conn, 'COMMIT'); + + updateStatus("Writing config file..."); + if (empty($password)) { + $sqlUrl = "pgsql://$username@$host/$database"; + } + else { + $sqlUrl = "pgsql://$username:$password@$host/$database"; + } + $res = writeConf($sitename, $sqlUrl, $fancy, 'pgsql'); + if (!$res) { + updateStatus("Can't write config file.", true); + showForm(); + return; + } + updateStatus("Done!"); + +} + +function mysql_db_installer($host, $database, $username, $password, $sitename, $fancy) { + updateStatus("Starting installation..."); + updateStatus("Checking database..."); + + $conn = mysql_connect($host, $username, $password); + if (!$conn) { + updateStatus("Can't connect to server '$host' as '$username'.", true); + showForm(); + return; + } + updateStatus("Changing to database..."); + $res = mysql_select_db($database, $conn); + if (!$res) { + updateStatus("Can't change to database.", true); + showForm(); + return; + } + updateStatus("Running database script..."); + $res = runDbScript(INSTALLDIR.'/db/laconica.sql', $conn); + if ($res === false) { + updateStatus("Can't run database script.", true); + showForm(); + return; + } + foreach (array('sms_carrier' => 'SMS carrier', + 'notice_source' => 'notice source', + 'foreign_services' => 'foreign service') + as $scr => $name) { + updateStatus(sprintf("Adding %s data to database...", $name)); + $res = runDbScript(INSTALLDIR.'/db/'.$scr.'.sql', $conn); + if ($res === false) { + updateStatus(sprintf("Can't run %d script.", $name), true); + showForm(); + return; + } + } + + updateStatus("Writing config file..."); + $sqlUrl = "mysqli://$username:$password@$host/$database"; + $res = writeConf($sitename, $sqlUrl, $fancy); + if (!$res) { + updateStatus("Can't write config file.", true); + showForm(); + return; + } + updateStatus("Done!"); + } +function writeConf($sitename, $sqlUrl, $fancy, $type='mysql') { $res = file_put_contents(INSTALLDIR.'/config.php', "<?php\n". @@ -264,11 +353,13 @@ function writeConf($sitename, $sqlUrl, $fancy) "\$config['site']['name'] = \"$sitename\";\n\n". ($fancy ? "\$config['site']['fancy'] = true;\n\n":''). "\$config['db']['database'] = \"$sqlUrl\";\n\n". + ($type == 'pgsql' ? "\$config['db']['quote_identifiers'] = true;\n\n" . + "\$config['db']['type'] = \"$type\";\n\n" : ''). "?>"); return $res; } -function runDbScript($filename, $conn) +function runDbScript($filename, $conn, $type='mysql') { $sql = trim(file_get_contents($filename)); $stmts = explode(';', $sql); @@ -277,8 +368,13 @@ function runDbScript($filename, $conn) if (!mb_strlen($stmt)) { continue; } - $res = mysql_query($stmt, $conn); + if ($type == 'mysql') { + $res = mysql_query($stmt, $conn); + } elseif ($type=='pgsql') { + $res = pg_query($conn, $stmt); + } if ($res === false) { + updateStatus("FAILED SQL: $stmt"); return $res; } } diff --git a/js/userdesign.go.js b/js/userdesign.go.js index 70dd9c7de..c53569bea 100644 --- a/js/userdesign.go.js +++ b/js/userdesign.go.js @@ -27,13 +27,14 @@ $(document).ready(function() { } } + /* rgb2hex written by R0bb13 <robertorebollo@gmail.com> */ function rgb2hex(rgb) { rgb = rgb.match(/^rgb\((\d+),\s*(\d+),\s*(\d+)\)$/); - function hex(x) { - hexDigits = new Array("0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F"); - return isNaN(x) ? "00" : hexDigits[(x - x % 16) / 16] + hexDigits[x % 16]; - } - return "#" + hex(rgb[1]) + hex(rgb[2]) + hex(rgb[3]); + return '#' + dec2hex(rgb[1]) + dec2hex(rgb[2]) + dec2hex(rgb[3]); + } + function dec2hex(x) { + hexDigits = new Array('0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'); + return isNaN(x) ? '00' : hexDigits[(x - x % 16) / 16] + hexDigits[x % 16]; } function UpdateColors(S) { diff --git a/js/util.js b/js/util.js index f3ed918cf..ef147bef4 100644 --- a/js/util.js +++ b/js/util.js @@ -25,7 +25,7 @@ $(document).ready(function(){ var counter = $("#notice_text-count"); counter.text(remaining); - if (remaining <= 0) { + if (remaining < 0) { $("#form_notice").addClass("warning"); } else { $("#form_notice").removeClass("warning"); @@ -244,7 +244,7 @@ function NoticeReply() { $('#content .notice').each(function() { var notice = $(this)[0]; $($('.notice_reply', notice)[0]).click(function() { - var nickname = ($('.author .nickname', notice).length > 0) ? $($('.author .nickname', notice)[0]) : $('.author .nickname'); + var nickname = ($('.author .nickname', notice).length > 0) ? $($('.author .nickname', notice)[0]) : $('.author .nickname.uid'); NoticeReplySet(nickname.text(), $($('.notice_id', notice)[0]).text()); return false; }); @@ -256,10 +256,15 @@ function NoticeReplySet(nick,id) { rgx_username = /^[0-9a-zA-Z\-_.]*$/; if (nick.match(rgx_username)) { replyto = "@" + nick + " "; - if ($("#notice_data-text").length) { - $("#notice_data-text").val(replyto); + var text = $("#notice_data-text"); + if (text.length) { + text.val(replyto + text.val()); $("#form_notice input#notice_in-reply-to").val(id); - $("#notice_data-text").focus(); + if (text.get(0).setSelectionRange) { + var len = text.val().length; + text.get(0).setSelectionRange(len,len); + text.get(0).focus(); + } return false; } } diff --git a/lib/attachmentlist.php b/lib/attachmentlist.php index f6a1b59d0..41d03f8e2 100644 --- a/lib/attachmentlist.php +++ b/lib/attachmentlist.php @@ -340,7 +340,12 @@ class Attachment extends AttachmentListItem case 'video': case 'link': if (!empty($this->oembed->html)) { - $this->out->raw($this->oembed->html); + require_once INSTALLDIR.'/extlib/htmLawed/htmLawed.php'; + $config = array( + 'safe'=>1, + 'elements'=>'*+object+embed'); + $this->out->raw(htmLawed($this->oembed->html,$config)); + //$this->out->raw($this->oembed->html); } break; diff --git a/lib/common.php b/lib/common.php index 507a2a281..8cd3ae2fc 100644 --- a/lib/common.php +++ b/lib/common.php @@ -19,7 +19,7 @@ if (!defined('LACONICA')) { exit(1); } -define('LACONICA_VERSION', '0.8.0'); +define('LACONICA_VERSION', '0.8.1pre1'); define('AVATAR_PROFILE_SIZE', 96); define('AVATAR_STREAM_SIZE', 48); diff --git a/lib/daemon.php b/lib/daemon.php index 9d89c63e7..231f5414e 100644 --- a/lib/daemon.php +++ b/lib/daemon.php @@ -24,6 +24,7 @@ if (!defined('LACONICA')) { class Daemon { var $daemonize = true; + var $_id = 'generic'; function __construct($daemonize = true) { @@ -35,6 +36,16 @@ class Daemon return null; } + function get_id() + { + return $this->_id; + } + + function set_id($id) + { + $this->_id = $id; + } + function background() { $pid = pcntl_fork(); diff --git a/lib/dbqueuemanager.php b/lib/dbqueuemanager.php index a37a8ffdf..19aa9f3ae 100644 --- a/lib/dbqueuemanager.php +++ b/lib/dbqueuemanager.php @@ -22,7 +22,6 @@ * @category QueueManager * @package Laconica * @author Evan Prodromou <evan@controlyourself.ca> - * @author Sarven Capadisli <csarven@controlyourself.ca> * @copyright 2009 Control Yourself, Inc. * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @link http://laconi.ca/ @@ -86,9 +85,15 @@ class DBQueueManager extends QueueManager $start = time(); $result = null; + $sleeptime = 1; + do { $qi = Queue_item::top($queue); - if (!empty($qi)) { + if (empty($qi)) { + $this->_log(LOG_DEBUG, "No new queue items, sleeping $sleeptime seconds."); + sleep($sleeptime); + $sleeptime *= 2; + } else { $notice = Notice::staticGet('id', $qi->notice_id); if (!empty($notice)) { $result = $notice; @@ -98,6 +103,7 @@ class DBQueueManager extends QueueManager $qi->free(); $qi = null; } + $sleeptime = 1; } } while (empty($result) && (is_null($timeout) || (time() - $start) < $timeout)); @@ -118,7 +124,7 @@ class DBQueueManager extends QueueManager } else { if (empty($qi->claimed)) { $this->_log(LOG_WARNING, 'Reluctantly releasing unclaimed queue item '. - 'for '.$notice->id.', queue '.$queue); + 'for '.$notice->id.', queue '.$queue); } $qi->delete(); $qi->free(); @@ -145,7 +151,7 @@ class DBQueueManager extends QueueManager } else { if (empty($qi->claimed)) { $this->_log(LOG_WARNING, 'Ignoring failure for unclaimed queue item '. - 'for '.$notice->id.', queue '.$queue); + 'for '.$notice->id.', queue '.$queue); } else { $orig = clone($qi); $qi->claimed = null; diff --git a/lib/groupsbymemberssection.php b/lib/groupsbymemberssection.php index 963e21f15..ad4884bf8 100644 --- a/lib/groupsbymemberssection.php +++ b/lib/groupsbymemberssection.php @@ -48,7 +48,7 @@ class GroupsByMembersSection extends GroupSection $qry = 'SELECT user_group.*, count(*) as value ' . 'FROM user_group JOIN group_member '. 'ON user_group.id = group_member.group_id ' . - 'GROUP BY user_group.id,user_group.nickname,user_group.fullname,user_group.homepage,user_group.description,user_group.location,user_group.original_logo,user_group.homepage_logo,user_group.stream_logo,user_group.mini_logo,user_group.created,user_group.modified ' . + 'GROUP BY user_group.id,user_group.nickname,user_group.fullname,user_group.homepage,user_group.description,user_group.location,user_group.original_logo,user_group.homepage_logo,user_group.stream_logo,user_group.mini_logo,user_group.created,user_group.modified,user_group.design_id ' . 'ORDER BY value DESC '; $limit = GROUPS_PER_SECTION; diff --git a/lib/groupsbypostssection.php b/lib/groupsbypostssection.php index 325b4033f..dc7925d5e 100644 --- a/lib/groupsbypostssection.php +++ b/lib/groupsbypostssection.php @@ -48,7 +48,7 @@ class GroupsByPostsSection extends GroupSection $qry = 'SELECT user_group.*, count(*) as value ' . 'FROM user_group JOIN group_inbox '. 'ON user_group.id = group_inbox.group_id ' . - 'GROUP BY user_group.id,user_group.nickname,user_group.fullname,user_group.homepage,user_group.description,user_group.location,user_group.original_logo,user_group.homepage_logo,user_group.stream_logo,user_group.mini_logo,user_group.created,user_group.modified ' . + 'GROUP BY user_group.id,user_group.nickname,user_group.fullname,user_group.homepage,user_group.description,user_group.location,user_group.original_logo,user_group.homepage_logo,user_group.stream_logo,user_group.mini_logo,user_group.created,user_group.modified,user_group.design_id ' . 'ORDER BY value DESC '; $limit = GROUPS_PER_SECTION; diff --git a/lib/grouptagcloudsection.php b/lib/grouptagcloudsection.php index 9b7a10f6b..0e0cbdd63 100644 --- a/lib/grouptagcloudsection.php +++ b/lib/grouptagcloudsection.php @@ -73,7 +73,7 @@ class GroupTagCloudSection extends TagCloudSection $quoted = array(); foreach ($names as $name) { - $quoted[] = "\"$name\""; + $quoted[] = "'$name'"; } $namestring = implode(',', $quoted); diff --git a/lib/language.php b/lib/language.php index 3ea3dd2aa..9ad2d31bd 100644 --- a/lib/language.php +++ b/lib/language.php @@ -53,7 +53,7 @@ function client_prefered_language($httplang) if (!empty($httplang[2][$i])) { // if no q default to 1.0 $client_langs[$httplang[2][$i]] = - ($httplang[6][$i]? (float) $httplang[6][$i] : 1.0); + ($httplang[6][$i]? (float) $httplang[6][$i] : 1.0 - ($i*0.01)); } if (!empty($httplang[3][$i]) && empty($client_langs[$httplang[3][$i]])) { // if a catchall default 0.01 lower diff --git a/lib/mail.php b/lib/mail.php index 781a7541b..0050ad810 100644 --- a/lib/mail.php +++ b/lib/mail.php @@ -121,7 +121,7 @@ function mail_notify_from() $domain = mail_domain(); - $notifyfrom = common_config('site', 'name') .' <noreply@'.$domain.'>'; + $notifyfrom = '"'.common_config('site', 'name') .'" <noreply@'.$domain.'>'; } return $notifyfrom; diff --git a/lib/messageform.php b/lib/messageform.php index b8878ec1f..8ea2b36c2 100644 --- a/lib/messageform.php +++ b/lib/messageform.php @@ -140,6 +140,12 @@ class MessageForm extends Form 'rows' => 4, 'name' => 'content'), ($this->content) ? $this->content : ''); + $this->out->elementStart('dl', 'form_note'); + $this->out->element('dt', null, _('Available characters')); + $this->out->element('dd', array('id' => 'notice_text-count'), + '140'); + $this->out->elementEnd('dl'); + } /** diff --git a/lib/noticelist.php b/lib/noticelist.php index 44726a17b..a8d5059ca 100644 --- a/lib/noticelist.php +++ b/lib/noticelist.php @@ -357,19 +357,14 @@ class NoticeListItem extends Widget preg_match('/^http/', $this->notice->uri)) { $noticeurl = $this->notice->uri; } - $this->out->elementStart('dl', 'timestamp'); - $this->out->element('dt', null, _('Published')); - $this->out->elementStart('dd', null); $this->out->elementStart('a', array('rel' => 'bookmark', + 'class' => 'timestamp', 'href' => $noticeurl)); $dt = common_date_iso8601($this->notice->created); $this->out->element('abbr', array('class' => 'published', 'title' => $dt), common_date_string($this->notice->created)); - $this->out->elementEnd('a'); - $this->out->elementEnd('dd'); - $this->out->elementEnd('dl'); } /** @@ -384,8 +379,8 @@ class NoticeListItem extends Widget function showNoticeSource() { if ($this->notice->source) { - $this->out->elementStart('dl', 'device'); - $this->out->element('dt', null, _('From')); + $this->out->elementStart('span', 'source'); + $this->out->text(_('from')); $source_name = _($this->notice->source); switch ($this->notice->source) { case 'web': @@ -394,22 +389,22 @@ class NoticeListItem extends Widget case 'omb': case 'system': case 'api': - $this->out->element('dd', null, $source_name); + $this->out->element('span', 'device', $source_name); break; default: $ns = Notice_source::staticGet($this->notice->source); if ($ns) { - $this->out->elementStart('dd', null); + $this->out->elementStart('span', 'device'); $this->out->element('a', array('href' => $ns->url, 'rel' => 'external'), $ns->name); - $this->out->elementEnd('dd'); + $this->out->elementEnd('span'); } else { - $this->out->element('dd', null, $source_name); + $this->out->element('span', 'device', $source_name); } break; } - $this->out->elementEnd('dl'); + $this->out->elementEnd('span'); } } @@ -429,13 +424,9 @@ class NoticeListItem extends Widget && $this->notice->conversation != $this->notice->id) { $convurl = common_local_url('conversation', array('id' => $this->notice->conversation)); - $this->out->elementStart('dl', 'response'); - $this->out->element('dt', null, _('To')); - $this->out->elementStart('dd'); - $this->out->element('a', array('href' => $convurl.'#notice-'.$this->notice->id), + $this->out->element('a', array('href' => $convurl.'#notice-'.$this->notice->id, + 'class' => 'response'), _('in context')); - $this->out->elementEnd('dd'); - $this->out->elementEnd('dl'); } } @@ -453,17 +444,12 @@ class NoticeListItem extends Widget if (common_logged_in()) { $reply_url = common_local_url('newnotice', array('replyto' => $this->profile->nickname)); - - $this->out->elementStart('dl', 'notice_reply'); - $this->out->element('dt', null, _('Reply to this notice')); - $this->out->elementStart('dd'); $this->out->elementStart('a', array('href' => $reply_url, + 'class' => 'notice_reply', 'title' => _('Reply to this notice'))); $this->out->text(_('Reply')); $this->out->element('span', 'notice_id', $this->notice->id); $this->out->elementEnd('a'); - $this->out->elementEnd('dd'); - $this->out->elementEnd('dl'); } } @@ -479,13 +465,9 @@ class NoticeListItem extends Widget if ($user && $this->notice->profile_id == $user->id) { $deleteurl = common_local_url('deletenotice', array('notice' => $this->notice->id)); - $this->out->elementStart('dl', 'notice_delete'); - $this->out->element('dt', null, _('Delete this notice')); - $this->out->elementStart('dd'); $this->out->element('a', array('href' => $deleteurl, + 'class' => 'notice_delete', 'title' => _('Delete this notice')), _('Delete')); - $this->out->elementEnd('dd'); - $this->out->elementEnd('dl'); } } diff --git a/lib/popularnoticesection.php b/lib/popularnoticesection.php index e47c9b385..167a6ff8d 100644 --- a/lib/popularnoticesection.php +++ b/lib/popularnoticesection.php @@ -74,11 +74,7 @@ class PopularNoticeSection extends NoticeSection $offset = 0; $limit = NOTICES_PER_SECTION + 1; - if (common_config('db', 'type') == 'pgsql') { - $qry .= ' LIMIT ' . $limit . ' OFFSET ' . $offset; - } else { - $qry .= ' LIMIT ' . $offset . ', ' . $limit; - } + $qry .= ' LIMIT ' . $limit . ' OFFSET ' . $offset; $notice = Memcached_DataObject::cachedQuery('Notice', sprintf($qry, common_config('popular', 'dropoff')), diff --git a/lib/profilesection.php b/lib/profilesection.php index 9ff243fb5..d463a07b0 100644 --- a/lib/profilesection.php +++ b/lib/profilesection.php @@ -97,7 +97,7 @@ class ProfileSection extends Section $this->out->elementEnd('a'); $this->out->elementEnd('span'); $this->out->elementEnd('td'); - if ($profile->value) { + if (isset($profile->value)) { $this->out->element('td', 'value', $profile->value); } diff --git a/lib/queuehandler.php b/lib/queuehandler.php index c2ff10f32..f11e5bd90 100644 --- a/lib/queuehandler.php +++ b/lib/queuehandler.php @@ -29,7 +29,6 @@ define('QUEUE_HANDLER_HIT_IDLE', 0); class QueueHandler extends Daemon { - var $_id = 'generic'; function __construct($id=null, $daemonize=true) { @@ -55,16 +54,6 @@ class QueueHandler extends Daemon return strtolower($this->class_name().'.'.$this->get_id()); } - function get_id() - { - return $this->_id; - } - - function set_id($id) - { - $this->_id = $id; - } - function transport() { return null; diff --git a/lib/router.php b/lib/router.php index e12138637..19839b997 100644 --- a/lib/router.php +++ b/lib/router.php @@ -113,6 +113,16 @@ class Router $m->connect('main/tagother/:id', array('action' => 'tagother')); + $m->connect('main/oembed.xml', + array('action' => 'api', + 'method' => 'oembed.xml', + 'apiaction' => 'oembed')); + + $m->connect('main/oembed.json', + array('action' => 'api', + 'method' => 'oembed.json', + 'apiaction' => 'oembed')); + // these take a code foreach (array('register', 'confirmaddress', 'recoverpassword') as $c) { @@ -206,7 +216,7 @@ class Router array('tag' => '[a-zA-Z0-9]+')); $m->connect('tag/:tag', array('action' => 'tag'), - array('tag' => '[a-zA-Z0-9]+')); + array('tag' => '[\pL\pN_\-\.]{1,64}')); $m->connect('peopletag/:tag', array('action' => 'peopletag'), @@ -394,6 +404,10 @@ class Router array('action' => 'api', 'apiaction' => 'laconica')); + $m->connect('api/laconica/:method', + array('action' => 'api', + 'apiaction' => 'laconica')); + // Groups $m->connect('api/laconica/groups/:method/:argument', array('action' => 'api', diff --git a/lib/rssaction.php b/lib/rssaction.php index 3b9f0fb47..0aca96566 100644 --- a/lib/rssaction.php +++ b/lib/rssaction.php @@ -39,6 +39,7 @@ class Rss10Action extends Action var $creators = array(); var $limit = DEFAULT_RSS_LIMIT; var $notices = null; + var $tags_already_output = array(); /** * Constructor @@ -96,11 +97,48 @@ class Rss10Action extends Action { // Parent handling, including cache check parent::handle($args); + + if (common_config('site', 'private')) { + if (!isset($_SERVER['PHP_AUTH_USER'])) { + + # This header makes basic auth go + header('WWW-Authenticate: Basic realm="Laconica RSS"'); + + # If the user hits cancel -- bam! + $this->show_basic_auth_error(); + return; + } else { + $nickname = $_SERVER['PHP_AUTH_USER']; + $password = $_SERVER['PHP_AUTH_PW']; + + if (!common_check_user($nickname, $password)) { + # basic authentication failed + list($proxy, $ip) = common_client_ip(); + + common_log(LOG_WARNING, "Failed RSS auth attempt, nickname = $nickname, proxy = $proxy, ip = $ip."); + $this->show_basic_auth_error(); + return; + } + } + } + // Get the list of notices $this->notices = $this->getNotices($this->limit); $this->showRss(); } + function show_basic_auth_error() + { + header('HTTP/1.1 401 Unauthorized'); + header('Content-Type: application/xml; charset=utf-8'); + $this->startXML(); + $this->elementStart('hash'); + $this->element('error', null, 'Could not authenticate you.'); + $this->element('request', null, $_SERVER['REQUEST_URI']); + $this->elementEnd('hash'); + $this->endXML(); + } + /** * Get the notices to output in this stream * @@ -188,24 +226,6 @@ class Rss10Action extends Action } } - // XXX: Surely there should be a common function to do this? - function extract_tags ($string) - { - $count = preg_match_all('/(?:^|\s)#([A-Za-z0-9_\-\.]{1,64})/', strtolower($string), $match); - if (!count) - { - return array(); - } - - $rv = array(); - foreach ($match[1] as $tag) - { - $rv[] = common_canonical_tag($tag); - } - - return array_unique($rv); - } - function showItem($notice) { $profile = Profile::staticGet($notice->profile_id); @@ -230,6 +250,11 @@ class Rss10Action extends Action $replyurl = common_local_url('shownotice', array('notice' => $notice->reply_to)); $this->element('sioc:reply_of', array('rdf:resource' => $replyurl)); } + if (!empty($notice->conversation)) { + $conversationurl = common_local_url('conversation', + array('id' => $notice->conversation)); + $this->element('sioc:has_discussion', array('rdf:resource' => $conversationurl)); + } $attachments = $notice->attachments(); if($attachments){ foreach($attachments as $attachment){ @@ -259,18 +284,28 @@ class Rss10Action extends Action $this->element('sioc:links_to', array('rdf:resource'=>$attachment->url)); } } - $tags = $this->extract_tags($notice->content); - if (!empty($tags)) { - foreach ($tags as $tag) - { - $tagpage = common_local_url('tag', array('tag' => $tag)); - $tagrss = common_local_url('tagrss', array('tag' => $tag)); + + $tag = new Notice_tag(); + $tag->notice_id = $notice->id; + if ($tag->find()) { + $entry['tags']=array(); + while ($tag->fetch()) { + $tagpage = common_local_url('tag', array('tag' => $tag->tag)); + + if ( in_array($tag, $this->tags_already_output) ) { + $this->element('ctag:tagged', array('rdf:resource'=>$tagpage.'#concept')); + continue; + } + + $tagrss = common_local_url('tagrss', array('tag' => $tag->tag)); $this->elementStart('ctag:tagged'); - $this->elementStart('ctag:Tag', array('rdf:about'=>$tagpage.'#concept', 'ctag:label'=>$tag)); + $this->elementStart('ctag:Tag', array('rdf:about'=>$tagpage.'#concept', 'ctag:label'=>$tag->tag)); $this->element('foaf:page', array('rdf:resource'=>$tagpage)); $this->element('rdfs:seeAlso', array('rdf:resource'=>$tagrss)); $this->elementEnd('ctag:Tag'); $this->elementEnd('ctag:tagged'); + + $this->tags_already_output[] = $tag->tag; } } $this->elementEnd('item'); diff --git a/lib/twitterapi.php b/lib/twitterapi.php index ab6c0d62c..4115d9dcb 100644 --- a/lib/twitterapi.php +++ b/lib/twitterapi.php @@ -186,24 +186,24 @@ class TwitterapiAction extends Action $twitter_status['favorited'] = false; } - # Enclosures + // Enclosures $attachments = $notice->attachments(); - - if (!empty($attachments)) { - - $twitter_status['attachments'] = array(); - - foreach ($attachments as $attachment) { - if ($attachment->isEnclosure()) { - $enclosure = array(); - $enclosure['url'] = $attachment->url; - $enclosure['mimetype'] = $attachment->mimetype; - $enclosure['size'] = $attachment->size; - $twitter_status['attachments'][] = $enclosure; - } + $enclosures = array(); + + foreach ($attachments as $attachment) { + if ($attachment->isEnclosure()) { + $enclosure = array(); + $enclosure['url'] = $attachment->url; + $enclosure['mimetype'] = $attachment->mimetype; + $enclosure['size'] = $attachment->size; + $enclosures[] = $enclosure; } } + if (!empty($enclosures)) { + $twitter_status['attachments'] = $enclosures; + } + if ($include_user) { # Don't get notice (recursive!) $twitter_user = $this->twitter_user_array($profile, false); @@ -213,12 +213,32 @@ class TwitterapiAction extends Action return $twitter_status; } + function twitter_group_array($group) + { + $twitter_group=array(); + $twitter_group['id']=$group->id; + $twitter_group['url']=$group->permalink(); + $twitter_group['nickname']=$group->nickname; + $twitter_group['fullname']=$group->fullname; + $twitter_group['homepage_url']=$group->homepage_url; + $twitter_group['original_logo']=$group->original_logo; + $twitter_group['homepage_logo']=$group->homepage_logo; + $twitter_group['stream_logo']=$group->stream_logo; + $twitter_group['mini_logo']=$group->mini_logo; + $twitter_group['homepage']=$group->homepage; + $twitter_group['description']=$group->description; + $twitter_group['location']=$group->location; + $twitter_group['created']=$this->date_twitter($group->created); + $twitter_group['modified']=$this->date_twitter($group->modified); + return $twitter_group; + } + function twitter_rss_entry_array($notice) { $profile = $notice->getProfile(); $entry = array(); - # We trim() to avoid extraneous whitespace in the output + // We trim() to avoid extraneous whitespace in the output $entry['content'] = common_xml_safe_str(trim($notice->rendered)); $entry['title'] = $profile->nickname . ': ' . common_xml_safe_str(trim($notice->content)); @@ -231,7 +251,26 @@ class TwitterapiAction extends Action $entry['updated'] = $entry['published']; $entry['author'] = $profile->getBestName(); - # Enclosure + // Enclosures + $attachments = $notice->attachments(); + $enclosures = array(); + + foreach ($attachments as $attachment) { + if ($attachment->isEnclosure()) { + $enclosure = array(); + $enclosure['url'] = $attachment->url; + $enclosure['mimetype'] = $attachment->mimetype; + $enclosure['size'] = $attachment->size; + $enclosures[] = $enclosure; + } + } + + if (!empty($enclosures)) { + $entry['enclosures'] = $enclosures; + } + +/* + // Enclosure $attachments = $notice->attachments(); if($attachments){ $entry['enclosures']=array(); @@ -245,8 +284,20 @@ class TwitterapiAction extends Action } } } +*/ - # RSS Item specific + // Tags/Categories + $tag = new Notice_tag(); + $tag->notice_id = $notice->id; + if ($tag->find()) { + $entry['tags']=array(); + while ($tag->fetch()) { + $entry['tags'][]=$tag->tag; + } + } + $tag->free(); + + // RSS Item specific $entry['description'] = $entry['content']; $entry['pubDate'] = common_date_rfc2822($notice->created); $entry['guid'] = $entry['link']; @@ -382,6 +433,15 @@ class TwitterapiAction extends Action $this->elementEnd('status'); } + function show_twitter_xml_group($twitter_group) + { + $this->elementStart('group'); + foreach($twitter_group as $element => $value) { + $this->element($element, null, $value); + } + $this->elementEnd('group'); + } + function show_twitter_xml_user($twitter_user, $role='user') { $this->elementStart($role); @@ -419,10 +479,16 @@ class TwitterapiAction extends Action $this->element('link', null, $entry['link']); # RSS only supports 1 enclosure per item - if($entry['enclosures']){ + if(array_key_exists('enclosures', $entry) and !empty($entry['enclosures'])){ $enclosure = $entry['enclosures'][0]; $this->element('enclosure', array('url'=>$enclosure['url'],'type'=>$enclosure['mimetype'],'length'=>$enclosure['size']), null); } + + if(array_key_exists('tags', $entry)){ + foreach($entry['tags'] as $tag){ + $this->element('category', null,$tag); + } + } $this->elementEnd('item'); } @@ -602,6 +668,22 @@ class TwitterapiAction extends Action $this->end_document('json'); } + function show_single_json_group($group) + { + $this->init_document('json'); + $twitter_group = $this->twitter_group_array($group); + $this->show_json_objects($twitter_group); + $this->end_document('json'); + } + + function show_single_xml_group($group) + { + $this->init_document('xml'); + $twitter_group = $this->twitter_group_array($group); + $this->show_twitter_xml_group($twitter_group); + $this->end_document('xml'); + } + // Anyone know what date format this is? // Twitter's dates look like this: "Mon Jul 14 23:52:38 +0000 2008" -- Zach function date_twitter($dt) diff --git a/lib/util.php b/lib/util.php index 9e8ec41d2..c8e318efe 100644 --- a/lib/util.php +++ b/lib/util.php @@ -140,7 +140,7 @@ function common_have_session() function common_ensure_session() { $c = null; - if (array_key_exists(session_name, $_COOKIE)) { + if (array_key_exists(session_name(), $_COOKIE)) { $c = $_COOKIE[session_name()]; } if (!common_have_session()) { @@ -404,7 +404,7 @@ function common_render_text($text) $r = preg_replace('/[\x{0}-\x{8}\x{b}-\x{c}\x{e}-\x{19}]/', '', $r); $r = common_replace_urls_callback($r, 'common_linkify'); - $r = preg_replace('/(^|\(|\[|\s+)#([A-Za-z0-9_\-\.]{1,64})/e', "'\\1#'.common_tag_link('\\2')", $r); + $r = preg_replace('/(^|\(|\[|\s+)#([\pL\pN_\-\.]{1,64})/e', "'\\1#'.common_tag_link('\\2')", $r); // XXX: machine tags return $r; } @@ -414,9 +414,9 @@ function common_replace_urls_callback($text, $callback, $notice_id = null) { $regex = '#'. '(?:'. '(?:'. - '(?:https?|ftps?|mms|rtsp|gopher|news|nntp|telnet|wais|file|prospero|webcal|xmpp|irc)://'. + '(?:https?|ftps?|mms|rtsp|gopher|news|nntp|telnet|wais|file|prospero|webcal|irc)://'. '|'. - '(?:mailto|aim|tel):'. + '(?:mailto|aim|tel|xmpp):'. ')'. '[^.\s]+\.[^\s]+'. '|'. @@ -595,7 +595,8 @@ function common_tag_link($tag) function common_canonical_tag($tag) { - return strtolower(str_replace(array('-', '_', '.'), '', $tag)); + $tag = mb_convert_case($tag, MB_CASE_LOWER, "UTF-8"); + return str_replace(array('-', '_', '.'), '', $tag); } function common_valid_profile_tag($str) @@ -1409,20 +1410,21 @@ function common_client_ip() return null; } - if ($_SERVER['HTTP_X_FORWARDED_FOR']) { - if ($_SERVER['HTTP_CLIENT_IP']) { + if (array_key_exists('HTTP_X_FORWARDED_FOR', $_SERVER)) { + if (array_key_exists('HTTP_CLIENT_IP', $_SERVER)) { $proxy = $_SERVER['HTTP_CLIENT_IP']; } else { $proxy = $_SERVER['REMOTE_ADDR']; } $ip = $_SERVER['HTTP_X_FORWARDED_FOR']; } else { - if ($_SERVER['HTTP_CLIENT_IP']) { + $proxy = null; + if (array_key_exists('HTTP_CLIENT_IP', $_SERVER)) { $ip = $_SERVER['HTTP_CLIENT_IP']; } else { $ip = $_SERVER['REMOTE_ADDR']; } } - return array($ip, $proxy); + return array($proxy, $ip); } diff --git a/lighttpd.conf.example b/lighttpd.conf.example new file mode 100644 index 000000000..b8baafc9e --- /dev/null +++ b/lighttpd.conf.example @@ -0,0 +1,2 @@ +# Add this line to lighttpd.conf to enable pseudo-rewrites using 404s +server.error-handler-404 = "/index.php" diff --git a/plugins/FBConnect/README b/plugins/FBConnect/README new file mode 100644 index 000000000..914b774cb --- /dev/null +++ b/plugins/FBConnect/README @@ -0,0 +1,77 @@ +This plugin allows you to utilize Facebook Connect with Laconica. +Supported Facebook Connect features: + +- Authenticate (register/login/logout -- works similar to OpenID) +- Associate an existing Laconica account with a Facebook account +- Disconnect a Facebook account from a Laconica account + +Future planned functionality: + +- Invite Facebook friends to use your Laconica installation +- Auto-subscribe Facebook friends already using Laconica +- Share Laconica favorite notices to your Facebook stream + +To use the plugin you will need to configure a Facebook application +to point to your Laconica installation (see the Installation section +below). + +Installation +============ + +If you don't already have the built-in Facebook application configured, +you'll need to log into Facebook and create/configure a new application. +Please follow the instructions in the section titled, "Setting Up Your +Application and Getting an API Key," on the following page of the +Facebook developer wiki: + + http://wiki.developers.facebook.com/index.php/Connect/Setting_Up_Your_Site + +If you already are using the build-in Laconica Facebook application, +you can modify your existing application's configuration using the +Facebook Developer Application on Facebook. Use it to edit your +application settings, and under the 'Connect' tab, change the 'Connect +URL' to be the main URL for your Laconica site. E.g.: + + http://SITE/PATH_TO_LACONICA/ + +After you application is created and configured, you'll need to add its +API key and secret to your Laconica config.php file: + + $config['facebook']['apikey'] = 'APIKEY'; + $config['facebook']['secret'] = 'SECRET'; + +Finally, to enable the plugin, add the following stanza to your +config.php: + + require_once(INSTALLDIR.'/plugins/FBConnect/FBConnectPlugin.php'); + $fbc = new FBConnectPlugin(); + +To try out the plugin, fire up your browser and connect to: + + http://SITE/PATH_TO_LACONICA/main/facebooklogin + +or, if you do not have fancy URLs turned on: + + http://SITE/PATH_TO_LACONICA/index.php/main/facebooklogin + +You should see a page with a blue button that says: "Connect with +Facebook". + +Connect/Disconnect existing account +=================================== + +If the Facebook Connect plugin is enabled, there will be a new Facebook +Connect Settings tab under each user's Connect menu. Users can connect +and disconnect to their Facebook accounts from it. Note: Before a user +can disconnect from Facebook, she must set a normal Laconica password. +Otherwise, she might not be able to login in to her account in the +future. This is usually only required for users who have used Facebook +Connect to register their Laconica account, and therefore haven't +already set a local password. + +Helpful links +============= + +Facebook Connect Homepage: +http://developers.facebook.com/connect.php + diff --git a/plugins/recaptcha/LICENSE b/plugins/recaptcha/LICENSE new file mode 100644 index 000000000..b612f71f0 --- /dev/null +++ b/plugins/recaptcha/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2007 reCAPTCHA -- http://recaptcha.net +AUTHORS: + Mike Crawford + Ben Maurer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/plugins/recaptcha/README b/plugins/recaptcha/README new file mode 100644 index 000000000..3100f697e --- /dev/null +++ b/plugins/recaptcha/README @@ -0,0 +1,23 @@ +Laconica reCAPTCHA plugin 0.2 8/3/09 +==================================== +Adds a captcha to your registration page to reduce automated spam bots registering. + +Use: +1. Get an API key from http://recaptcha.net + +2. In config.php add: +include_once('plugins/recaptcha.php'); +$captcha = new recaptcha(publickey, privatekey, showErrors); + +Changelog +========= +0.1 initial release +0.2 Work around for webkit browsers + +reCAPTCHA README +================ + +The reCAPTCHA PHP Lirary helps you use the reCAPTCHA API. Documentation +for this library can be found at + + http://recaptcha.net/plugins/php diff --git a/plugins/recaptcha/recaptcha.php b/plugins/recaptcha/recaptcha.php new file mode 100644 index 000000000..5ef8352d1 --- /dev/null +++ b/plugins/recaptcha/recaptcha.php @@ -0,0 +1,106 @@ +<?php +/** + * Laconica, the distributed open-source microblogging tool + * + * Plugin to show reCaptcha when a user registers + * + * PHP version 5 + * + * LICENCE: This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * @category Plugin + * @package Laconica + * @author Eric Helgeson <erichelgeson@gmail.com> + * @copyright 2009 + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +if (!defined('LACONICA')) { + exit(1); +} + +define('RECAPTCHA', '0.2'); + +class recaptcha extends Plugin +{ + var $private_key; + var $public_key; + var $display_errors; + var $failed; + var $ssl; + + function __construct($public_key, $private_key, $display_errors=false) + { + parent::__construct(); + require_once(INSTALLDIR.'/plugins/recaptcha/recaptchalib.php'); + $this->public_key = $public_key; + $this->private_key = $private_key; + $this->display_errors = $display_errors; + } + + function checkssl(){ + if(common_config('site', 'ssl') === 'sometimes' || common_config('site', 'ssl') === 'always') { + return true; + } + return false; + } + + function onStartShowHTML($action) + { + //XXX: Horrible hack to make Safari, FF2, and Chrome work with + //reChapcha. reChapcha beaks xhtml strict + header('Content-Type: text/html'); + + $action->extraHeaders(); + + $action->startXML('html', + '-//W3C//DTD XHTML 1.0 Strict//EN', + 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd'); + + $action->raw('<style type="text/css">#recaptcha_area{float:left;}</style>'); + return false; + } + + function onEndRegistrationFormData($action) + { + $action->elementStart('li'); + $action->raw('<label for="recaptcha_area">Captcha</label>'); + if($this->checkssl() === true){ + $action->raw(recaptcha_get_html($this->public_key), null, true); + } else { + $action->raw(recaptcha_get_html($this->public_key)); + } + $action->elementEnd('li'); + return true; + } + + function onStartRegistrationTry($action) + { + $resp = recaptcha_check_answer ($this->private_key, + $_SERVER["REMOTE_ADDR"], + $action->trimmed('recaptcha_challenge_field'), + $action->trimmed('recaptcha_response_field')); + + if (!$resp->is_valid) + { + if($this->display_errors) + { + $action->showForm ("(reCAPTCHA said: " . $resp->error . ")"); + } + $action->showForm("Captcha does not match!"); + return false; + } + } +} diff --git a/plugins/recaptcha/recaptchalib.php b/plugins/recaptcha/recaptchalib.php new file mode 100644 index 000000000..897c50981 --- /dev/null +++ b/plugins/recaptcha/recaptchalib.php @@ -0,0 +1,277 @@ +<?php +/* + * This is a PHP library that handles calling reCAPTCHA. + * - Documentation and latest version + * http://recaptcha.net/plugins/php/ + * - Get a reCAPTCHA API Key + * http://recaptcha.net/api/getkey + * - Discussion group + * http://groups.google.com/group/recaptcha + * + * Copyright (c) 2007 reCAPTCHA -- http://recaptcha.net + * AUTHORS: + * Mike Crawford + * Ben Maurer + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * The reCAPTCHA server URL's + */ +define("RECAPTCHA_API_SERVER", "http://api.recaptcha.net"); +define("RECAPTCHA_API_SECURE_SERVER", "https://api-secure.recaptcha.net"); +define("RECAPTCHA_VERIFY_SERVER", "api-verify.recaptcha.net"); + +/** + * Encodes the given data into a query string format + * @param $data - array of string elements to be encoded + * @return string - encoded request + */ +function _recaptcha_qsencode ($data) { + $req = ""; + foreach ( $data as $key => $value ) + $req .= $key . '=' . urlencode( stripslashes($value) ) . '&'; + + // Cut the last '&' + $req=substr($req,0,strlen($req)-1); + return $req; +} + + + +/** + * Submits an HTTP POST to a reCAPTCHA server + * @param string $host + * @param string $path + * @param array $data + * @param int port + * @return array response + */ +function _recaptcha_http_post($host, $path, $data, $port = 80) { + + $req = _recaptcha_qsencode ($data); + + $http_request = "POST $path HTTP/1.0\r\n"; + $http_request .= "Host: $host\r\n"; + $http_request .= "Content-Type: application/x-www-form-urlencoded;\r\n"; + $http_request .= "Content-Length: " . strlen($req) . "\r\n"; + $http_request .= "User-Agent: reCAPTCHA/PHP\r\n"; + $http_request .= "\r\n"; + $http_request .= $req; + + $response = ''; + if( false == ( $fs = @fsockopen($host, $port, $errno, $errstr, 10) ) ) { + die ('Could not open socket'); + } + + fwrite($fs, $http_request); + + while ( !feof($fs) ) + $response .= fgets($fs, 1160); // One TCP-IP packet + fclose($fs); + $response = explode("\r\n\r\n", $response, 2); + + return $response; +} + + + +/** + * Gets the challenge HTML (javascript and non-javascript version). + * This is called from the browser, and the resulting reCAPTCHA HTML widget + * is embedded within the HTML form it was called from. + * @param string $pubkey A public key for reCAPTCHA + * @param string $error The error given by reCAPTCHA (optional, default is null) + * @param boolean $use_ssl Should the request be made over ssl? (optional, default is false) + + * @return string - The HTML to be embedded in the user's form. + */ +function recaptcha_get_html ($pubkey, $error = null, $use_ssl = false) +{ + if ($pubkey == null || $pubkey == '') { + die ("To use reCAPTCHA you must get an API key from <a href='http://recaptcha.net/api/getkey'>http://recaptcha.net/api/getkey</a>"); + } + + if ($use_ssl) { + $server = RECAPTCHA_API_SECURE_SERVER; + } else { + $server = RECAPTCHA_API_SERVER; + } + + $errorpart = ""; + if ($error) { + $errorpart = "&error=" . $error; + } + return '<script type="text/javascript" src="'. $server . '/challenge?k=' . $pubkey . $errorpart . '"></script> + + <noscript> + <iframe src="'. $server . '/noscript?k=' . $pubkey . $errorpart . '" height="300" width="500" frameborder="0"></iframe><br/> + <textarea name="recaptcha_challenge_field" rows="3" cols="40"></textarea> + <input type="hidden" name="recaptcha_response_field" value="manual_challenge"/> + </noscript>'; +} + + + + +/** + * A ReCaptchaResponse is returned from recaptcha_check_answer() + */ +class ReCaptchaResponse { + var $is_valid; + var $error; +} + + +/** + * Calls an HTTP POST function to verify if the user's guess was correct + * @param string $privkey + * @param string $remoteip + * @param string $challenge + * @param string $response + * @param array $extra_params an array of extra variables to post to the server + * @return ReCaptchaResponse + */ +function recaptcha_check_answer ($privkey, $remoteip, $challenge, $response, $extra_params = array()) +{ + if ($privkey == null || $privkey == '') { + die ("To use reCAPTCHA you must get an API key from <a href='http://recaptcha.net/api/getkey'>http://recaptcha.net/api/getkey</a>"); + } + + if ($remoteip == null || $remoteip == '') { + die ("For security reasons, you must pass the remote ip to reCAPTCHA"); + } + + + + //discard spam submissions + if ($challenge == null || strlen($challenge) == 0 || $response == null || strlen($response) == 0) { + $recaptcha_response = new ReCaptchaResponse(); + $recaptcha_response->is_valid = false; + $recaptcha_response->error = 'incorrect-captcha-sol'; + return $recaptcha_response; + } + + $response = _recaptcha_http_post (RECAPTCHA_VERIFY_SERVER, "/verify", + array ( + 'privatekey' => $privkey, + 'remoteip' => $remoteip, + 'challenge' => $challenge, + 'response' => $response + ) + $extra_params + ); + + $answers = explode ("\n", $response [1]); + $recaptcha_response = new ReCaptchaResponse(); + + if (trim ($answers [0]) == 'true') { + $recaptcha_response->is_valid = true; + } + else { + $recaptcha_response->is_valid = false; + $recaptcha_response->error = $answers [1]; + } + return $recaptcha_response; + +} + +/** + * gets a URL where the user can sign up for reCAPTCHA. If your application + * has a configuration page where you enter a key, you should provide a link + * using this function. + * @param string $domain The domain where the page is hosted + * @param string $appname The name of your application + */ +function recaptcha_get_signup_url ($domain = null, $appname = null) { + return "http://recaptcha.net/api/getkey?" . _recaptcha_qsencode (array ('domain' => $domain, 'app' => $appname)); +} + +function _recaptcha_aes_pad($val) { + $block_size = 16; + $numpad = $block_size - (strlen ($val) % $block_size); + return str_pad($val, strlen ($val) + $numpad, chr($numpad)); +} + +/* Mailhide related code */ + +function _recaptcha_aes_encrypt($val,$ky) { + if (! function_exists ("mcrypt_encrypt")) { + die ("To use reCAPTCHA Mailhide, you need to have the mcrypt php module installed."); + } + $mode=MCRYPT_MODE_CBC; + $enc=MCRYPT_RIJNDAEL_128; + $val=_recaptcha_aes_pad($val); + return mcrypt_encrypt($enc, $ky, $val, $mode, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); +} + + +function _recaptcha_mailhide_urlbase64 ($x) { + return strtr(base64_encode ($x), '+/', '-_'); +} + +/* gets the reCAPTCHA Mailhide url for a given email, public key and private key */ +function recaptcha_mailhide_url($pubkey, $privkey, $email) { + if ($pubkey == '' || $pubkey == null || $privkey == "" || $privkey == null) { + die ("To use reCAPTCHA Mailhide, you have to sign up for a public and private key, " . + "you can do so at <a href='http://mailhide.recaptcha.net/apikey'>http://mailhide.recaptcha.net/apikey</a>"); + } + + + $ky = pack('H*', $privkey); + $cryptmail = _recaptcha_aes_encrypt ($email, $ky); + + return "http://mailhide.recaptcha.net/d?k=" . $pubkey . "&c=" . _recaptcha_mailhide_urlbase64 ($cryptmail); +} + +/** + * gets the parts of the email to expose to the user. + * eg, given johndoe@example,com return ["john", "example.com"]. + * the email is then displayed as john...@example.com + */ +function _recaptcha_mailhide_email_parts ($email) { + $arr = preg_split("/@/", $email ); + + if (strlen ($arr[0]) <= 4) { + $arr[0] = substr ($arr[0], 0, 1); + } else if (strlen ($arr[0]) <= 6) { + $arr[0] = substr ($arr[0], 0, 3); + } else { + $arr[0] = substr ($arr[0], 0, 4); + } + return $arr; +} + +/** + * Gets html to display an email address given a public an private key. + * to get a key, go to: + * + * http://mailhide.recaptcha.net/apikey + */ +function recaptcha_mailhide_html($pubkey, $privkey, $email) { + $emailparts = _recaptcha_mailhide_email_parts ($email); + $url = recaptcha_mailhide_url ($pubkey, $privkey, $email); + + return htmlentities($emailparts[0]) . "<a href='" . htmlentities ($url) . + "' onclick=\"window.open('" . htmlentities ($url) . "', '', 'toolbar=0,scrollbars=0,location=0,statusbar=0,menubar=0,resizable=0,width=500,height=300'); return false;\" title=\"Reveal this e-mail address\">...</a>@" . htmlentities ($emailparts [1]); + +} + + +?> diff --git a/scripts/createsim.php b/scripts/createsim.php new file mode 100644 index 000000000..71827ba5b --- /dev/null +++ b/scripts/createsim.php @@ -0,0 +1,142 @@ +#!/usr/bin/env php +<?php +/* + * Laconica - a distributed open-source microblogging tool + * Copyright (C) 2008, 2009, Control Yourself, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$shortoptions = 'u:n:b:t:x:'; +$longoptions = array('users=', 'notices=', 'subscriptions=', 'tags=', 'prefix='); + +$helptext = <<<END_OF_CREATESIM_HELP +Creates a lot of test users and notices to (loosely) simulate a real server. + + -u --users Number of users (default 100) + -n --notices Average notices per user (default 100) + -b --subscriptions Average subscriptions per user (default no. users/20) + -t --tags Number of distinct hash tags (default 10000) + -x --prefix User name prefix (default 'testuser') + +END_OF_CREATESIM_HELP; + +require_once INSTALLDIR.'/scripts/commandline.inc'; + +// XXX: make these command-line options + +function newUser($i) +{ + global $userprefix; + User::register(array('nickname' => sprintf('%s%d', $userprefix, $i), + 'password' => sprintf('password%d', $i), + 'fullname' => sprintf('Test User %d', $i))); +} + +function newNotice($i, $tagmax) +{ + global $userprefix; + + $n = rand(0, $i - 1); + $user = User::staticGet('nickname', sprintf('%s%d', $userprefix, $n)); + + $is_reply = rand(0, 4); + + $content = 'Test notice content'; + + if ($is_reply == 0) { + $n = rand(0, $i - 1); + $content = "@$userprefix$n " . $content; + } + + $has_hash = rand(0, 2); + + if ($has_hash == 0) { + $hashcount = rand(0, 2); + for ($j = 0; $j < $hashcount; $j++) { + $h = rand(0, $tagmax); + $content .= " #tag{$h}"; + } + } + + $notice = Notice::saveNew($user->id, $content, 'system'); +} + +function newSub($i) +{ + global $userprefix; + $f = rand(0, $i - 1); + + $fromnick = sprintf('%s%d', $userprefix, $f); + + $from = User::staticGet('nickname', $fromnick); + + if (empty($from)) { + throw new Exception("Can't find user '$fromnick'."); + } + + $t = rand(0, $i - 1); + + if ($t == $f) { + $t++; + if ($t > $i - 1) { + $t = 0; + } + } + + $tunic = sprintf('%s%d', $userprefix, $t); + + $to = User::staticGet('nickname', $tunic); + + if (empty($from)) { + throw new Exception("Can't find user '$tunic'."); + } + + subs_subscribe_to($from, $to); +} + +function main($usercount, $noticeavg, $subsavg, $tagmax) +{ + $n = 1; + + newUser(0); + + // # registrations + # notices + # subs + + $events = $usercount + ($usercount * ($noticeavg + $subsavg)); + + for ($i = 0; $i < $events; $i++) + { + $e = rand(0, 1 + $noticeavg + $subsavg); + + if ($e == 0) { + newUser($n); + $n++; + } else if ($e < $noticeavg + 1) { + newNotice($n, $tagmax); + } else { + newSub($n); + } + } +} + +$usercount = (have_option('u', 'users')) ? get_option_value('u', 'users') : 100; +$noticeavg = (have_option('n', 'notices')) ? get_option_value('n', 'notices') : 100; +$subsavg = (have_option('b', 'subscriptions')) ? get_option_value('b', 'subscriptions') : max($usercount/20, 10); +$tagmax = (have_option('t', 'tags')) ? get_option_value('t', 'tags') : 10000; +$userprefix = (have_option('x', 'prefix')) ? get_option_value('x', 'prefix') : 'testuser'; + +main($usercount, $noticeavg, $subsavg, $tagmax); diff --git a/scripts/getvaliddaemons.php b/scripts/getvaliddaemons.php index 97c230784..1e4546dff 100755 --- a/scripts/getvaliddaemons.php +++ b/scripts/getvaliddaemons.php @@ -28,7 +28,8 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); $helptext = <<<ENDOFHELP -getvaliddaemons.php - print out the currently configured PID directory +getvaliddaemons.php - print out a list of valid daemons that should be started +by the startdaemons script ENDOFHELP; diff --git a/scripts/maildaemon.php b/scripts/maildaemon.php index 11ddf06b7..3ef4d0638 100755 --- a/scripts/maildaemon.php +++ b/scripts/maildaemon.php @@ -299,25 +299,43 @@ class MailerDaemon $attachments = array(); + $this->extract_part($parsed,$msg,$attachments); + + return array($from, $to, $msg, $attachments); + } + + function extract_part($parsed,&$msg,&$attachments){ if ($parsed->ctype_primary == 'multipart') { - foreach ($parsed->parts as $part) { - if ($part->ctype_primary == 'text' && - $part->ctype_secondary == 'plain') { - $msg = $part->body; - }else{ - if ($part->body) { - $attachment = tmpfile(); - fwrite($attachment, $part->body); - $attachments[] = $attachment; - } + if($parsed->ctype_secondary == 'alternative'){ + $altmsg = $this->extract_msg_from_multipart_alternative_part($parsed); + if(!empty($altmsg)) $msg = $altmsg; + }else{ + foreach($parsed->parts as $part){ + $this->extract_part($part,$msg,$attachments); } } - } else if ($type == 'text/plain') { + } else if ($parsed->ctype_primary == 'text' + && $parsed->ctype_secondary=='plain') { $msg = $parsed->body; - } else { - $this->unsupported_type($type); + if(strtolower($parsed->ctype_parameters['charset']) != "utf-8"){ + $msg = utf8_encode($msg); + } + }else if(!empty($parsed->body)){ + if(common_config('attachments', 'uploads')){ + //only save attachments if uploads are enabled + $attachment = tmpfile(); + fwrite($attachment, $parsed->body); + $attachments[] = $attachment; + } } - return array($from, $to, $msg, $attachments); + } + + function extract_msg_from_multipart_alternative_part($parsed){ + foreach ($parsed->parts as $part) { + $this->extract_part($part,$msg,$attachments); + } + //we don't want any attachments that are a result of this parsing + return $msg; } function unsupported_type($type) diff --git a/scripts/triminboxes.php b/scripts/triminboxes.php index b2135d682..27e200fef 100644 --- a/scripts/triminboxes.php +++ b/scripts/triminboxes.php @@ -52,43 +52,5 @@ if (!empty($id)) { $cnt = $user->find(); while ($user->fetch()) { - - $inbox_entry = new Notice_inbox(); - $inbox_entry->user_id = $user->id; - $inbox_entry->orderBy('created DESC'); - $inbox_entry->limit(1000, 1); - - $id = null; - - if ($inbox_entry->find(true)) { - $id = $inbox_entry->notice_id; - } - - $inbox_entry->free(); - unset($inbox_entry); - - if (is_null($id)) { - continue; - } - - $start = microtime(true); - - $old_inbox = new Notice_inbox(); - $cnt = $old_inbox->query('DELETE from notice_inbox WHERE user_id = ' . $user->id . ' AND notice_id < ' . $id); - $old_inbox->free(); - unset($old_inbox); - - print "Deleted $cnt notices for $user->nickname ($user->id).\n"; - - $finish = microtime(true); - - $delay = 3.0 * ($finish - $start); - - print "Delaying $delay seconds..."; - - // Wait to let slaves catch up - - usleep($delay * 1000000); - - print "DONE.\n"; + Notice_inbox::gc($user->id); } diff --git a/scripts/twitterstatusfetcher.php b/scripts/twitterstatusfetcher.php index 4a1ed8977..e1745cfc0 100755 --- a/scripts/twitterstatusfetcher.php +++ b/scripts/twitterstatusfetcher.php @@ -25,19 +25,18 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); define('MAXCHILDREN', 2); define('POLL_INTERVAL', 60); // in seconds -$shortoptions = 'i::'; -$longoptions = array('id::'); +$shortoptions = 'di::'; +$longoptions = array('id::', 'debug'); $helptext = <<<END_OF_TRIM_HELP Batch script for retrieving Twitter messages from foreign service. - -i --id Identity (default 'generic') + -i --id Identity (default 'generic') + -d --debug Debug (lots of log output) END_OF_TRIM_HELP; -require_once INSTALLDIR.'/scripts/commandline.inc'; - -require_once INSTALLDIR . '/lib/common.php'; +require_once INSTALLDIR .'/scripts/commandline.inc'; require_once INSTALLDIR . '/lib/daemon.php'; /** @@ -61,6 +60,15 @@ class TwitterStatusFetcher extends Daemon { private $_children = array(); + function __construct($id=null, $daemonize=true) + { + parent::__construct($daemonize); + + if ($id) { + $this->set_id($id); + } + } + /** * Name of this daemon * @@ -80,6 +88,11 @@ class TwitterStatusFetcher extends Daemon function run() { + if (defined('SCRIPT_DEBUG')) { + common_debug($this->name() . + ': debugging log output enabled.'); + } + do { $flinks = $this->refreshFlinks(); @@ -640,6 +653,10 @@ if (have_option('i')) { $id = null; } +if (have_option('d') || have_option('debug')) { + define('SCRIPT_DEBUG', true); +} + $fetcher = new TwitterStatusFetcher($id); $fetcher->runOnce(); diff --git a/scripts/xmppdaemon.php b/scripts/xmppdaemon.php index 488b4b514..69512f243 100755 --- a/scripts/xmppdaemon.php +++ b/scripts/xmppdaemon.php @@ -175,6 +175,10 @@ class XMPPDaemon extends Daemon $user = $this->get_user($from); + // For common_current_user to work + global $_cur; + $_cur = $user; + if (!$user) { $this->from_site($from, 'Unknown user; go to ' . common_local_url('imsettings') . @@ -211,6 +215,7 @@ class XMPPDaemon extends Daemon $user->free(); unset($user); + unset($_cur); unset($pl['xml']); $pl['xml'] = null; diff --git a/theme/base/css/display.css b/theme/base/css/display.css index 3604f193a..364ea0b62 100644 --- a/theme/base/css/display.css +++ b/theme/base/css/display.css @@ -482,7 +482,7 @@ height:16px; } #form_notice .form_note { position:absolute; -top:99px; +bottom:2px; right:98px; z-index:9; } @@ -863,7 +863,7 @@ clear:left; float:left; font-size:0.95em; margin-left:59px; -width:60%; +width:50%; } #showstream .notice div.entry-content, #shownotice .notice div.entry-content { @@ -876,22 +876,9 @@ float:left; font-size:1.025em; } -.notice div.entry-content dl, -.notice div.entry-content dt, -.notice div.entry-content dd { -display:inline; -} - -.notice div.entry-content .timestamp dt, -.notice div.entry-content .response dt { -display:none; -} -.notice div.entry-content .timestamp a { +.notice div.entry-content .timestamp { display:inline-block; } -.notice div.entry-content .device dt { -text-transform:lowercase; -} .notice-options { position:relative; @@ -921,38 +908,28 @@ left:29px; .notice-options .notice_delete { right:0; } -.notice-options .notice_reply dt { -display:none; -} - .notice-options input, .notice-options a { text-indent:-9999px; outline:none; } - -.notice-options .notice_reply a, .notice-options input.submit { display:block; border:0; } -.notice-options .notice_reply a, -.notice-options .notice_delete a { +.notice-options .notice_reply, +.notice-options .notice_delete { text-decoration:none; padding-left:16px; } - .notice-options form input.submit { width:16px; padding:2px 0; } - -.notice-options .notice_delete dt, .notice-options .form_favor legend, .notice-options .form_disfavor legend { display:none; } -.notice-options .notice_delete fieldset, .notice-options .form_favor fieldset, .notice-options .form_disfavor fieldset { border:0; diff --git a/theme/base/css/ie6.css b/theme/base/css/ie6.css index dde4d6fc7..edc49478f 100644 --- a/theme/base/css/ie6.css +++ b/theme/base/css/ie6.css @@ -12,7 +12,7 @@ margin:0 auto; } #content { -width:70%; +width:69%; } #aside_primary { padding:5%; @@ -35,3 +35,6 @@ width:20%; width:50%; margin-left:30px; } +.notice-options a { +width:16px; +}
\ No newline at end of file diff --git a/theme/cloudy/css/display.css b/theme/cloudy/css/display.css index 12f186a56..e3c5c3cc0 100644 --- a/theme/cloudy/css/display.css +++ b/theme/cloudy/css/display.css @@ -12,12 +12,12 @@ img { display:block; border:0; } a abbr { cursor: pointer; border-bottom:0; } table { border-collapse:collapse; } ol { list-style-position:inside; } -html { font-size: 100%; background-color:#fff; } +html { font-size: 87.5%; } body { background-color:#fff; color:#000; font-family:sans-serif; -font-size:0.75em; +font-size:1em; line-height:normal; position:relative; height:100%; @@ -414,9 +414,12 @@ width:518px; min-height:322px; padding:20px; float:left; -border-radius-topleft:4px; --moz-border-radius-topleft:4px; --webkit-border-top-left-radius:4px; +border-radius:4px; +-moz-border-radius:4px; +-webkit-border-radius:4px; +border-radius-topright:0; +-moz-border-radius-topright:0; +-webkit-border-top-right-radius:0; border-style:solid; border-width:1px; } @@ -471,6 +474,24 @@ margin-bottom:7px; #form_notice #notice_submit label { display:none; } +#form_notice label[for=notice_data-attach], +#form_notice #notice_data-attach { +position:absolute; +top:87px; +cursor:pointer; +} +#form_notice label[for=notice_data-attach] { +text-indent:-9999px; +left:82%; +width:16px; +height:16px; +} +#form_notice #notice_data-attach { +left:40%; +padding:0; +height:16px; +} + #form_notice .form_note { position:absolute; top:-10px; @@ -488,6 +509,7 @@ font-weight:bold; line-height:1.15; padding:1px 2px; } + #form_notice #notice_action-submit { width:14%; height:35px; @@ -505,6 +527,29 @@ margin-bottom:7px; margin-left:18px; float:left; } +#form_notice .error, +#form_notice .success { +float:left; +clear:both; +width:83.5%; +margin-bottom:0; +line-height:1.618; +position:absolute; +top:87px; +left:0; +} +#form_notice #notice_data-attach_selected code { +float:left; +width:90%; +display:block; +font-size:1.1em; +line-height:1.8; +overflow:auto; +} +#form_notice #notice_data-attach_selected button { +float:right; +font-size:0.8em; +} /* entity_profile */ @@ -746,17 +791,18 @@ float:left; width:100%; border-top-width:1px; border-top-style:dotted; -font-size:1.2em; } .notices li { list-style-type:none; line-height:1.1; -width:94%; -padding-right:5%; -padding-left:1%; min-height:47px; } - +.notices .notices { +margin-top:7px; +margin-left:2%; +width:98%; +float:left; +} /* NOTICES */ #notices_primary { @@ -805,13 +851,17 @@ text-decoration:underline; .notice .entry-title { float:none; -display:inline; -width:100%; overflow:hidden; +display:inline; } #shownotice .notice .entry-title { font-size:2.2em; } +#conversation .notice .entry-title { +display:block; +width:90%; +} + .notice p.entry-content { display:inline; @@ -887,13 +937,11 @@ outline:none; .notice-options { padding-left:2%; float:left; -width:50%; font-size:0.95em; -width:12.5%; +width:16px; float:right; -display:none; } -.notices li.hover div.notice-options { +.notices li:hover div.notice-options { display:block; } @@ -917,7 +965,7 @@ top:30px; right:7px; } .notice-options .notice_delete { -bottom:7px; +top:47px; right:7px; } .notice-options .notice_reply dt { @@ -941,6 +989,10 @@ text-decoration:none; padding-left:16px; } +.notice-options .notice_reply .notice_id { +display:none; +} + .notice-options form input.submit { width:16px; padding:2px 0; @@ -958,6 +1010,97 @@ border:0; padding:0; } +.notice .attachment { +position:relative; +padding-left:16px; +} +#attachments .attachment { +padding-left:0; +} +.notice .attachment img { +position:absolute; +top:18px; +left:0; +z-index:99; +} +#shownotice .notice .attachment img { +position:static; +} + +#attachments { +clear:both; +float:left; +width:100%; +margin-top:18px; +} +#attachments dt { +font-weight:bold; +font-size:1.3em; +margin-bottom:4px; +} + +#attachments ol li { +margin-bottom:18px; +list-style-type:decimal; +float:left; +clear:both; +} + +#jOverlayContent, +#jOverlayContent #content, +#jOverlayContent #content_inner { +width: auto !important; +margin-bottom:0; +} +#jOverlayContent #content { +padding:11px; +min-height:auto; +} +#jOverlayContent .external span { +display:block; +margin-bottom:11px; +} +#jOverlayContent button { +position:absolute; +top:0; +right:0; +width:29px; +height:29px; +text-align:center; +font-weight:bold; +padding:0; +} +#jOverlayContent h1 { +max-width:425px; +} +#jOverlayContent #content { +border-radius:7px; +-moz-border-radius:7px; +-webkit-border-radius:7px; +} +#jOverlayLoading { +top:5%; +left:40%; +} +#attachment_view img { +max-width:480px; +max-height:480px; +} +#attachment_view #oembed_info { +margin-top:11px; +} +#attachment_view #oembed_info dt, +#attachment_view #oembed_info dd { +float:left; +} +#attachment_view #oembed_info dt { +clear:left; +margin-right:11px; +font-weight:bold; +} +#attachment_view #oembed_info dt:after { +content: ":"; +} #usergroups #new_group { float: left; @@ -1218,7 +1361,8 @@ clear:both; #outbox.user_in #content, #subscriptions.user_in #content, #subscribers.user_in #content, -#showgroup.user_in #content { +#showgroup.user_in #content, +#conversation.user_in #content { padding-top:160px; } @@ -1243,6 +1387,14 @@ padding-top:160px; display:none; } +#jOverlayContent #core #content { +padding-top:11px; +} +#jOverlayContent #core { +background:none; +padding-top:0; +} + html, body, @@ -1311,24 +1463,23 @@ border-top-color:#87B4C8; } -#content .notice p.entry-content a:visited { -background-color:#fcfcfc; -} -#content .notice p.entry-content .vcard a { -background-color:#fcfffc; -} - #aside_primary { background-color:#DDFFCC; } - #notice_text-count { -color:#333; +color:#000000; } #form_notice.warning #notice_text-count { -color:#000; +color:#000000; +} +#form_notice label[for=notice_data-attach] { +background:transparent url(../../base/images/icons/twotone/green/clip-01.gif) no-repeat 0 45%; } +#form_notice #notice_data-attach { +opacity:0; +} + #form_notice.processing #notice_action-submit { background:#fff url(../../base/images/icons/icon_processing.gif) no-repeat 47% 47%; cursor:wait; @@ -1426,8 +1577,11 @@ background-image:url(../images/icons/twotone/green/shield.gif); /* NOTICES */ -.notices li.over { -background-color:#fcfcfc; +.notice .attachment { +background:transparent url(../../base/images/icons/twotone/green/clip-02.gif) no-repeat 0 45%; +} +#attachments .attachment { +background:none; } .notice-options .notice_reply a, @@ -1451,10 +1605,24 @@ background:transparent url(../images/icons/icon_trash.gif) no-repeat 0 45%; .notices div.notice-options { opacity:0.4; } -.notices li.hover div.entry-content, -.notices li.hover div.notice-options { +.notices li:hover div.entry-content, +.notices li:hover div.notice-options { opacity:1; } +.notices .notices { +background-color:rgba(200, 200, 200, 0.01); +} +.notices .notices .notices { +background-color:rgba(200, 200, 200, 0.02); +} +.notices .notices .notices .notices { +background-color:rgba(200, 200, 200, 0.03); +} +.notices .notices .notices .notices .notices { +background-color:rgba(200, 200, 200, 0.04); +} + + div.entry-content { color:#333; } @@ -1462,8 +1630,11 @@ div.notice-options a, div.notice-options input { font-family:sans-serif; } -.notices li.hover { -background-color:#fcfcfc; +#content .notices li:hover { +background-color:rgba(240, 240, 240, 0.2); +} +#conversation .notices li:hover { +background-color:transparent; } /*END: NOTICES */ diff --git a/theme/cloudy/css/ie.css b/theme/cloudy/css/ie.css index 095122100..a698676fb 100644 --- a/theme/cloudy/css/ie.css +++ b/theme/cloudy/css/ie.css @@ -8,13 +8,41 @@ color:#fff; background-color:#ddffcc; } +#form_notice { +width:525px; +} +#form_notice .form_note { +top:-5px; +right:0; +} +#form_notice textarea { +width:97.75%; +} +#form_notice .form_note + label { +position:absolute; +top:87px; +left:77%; +text-indent:-9999px; +height:16px; +width:16px; +display:block; +} +#form_notice #notice_data-attach { +filter: alpha(opacity = 0); +left:33.5%; +} + +#form_notice #notice_action-submit { +right:0; +} + #aside_primary { width:181px; } #form_notice, #anon_notice { -top:158px; +top:190px; } #public #content, @@ -32,3 +60,13 @@ top:158px; #subscribers #content { padding-top:138px; } + +.notice { +z-index:1; +} +.notice:hover { +z-index:9999; +} +.notice .thumbnail img { +z-index:9999; +} diff --git a/theme/default/css/display.css b/theme/default/css/display.css index 251d6706b..921a6b27b 100644 --- a/theme/default/css/display.css +++ b/theme/default/css/display.css @@ -214,11 +214,7 @@ background:transparent url(../../base/images/icons/twotone/green/clip-02.gif) no #attachments .attachment { background:none; } -.notice-options .notice_reply a, -.notice-options form input.submit { -background-color:transparent; -} -.notice-options .notice_reply a { +.notice-options .notice_reply { background:transparent url(../../base/images/icons/twotone/green/reply.gif) no-repeat 0 45%; } .notice-options form.form_favor input.submit { @@ -227,7 +223,7 @@ background:transparent url(../../base/images/icons/twotone/green/favourite.gif) .notice-options form.form_disfavor input.submit { background:transparent url(../../base/images/icons/twotone/green/disfavourite.gif) no-repeat 0 45%; } -.notice-options .notice_delete a { +.notice-options .notice_delete { background:transparent url(../../base/images/icons/twotone/green/trash.gif) no-repeat 0 45%; } @@ -239,9 +235,6 @@ opacity:0.4; .notices li:hover div.notice-options { opacity:1; } -div.entry-content { -color:#333333; -} div.notice-options a, div.notice-options input { font-family:sans-serif; diff --git a/theme/identica/css/display.css b/theme/identica/css/display.css index 42a4573a7..8af5644b6 100644 --- a/theme/identica/css/display.css +++ b/theme/identica/css/display.css @@ -214,11 +214,7 @@ background:transparent url(../../base/images/icons/twotone/green/clip-02.gif) no #attachments .attachment { background:none; } -.notice-options .notice_reply a, -.notice-options form input.submit { -background-color:transparent; -} -.notice-options .notice_reply a { +.notice-options .notice_reply { background:transparent url(../../base/images/icons/twotone/green/reply.gif) no-repeat 0 45%; } .notice-options form.form_favor input.submit { @@ -227,7 +223,7 @@ background:transparent url(../../base/images/icons/twotone/green/favourite.gif) .notice-options form.form_disfavor input.submit { background:transparent url(../../base/images/icons/twotone/green/disfavourite.gif) no-repeat 0 45%; } -.notice-options .notice_delete a { +.notice-options .notice_delete { background:transparent url(../../base/images/icons/twotone/green/trash.gif) no-repeat 0 45%; } @@ -239,9 +235,6 @@ opacity:0.4; .notices li:hover div.notice-options { opacity:1; } -div.entry-content { -color:#333333; -} div.notice-options a, div.notice-options input { font-family:sans-serif; diff --git a/theme/pigeonthoughts/css/base.css b/theme/pigeonthoughts/css/base.css index 9866e2d2c..980d7ddd7 100644 --- a/theme/pigeonthoughts/css/base.css +++ b/theme/pigeonthoughts/css/base.css @@ -383,7 +383,7 @@ margin-bottom:1em; } #content { -width:49.009%; +width:50%; min-height:259px; float:left; padding:0 18px; @@ -402,7 +402,7 @@ float:left; width:45.917%; min-height:259px; float:left; -margin-left:1.385%; +margin-left:0.25%; padding-bottom:47px; } @@ -736,11 +736,10 @@ margin-right:11px; .notice, .profile { position:relative; -padding-top:11px; -padding-bottom:11px; +padding:11px 2%; clear:both; float:left; -width:96.41%; +width:95.7%; border-width:1px; border-style:solid; margin-bottom:11px; @@ -848,23 +847,9 @@ margin-left:0; float:left; font-size:1.025em; } - -.notice div.entry-content dl, -.notice div.entry-content dt, -.notice div.entry-content dd { -display:inline; -} - -.notice div.entry-content .timestamp dt, -.notice div.entry-content .response dt { -display:none; -} -.notice div.entry-content .timestamp a { +.notice div.entry-content .timestamp { display:inline-block; } -.notice div.entry-content .device dt { -text-transform:lowercase; -} .notice-options { position:relative; @@ -894,38 +879,28 @@ left:29px; .notice-options .notice_delete { right:0; } -.notice-options .notice_reply dt { -display:none; -} - .notice-options input, .notice-options a { text-indent:-9999px; outline:none; } - -.notice-options .notice_reply a, .notice-options input.submit { display:block; border:0; } -.notice-options .notice_reply a, -.notice-options .notice_delete a { +.notice-options .notice_reply, +.notice-options .notice_delete { text-decoration:none; padding-left:16px; } - .notice-options form input.submit { width:16px; padding:2px 0; } - -.notice-options .notice_delete dt, .notice-options .form_favor legend, .notice-options .form_disfavor legend { display:none; } -.notice-options .notice_delete fieldset, .notice-options .form_favor fieldset, .notice-options .form_disfavor fieldset { border:0; @@ -993,13 +968,36 @@ font-weight:bold; padding:0; } #jOverlayContent h1 { -max-width:475px; +max-width:425px; } #jOverlayContent #content { border-radius:7px; -moz-border-radius:7px; -webkit-border-radius:7px; } +#jOverlayLoading { +top:5%; +left:40%; +} +#attachment_view img { +max-width:480px; +max-height:480px; +} +#attachment_view #oembed_info { +margin-top:11px; +} +#attachment_view #oembed_info dt, +#attachment_view #oembed_info dd { +float:left; +} +#attachment_view #oembed_info dt { +clear:left; +margin-right:11px; +font-weight:bold; +} +#attachment_view #oembed_info dt:after { +content: ":"; +} #usergroups #new_group { float: left; @@ -1058,8 +1056,6 @@ top:3px; left:3px; } - - .pagination { float:left; clear:both; @@ -1105,7 +1101,6 @@ padding-right:30px; } /* END: NOTICE */ - .hentry .entry-content p { margin-bottom:18px; } @@ -1122,7 +1117,6 @@ margin-bottom:18px; margin-left:18px; } - /* TOP_POSTERS */ .section tbody td { padding-right:11px; @@ -1150,7 +1144,6 @@ margin-right:0; display:none; } - /* tagcloud */ .tag-cloud { list-style-type:none; @@ -1233,6 +1226,11 @@ clear:both; margin-bottom:0; } +#form_settings_design #settings_design_background-image img { +max-width:480px; +max-height:480px; +} + #form_settings_design #settings_design_color .form_data, #form_settings_design #color-picker { float:left; diff --git a/theme/pigeonthoughts/css/display.css b/theme/pigeonthoughts/css/display.css index 01af500bf..1bf37bf73 100644 --- a/theme/pigeonthoughts/css/display.css +++ b/theme/pigeonthoughts/css/display.css @@ -14,7 +14,8 @@ background:url(../images/illustrations/illu_pigeons-01.png) no-repeat 0 100%; } body, -a:active { +a:active, +#content { background-color:#AEA187; } body { @@ -268,11 +269,7 @@ background:transparent url(../../base/images/icons/twotone/green/clip-02.gif) no #attachments .attachment { background:none; } -.notice-options .notice_reply a, -.notice-options form input.submit { -background-color:transparent; -} -.notice-options .notice_reply a { +.notice-options .notice_reply { background:transparent url(../../base/images/icons/twotone/green/reply.gif) no-repeat 0 45%; } .notice-options form.form_favor input.submit { @@ -281,7 +278,7 @@ background:transparent url(../../base/images/icons/twotone/green/favourite.gif) .notice-options form.form_disfavor input.submit { background:transparent url(../../base/images/icons/twotone/green/disfavourite.gif) no-repeat 0 45%; } -.notice-options .notice_delete a { +.notice-options .notice_delete { background:transparent url(../../base/images/icons/twotone/green/trash.gif) no-repeat 0 45%; } |