From 698228acb9a6262f2fb4e0c82118249aa94cc0f8 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 13:16:59 -0400 Subject: Revert "Revert "run 'set names' after each connection to deal with UTF8 correctly"" This reverts commit 86770ccde7914219a0a572ced6dd21fa65566e1d. --- classes/Memcached_DataObject.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 5f71f716b..877bbf2e0 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,4 +227,20 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } + + // We overload so that 'SET NAMES "utf8"' is called for + // each connection + + function _connect() + { + global $_DB_DATAOBJECT; + $exists = !empty($this->_database_dsn_md5) && + isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); + $result = parent::_connect(); + if (!$exists) { + $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; + $DB->query('SET NAMES "utf8"'); + } + return $result; + } } -- cgit v1.2.3-54-g00ecf From 18b130456287a24ac9024b57dc7a38d1dcaa498f Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 25 May 2009 14:23:00 -0700 Subject: Add EventBox to notice sources --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index 7c31a9af4..221b1ee01 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -7,6 +7,7 @@ VALUES ('cliqset', 'Cliqset', 'http://www.cliqset.com/', now()), ('deskbar','Deskbar-Applet','http://www.gnome.org/projects/deskbar-applet/', now()), ('Do','Gnome Do','http://do.davebsd.com/wiki/index.php?title=Microblog_Plugin', now()), + ('eventbox','EventBox','http://thecosmicmachine.com/eventbox/ ', now()), ('Facebook','Facebook','http://apps.facebook.com/identica/', now()), ('feed2omb','feed2omb','http://projects.ciarang.com/p/feed2omb/', now()), ('gravity', 'Gravity', 'http://mobileways.de/gravity', now()), -- cgit v1.2.3-54-g00ecf From 5f2cf2553cbe97b9983c8c5d0a85abb00b1a8375 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 26 May 2009 02:23:19 +0000 Subject: Ticket #1196 - Fixed: Warning on invite tab in Facebook App --- actions/facebookinvite.php | 65 +++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/actions/facebookinvite.php b/actions/facebookinvite.php index 1302064ad..2207580f7 100644 --- a/actions/facebookinvite.php +++ b/actions/facebookinvite.php @@ -17,7 +17,9 @@ * along with this program. If not, see . */ -if (!defined('LACONICA')) { exit(1); } +if (!defined('LACONICA')) { + exit(1); +} require_once(INSTALLDIR.'/lib/facebookaction.php'); @@ -67,7 +69,7 @@ class FacebookinviteAction extends FacebookAction function showSuccessContent() { - $this->element('h2', null, sprintf(_('Thanks for inviting your friends to use %s'), + $this->element('h2', null, sprintf(_('Thanks for inviting your friends to use %s'), common_config('site', 'name'))); $this->element('p', null, _('Invitations have been sent to the following users:')); @@ -89,16 +91,6 @@ class FacebookinviteAction extends FacebookAction function showFormContent() { - - // Get a list of users who are already using the app for exclusion - $exclude_ids = $this->facebook->api_client->friends_getAppUsers(); - $exclude_ids_csv = null; - - // fbml needs these as a csv string, not an array - if ($exclude_ids) { - $exclude_ids_csv = implode(',', $exclude_ids); - } - $content = sprintf(_('You have been invited to %s'), common_config('site', 'name')) . htmlentities(''); @@ -109,36 +101,43 @@ class FacebookinviteAction extends FacebookAction 'content' => $content)); $this->hidden('invite', 'true'); $actiontext = sprintf(_('Invite your friends to use %s'), common_config('site', 'name')); - - $multi_params = array('showborder' => 'false'); + + $multi_params = array('showborder' => 'false'); $multi_params['actiontext'] = $actiontext; - - if ($exclude_ids_csv) { + $multi_params['bypass'] = 'cancel'; + + // Get a list of users who are already using the app for exclusion + $exclude_ids = $this->facebook->api_client->friends_getAppUsers(); + $exclude_ids_csv = null; + + // fbml needs these as a csv string, not an array + if ($exclude_ids) { + $exclude_ids_csv = implode(',', $exclude_ids); $multi_params['exclude_ids'] = $exclude_ids_csv; } - $multi_params['bypass'] = 'cancel'; - $this->element('fb:multi-friend-selector', $multi_params); - $this->elementEnd('fb:request-form'); - $this->element('h2', null, sprintf(_('Friends already using %s:'), - common_config('site', 'name'))); - $this->elementStart('ul', array('id' => 'facebook-friends')); - - foreach ($exclude_ids as $friend) { - $this->elementStart('li'); - $this->element('fb:profile-pic', array('uid' => $friend, 'size' => 'square')); - $this->element('fb:name', array('uid' => $friend, - 'capitalize' => 'true')); - $this->elementEnd('li'); - } + if ($exclude_ids) { - $this->elementEnd("ul"); + $this->element('h2', null, sprintf(_('Friends already using %s:'), + common_config('site', 'name'))); + $this->elementStart('ul', array('id' => 'facebook-friends')); + + foreach ($exclude_ids as $friend) { + $this->elementStart('li'); + $this->element('fb:profile-pic', array('uid' => $friend, 'size' => 'square')); + $this->element('fb:name', array('uid' => $friend, + 'capitalize' => 'true')); + $this->elementEnd('li'); + } + + $this->elementEnd("ul"); + } } - - function title() + + function title() { return sprintf(_('Send invitations')); } -- cgit v1.2.3-54-g00ecf From a1f83b293394af34782f74640c3d29f800bdc0bf Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 26 May 2009 15:22:12 -0400 Subject: added qtwitter --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index 7c31a9af4..e7ccadde3 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -28,6 +28,7 @@ VALUES ('pingvine','PingVine','http://pingvine.com/', now()), ('pocketwit','PockeTwit','http://code.google.com/p/pocketwit/', now()), ('posty','Posty','http://spreadingfunkyness.com/posty/', now()), + ('qtwitter','qTwitter','http://qtwitter.ayoy.net/', now()), ('royalewithcheese','Royale With Cheese','http://p.hellyeah.org/', now()), ('rssdent','rssdent','http://github.com/zcopley/rssdent/tree/master', now()), ('rygh.no','rygh.no','http://rygh.no/', now()), -- cgit v1.2.3-54-g00ecf From 74b08bff53b614a0d3da56e848940cd3e9ca70ce Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 26 May 2009 15:27:13 -0400 Subject: Added AgentSolo.com --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index a22967b1b..ac73d3d13 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -2,6 +2,7 @@ INSERT INTO notice_source (code, name, url, created) VALUES ('adium', 'Adium', 'http://www.adiumx.com/', now()), + ('AgentSolo.com','AgentSolo.com','http://www.agentsolo.com/', now()), ('betwittered','BeTwittered','http://www.32hours.com/betwitteredinfo/', now()), ('bti','bti','http://gregkh.github.com/bti/', now()), ('cliqset', 'Cliqset', 'http://www.cliqset.com/', now()), -- cgit v1.2.3-54-g00ecf From b2c28faf526e55ddc97d42692048e644b004e1f8 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 26 May 2009 21:14:42 +0000 Subject: Removed silly "dented:" status prefix from Facebook App --- actions/facebookhome.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/actions/facebookhome.php b/actions/facebookhome.php index 5946e6c98..4c2b26355 100644 --- a/actions/facebookhome.php +++ b/actions/facebookhome.php @@ -138,9 +138,6 @@ class FacebookhomeAction extends FacebookAction function setDefaults() { - // A default prefix string for notices - $this->facebook->api_client->data_setUserPreference( - FACEBOOK_NOTICE_PREFIX, 'dented: '); $this->facebook->api_client->data_setUserPreference( FACEBOOK_PROMPTED_UPDATE_PREF, 'false'); } -- cgit v1.2.3-54-g00ecf From 2b81dcb25329c496c7c4f2c99334b79ba0421c1e Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 15 May 2009 21:44:58 +0000 Subject: Better err handling when trying to get Facebook client --- lib/facebookutil.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/facebookutil.php b/lib/facebookutil.php index ec3987273..242d2e06f 100644 --- a/lib/facebookutil.php +++ b/lib/facebookutil.php @@ -27,9 +27,21 @@ define("FACEBOOK_PROMPTED_UPDATE_PREF", 2); function getFacebook() { + static $facebook = null; + $apikey = common_config('facebook', 'apikey'); $secret = common_config('facebook', 'secret'); - return new Facebook($apikey, $secret); + + if ($facebook === null) { + $facebook = new Facebook($apikey, $secret); + } + + if (!$facebook) { + common_log(LOG_ERR, 'Could not make new Facebook client obj!', + __FILE__); + } + + return $facebook; } function updateProfileBox($facebook, $flink, $notice) { @@ -92,7 +104,6 @@ function isFacebookBound($notice, $flink) { } - function facebookBroadcastNotice($notice) { $facebook = getFacebook(); -- cgit v1.2.3-54-g00ecf From d068680e1913a32eba0f5b0e1e746ac66ec638d3 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 27 May 2009 14:57:45 -0400 Subject: optionally use SET NAMES for utf8 to DB --- README | 4 ++++ classes/Memcached_DataObject.php | 4 +++- lib/common.php | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README b/README index 7b2dcacc5..c23d4e464 100644 --- a/README +++ b/README @@ -906,6 +906,10 @@ mirror: you can set this to an array of DSNs, like the above and adding the slaves to this array. Note that if you want some requests to go to the 'database' (master) server, you'll need to include it in this array, too. +utf8: whether to talk to the database in UTF-8 mode. This is the default + with new installations, but older sites may want to turn it off + until they get their databases fixed up. See "UTF-8 database" + above for details. syslog ------ diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 877bbf2e0..52ad4100f 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -239,7 +239,9 @@ class Memcached_DataObject extends DB_DataObject $result = parent::_connect(); if (!$exists) { $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; - $DB->query('SET NAMES "utf8"'); + if (common_config('db', 'utf8')) { + $DB->query('SET NAMES "utf8"'); + } } return $result; } diff --git a/lib/common.php b/lib/common.php index f983c4d16..8f95c2361 100644 --- a/lib/common.php +++ b/lib/common.php @@ -167,6 +167,7 @@ $config['db'] = 'require_prefix' => 'classes/', 'class_prefix' => '', 'mirror' => null, + 'utf8' => true, 'db_driver' => 'DB', # XXX: JanRain libs only work with DB 'quote_identifiers' => false, 'type' => 'mysql' ); -- cgit v1.2.3-54-g00ecf From 497d1bfe0283598021f2be2bd6664ac8bbab25ce Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 27 May 2009 19:12:37 -0400 Subject: Add fixup_utf8 to fixup problems with UTF-8 in a database Fixes up an old database to store strings in UTF-8. Hopefully works! --- scripts/fixup_utf8.php | 131 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 scripts/fixup_utf8.php diff --git a/scripts/fixup_utf8.php b/scripts/fixup_utf8.php new file mode 100644 index 000000000..0763c72c9 --- /dev/null +++ b/scripts/fixup_utf8.php @@ -0,0 +1,131 @@ +#!/usr/bin/env php +. + */ + +# Abort if called from a web server +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(1); +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); +require_once('DB.php'); + +function main() { + + $dbl = doConnect('latin1'); + + if (empty($dbl)) { + return; + } + + $dbu = doConnect('utf8'); + + if (empty($dbu)) { + return; + } + + // Do a separate DB connection + + $sth = $dbu->prepare("UPDATE notice SET content = UNHEX(?), rendered = UNHEX(?) WHERE id = ?"); + + if (PEAR::isError($sth)) { + echo "ERROR: " . $sth->getMessage() . "\n"; + return; + } + + $rn = $dbl->query('SELECT id, content, rendered FROM notice ' . + 'WHERE LENGTH(content) != CHAR_LENGTH(content)'); + + if (PEAR::isError($rn)) { + echo "ERROR: " . $rn->getMessage() . "\n"; + return; + } + + echo "Number of rows: " . $rn->numRows() . "\n"; + + $notice = array(); + + while (DB_OK == $rn->fetchInto($notice)) { + + $id = ($notice[0])+0; + $content = bin2hex($notice[1]); + $rendered = bin2hex($notice[2]); + + echo "$id..."; + + $result =& $dbu->execute($sth, array($content, $rendered, $id)); + + if (PEAR::isError($result)) { + echo "ERROR: " . $result->getMessage() . "\n"; + continue; + } + + $cnt = $dbu->affectedRows(); + + if ($cnt != 1) { + echo "ERROR: 0 rows affected\n"; + continue; + } + + $notice = Notice::staticGet('id', $id); + $notice->decache(); + + echo "OK\n"; + } +} + +function doConnect($charset) +{ + $db = DB::connect(common_config('db', 'database'), + array('persistent' => false)); + + if (PEAR::isError($db)) { + echo "ERROR: " . $db->getMessage() . "\n"; + return NULL; + } + + $result = $db->query("SET NAMES $charset"); + + if (PEAR::isError($result)) { + echo "ERROR: " . $result->getMessage() . "\n"; + $db->disconnect(); + return NULL; + } + + $result = $db->autoCommit(true); + + if (PEAR::isError($result)) { + echo "ERROR: " . $result->getMessage() . "\n"; + $db->disconnect(); + return NULL; + } + + return $db; +} + +main(); -- cgit v1.2.3-54-g00ecf From 121cb6afb1822517e215eef23a5cb87796ecea35 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 28 May 2009 16:35:34 -0400 Subject: document utf8 problems and solutions --- README | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README b/README index c23d4e464..4649eb79c 100644 --- a/README +++ b/README @@ -690,6 +690,13 @@ to users on a remote site. (Or not... it's not well tested.) The Upgrading ========= +IMPORTANT NOTE: Laconica 0.7.4 introduced a fix for some +incorrectly-stored international characters ("UTF-8"). For new +installations, it will now store non-ASCII characters correctly. +However, older installations will have the incorrect storage, and will +consequently show up "wrong" in browsers. See below for how to deal +with this situation. + If you've been using Laconica 0.6, 0.5 or lower, or if you've been tracking the "git" version of the software, you will probably want to upgrade and keep your existing data. There is no automated upgrade @@ -779,6 +786,29 @@ problem. 3. When fixup_inboxes is finished, you can set the enabled flag to 'true'. +UTF-8 Database +-------------- + +Laconica 0.7.4 introduced a fix for some incorrectly-stored +international characters ("UTF-8"). This fix is not +backwards-compatible; installations from before 0.7.4 will show +non-ASCII characters of old notices incorrectly. This section explains +what to do. + +0. You can disable the new behaviour by setting the 'db''utf8' config + option to "false". You should only do this until you're ready to + convert your DB to the new format. +1. When you're ready to convert, you can run the fixup_utf8.php script + in the scripts/ subdirectory. If you've had the "new behaviour" + enabled (probably a good idea), you can give the ID of the first + "new" notice as a parameter, and only notices before that one will + be converted. Notices are converted in reverse chronological order, + so the most recent (and visible) ones will be converted first. The + script should work whether or not you have the 'db''utf8' config + option enabled. +2. When you're ready, set $config['db']['utf8'] to true, so that + new notices will be stored correctly. + Configuration options ===================== -- cgit v1.2.3-54-g00ecf From daf845dbe67a909e2bc9e039d3c2ccc560345b4d Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 28 May 2009 16:36:07 -0400 Subject: take an argument for fixup_utf8 --- scripts/fixup_utf8.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/fixup_utf8.php b/scripts/fixup_utf8.php index 0763c72c9..e5021ff34 100644 --- a/scripts/fixup_utf8.php +++ b/scripts/fixup_utf8.php @@ -35,7 +35,7 @@ define('LACONICA', true); require_once(INSTALLDIR . '/lib/common.php'); require_once('DB.php'); -function main() { +function fixup_utf8($id) { $dbl = doConnect('latin1'); @@ -58,8 +58,16 @@ function main() { return; } - $rn = $dbl->query('SELECT id, content, rendered FROM notice ' . - 'WHERE LENGTH(content) != CHAR_LENGTH(content)'); + $sql = 'SELECT id, content, rendered FROM notice ' . + 'WHERE LENGTH(content) != CHAR_LENGTH(content)'; + + if (!empty($id)) { + $sql .= ' AND id < ' . $id; + } + + $sql .= ' ORDER BY id DESC'; + + $rn = $dbl->query($sql); if (PEAR::isError($rn)) { echo "ERROR: " . $rn->getMessage() . "\n"; @@ -128,4 +136,6 @@ function doConnect($charset) return $db; } -main(); +$id = ($argc > 1) ? $argv[1] : null; + +fixup_utf8($id); -- cgit v1.2.3-54-g00ecf