From 7a2d72fe28e29c67f2a9e8fdb59c6c5c9b38d5e6 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 4 Jan 2010 12:49:25 -0500 Subject: Enable memcache automatic compression, starting at 20k and only if compression gain is greater than 20%. Allows storage of larger objects (over 1mb in size uncompressed), such as huge LDAP schemas. Should also improve cache efficiency (allows more stuff to be stored in same memory) and reduce network latency (less data transfer) (redo commit 1e9c03e1993b5d2978ac4c5213a8a64e0150b4a2 which was apparently lost during pluginization) --- plugins/MemcachePlugin.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/MemcachePlugin.php b/plugins/MemcachePlugin.php index acbec135e..78e2b2406 100644 --- a/plugins/MemcachePlugin.php +++ b/plugins/MemcachePlugin.php @@ -156,6 +156,11 @@ class MemcachePlugin extends Plugin } $this->_conn->addServer($host, $port); } + //Compress items stored in the cache if they're over 2k in size + //and the compression would save more than 20%. + //Allows the cache to store objects larger than 1MB (if they + //compress to less than 1MB), and improves cache memory efficiency. + $this->_conn->setCompressThreshold(20000, 0.2); } } } -- cgit v1.2.3-54-g00ecf From 783a2e249bac2ccd02b6ae3351dfe83630b71cc0 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 10:30:19 -0800 Subject: Fix for auto_increment parameter in auto-created tables via checkschema. Update FeedSub plugin for non-Plugin_DataObject setup and working checkschema updates. --- lib/columndef.php | 1 + lib/schema.php | 4 ++ plugins/FeedSub/FeedSubPlugin.php | 7 ++- plugins/FeedSub/feedinfo.php | 108 +++++++++++++++++++++++++++++--------- 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/lib/columndef.php b/lib/columndef.php index 1bae6b33b..ac2fcd23e 100644 --- a/lib/columndef.php +++ b/lib/columndef.php @@ -74,6 +74,7 @@ class ColumnDef * @param string $key type of key * @param value $default default value * @param value $extra unused + * @param boolean $auto_increment */ function __construct($name=null, $type=null, $size=null, diff --git a/lib/schema.php b/lib/schema.php index a8ba91b87..6fe442d56 100644 --- a/lib/schema.php +++ b/lib/schema.php @@ -523,6 +523,10 @@ class Schema } else { $sql .= ($cd->nullable) ? "null " : "not null "; } + + if (!empty($cd->auto_increment)) { + $sql .= " auto_increment "; + } return $sql; } diff --git a/plugins/FeedSub/FeedSubPlugin.php b/plugins/FeedSub/FeedSubPlugin.php index 857a9794d..e49e2a648 100644 --- a/plugins/FeedSub/FeedSubPlugin.php +++ b/plugins/FeedSub/FeedSubPlugin.php @@ -105,12 +105,11 @@ class FeedSubPlugin extends Plugin return true; } - /* - // auto increment seems to be broken function onCheckSchema() { + // warning: the autoincrement doesn't seem to set. + // alter table feedinfo change column id id int(11) not null auto_increment; $schema = Schema::get(); - $schema->ensureDataObject('Feedinfo'); + $schema->ensureTable('feedinfo', Feedinfo::schemaDef()); return true; } - */ } diff --git a/plugins/FeedSub/feedinfo.php b/plugins/FeedSub/feedinfo.php index fff66afe9..b166bd6e1 100644 --- a/plugins/FeedSub/feedinfo.php +++ b/plugins/FeedSub/feedinfo.php @@ -31,7 +31,7 @@ class FeedDBException extends FeedSubException } } -class Feedinfo extends Plugin_DataObject +class Feedinfo extends Memcached_DataObject { public $__table = 'feedinfo'; @@ -56,34 +56,90 @@ class Feedinfo extends Plugin_DataObject return parent::staticGet(__CLASS__, $k, $v); } - function tableDef() + /** + * return table definition for DB_DataObject + * + * DB_DataObject needs to know something about the table to manipulate + * instances. This method provides all the DB_DataObject needs to know. + * + * @return array array of column definitions + */ + + function table() + { + return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + 'profile_id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + 'feeduri' => DB_DATAOBJECT_STR + DB_DATAOBJECT_NOTNULL, + 'homeuri' => DB_DATAOBJECT_STR + DB_DATAOBJECT_NOTNULL, + 'huburi' => DB_DATAOBJECT_STR + DB_DATAOBJECT_NOTNULL, + 'verify_token' => DB_DATAOBJECT_STR, + 'sub_start' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME, + 'sub_end' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME, + 'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL, + 'lastupdate' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL); + } + + static function schemaDef() + { + return array(new ColumnDef('id', 'integer', + /*size*/ null, + /*nullable*/ false, + /*key*/ 'PRI', + /*default*/ '0', + /*extra*/ null, + /*auto_increment*/ true), + new ColumnDef('profile_id', 'integer', + null, false), + new ColumnDef('feeduri', 'varchar', + 255, false, 'UNI'), + new ColumnDef('homeuri', 'varchar', + 255, false), + new ColumnDef('huburi', 'varchar', + 255, false), + new ColumnDef('verify_token', 'varchar', + 32, true), + new ColumnDef('sub_start', 'datetime', + null, true), + new ColumnDef('sub_end', 'datetime', + null, true), + new ColumnDef('created', 'datetime', + null, false), + new ColumnDef('lastupdate', 'datetime', + null, false)); + } + + /** + * return key definitions for DB_DataObject + * + * DB_DataObject needs to know about keys that the table has; this function + * defines them. + * + * @return array key definitions + */ + + function keys() { - class_exists('Schema'); // autoload hack - // warning: the autoincrement doesn't seem to set. - // alter table feedinfo change column id id int(11) not null auto_increment; - return new TableDef($this->__table, - array(new ColumnDef('id', 'integer', - null, false, 'PRI', '0', null, true), - new ColumnDef('profile_id', 'integer', - null, false), - new ColumnDef('feeduri', 'varchar', - 255, false, 'UNI'), - new ColumnDef('homeuri', 'varchar', - 255, false), - new ColumnDef('huburi', 'varchar', - 255, false), - new ColumnDef('verify_token', 'varchar', - 32, true), - new ColumnDef('sub_start', 'datetime', - null, true), - new ColumnDef('sub_end', 'datetime', - null, true), - new ColumnDef('created', 'datetime', - null, false), - new ColumnDef('lastupdate', 'datetime', - null, false))); + return array('id' => 'P'); //? } + /** + * return key definitions for Memcached_DataObject + * + * Our caching system uses the same key definitions, but uses a different + * method to get them. + * + * @return array key definitions + */ + + function keyTypes() + { + return $this->keys(); + } + + /** + * Fetch the StatusNet-side profile for this feed + * @return Profile + */ public function getProfile() { return Profile::staticGet('id', $this->profile_id); -- cgit v1.2.3-54-g00ecf From e440b69e1a3c73796535298b3b20d6678431f9b6 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 4 Jan 2010 13:33:52 -0500 Subject: Allow an authentication plugin with the same provider_name other than the one that actually checked the password to autoregister a user Allows for SSO-type plugins that don't have any information about the user other than their username to do autoregistration --- plugins/Authentication/AuthenticationPlugin.php | 28 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/plugins/Authentication/AuthenticationPlugin.php b/plugins/Authentication/AuthenticationPlugin.php index a76848b04..75e8d2b76 100644 --- a/plugins/Authentication/AuthenticationPlugin.php +++ b/plugins/Authentication/AuthenticationPlugin.php @@ -99,6 +99,23 @@ abstract class AuthenticationPlugin extends Plugin } } + /** + * Internal AutoRegister event handler + * @param nickname + * @param provider_name + * @param user - the newly registered user + */ + function onAutoRegister($nickname, $provider_name, &$user) + { + if($provider_name == $this->provider_name && $this->autoregistration){ + $user = $this->autoregister($nickname); + if($user){ + User_username::register($user,$nickname,$this->provider_name); + return false; + } + } + } + function onStartCheckPassword($nickname, $password, &$authenticatedUser){ //map the nickname to a username $user_username = new User_username(); @@ -127,13 +144,10 @@ abstract class AuthenticationPlugin extends Plugin } } }else{ - if($this->autoregistration){ - $authenticated = $this->checkPassword($nickname, $password); - if($authenticated){ - $user = $this->autoregister($nickname); - if($user){ - $authenticatedUser = $user; - User_username::register($authenticatedUser,$nickname,$this->provider_name); + $authenticated = $this->checkPassword($nickname, $password); + if($authenticated){ + if(Event::handle('AutoRegister', array($nickname, $this->provider_name, &$authenticatedUser))){ + if($authenticatedUser){ return false; } } -- cgit v1.2.3-54-g00ecf From 11d7365a15bb8c8cb856bf0d545a0b550f3dbe26 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 10:39:11 -0800 Subject: Don't spew notices when building tag cloud if there is no popularity sum to divide by. --- actions/publictagcloud.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index e7f6ee36c..5c7074029 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -136,7 +136,12 @@ class PublictagcloudAction extends Action $this->elementStart('dd'); $this->elementStart('ul', 'tags xoxo tag-cloud'); foreach ($tw as $tag => $weight) { - $this->showTag($tag, $weight, $weight/$sum); + if ($sum) { + $weightedSum = $weight/$sum; + } else { + $weightedSum = 1; + } + $this->showTag($tag, $weight, $weightedSum); } $this->elementEnd('ul'); $this->elementEnd('dd'); -- cgit v1.2.3-54-g00ecf From 38912b34c7f9d0a377feeb76572003a36c61ee7e Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 10:41:52 -0800 Subject: Drop the overly-prominent link to checklibs display; it's unnecessary and just confuses people. The ability's still there to aid in debugging, but it won't be tempting people to click on it. --- install.php | 1 - 1 file changed, 1 deletion(-) diff --git a/install.php b/install.php index 1c62bb2b2..435f6d63b 100644 --- a/install.php +++ b/install.php @@ -454,7 +454,6 @@ function showForm()

Enter your database connection information below to initialize the database.

-

StatusNet bundles a number of libraries for ease of installation. You can see what bundled libraries you are using, versus what libraries are installed on your server.

-- cgit v1.2.3-54-g00ecf From 8f362e9956ef699511344e0a494669ae78aa93eb Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 08:53:28 -1000 Subject: user_id is a non-autoincrement pkey for user_location_prefs --- classes/statusnet.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/statusnet.ini b/classes/statusnet.ini index 4077746c4..ac31148da 100644 --- a/classes/statusnet.ini +++ b/classes/statusnet.ini @@ -572,5 +572,5 @@ created = 142 modified = 384 [user_location_prefs__keys] -user_id = U +user_id = K -- cgit v1.2.3-54-g00ecf From 5a1ea0b9b28138bc4e06d96e2910b0cf3be1eab2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 08:59:19 -1000 Subject: Stop caching unfindable keys There were some problems with the automated cache/uncache system for data objects that made us cache unfindable keys (with null attributes and sometimes null names). Fixed those problems and refactored the encache() and decache() methods so they use a helper to find the cache keys to use. --- classes/Memcached_DataObject.php | 83 ++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 1608720d1..020d813b2 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -152,52 +152,69 @@ class Memcached_DataObject extends DB_DataObject function encache() { $c = $this->memcache(); + if (!$c) { return false; - } else { - $pkey = array(); - $pval = array(); - $types = $this->keyTypes(); - ksort($types); - foreach ($types as $key => $type) { - if ($type == 'K') { - $pkey[] = $key; - $pval[] = $this->$key; - } else { - $c->set($this->cacheKey($this->tableName(), $key, $this->$key), $this); - } - } - # XXX: should work for both compound and scalar pkeys - $pvals = implode(',', $pval); - $pkeys = implode(',', $pkey); - $c->set($this->cacheKey($this->tableName(), $pkeys, $pvals), $this); + } + + $keys = $this->_allCacheKeys(); + + foreach ($keys as $key) { + $c->set($key, $this); } } function decache() { $c = $this->memcache(); + if (!$c) { return false; - } else { - $pkey = array(); - $pval = array(); - $types = $this->keyTypes(); - ksort($types); - foreach ($types as $key => $type) { - if ($type == 'K') { - $pkey[] = $key; - $pval[] = $this->$key; - } else { - $c->delete($this->cacheKey($this->tableName(), $key, $this->$key)); + } + + $keys = $this->_allCacheKeys(); + + foreach ($keys as $key) { + $c->delete($key, $this); + } + } + + function _allCacheKeys() + { + $ckeys = array(); + + $types = $this->keyTypes(); + ksort($types); + + $pkey = array(); + $pval = array(); + + foreach ($types as $key => $type) { + + assert(!empty($key)); + + if ($type == 'U') { + if (empty($this->$key)) { + continue; } + $ckeys[] = $this->cacheKey($this->tableName(), $key, $this->$key); + } else if ($type == 'K' || $type == 'N') { + $pkey[] = $key; + $pval[] = $this->$key; + } else { + throw new Exception("Unknown key type $key => $type for " . $this->tableName()); } - # should work for both compound and scalar pkeys - # XXX: comma works for now but may not be safe separator for future keys - $pvals = implode(',', $pval); - $pkeys = implode(',', $pkey); - $c->delete($this->cacheKey($this->tableName(), $pkeys, $pvals)); } + + assert(count($pkey) > 0); + + // XXX: should work for both compound and scalar pkeys + $pvals = implode(',', $pval); + $pkeys = implode(',', $pkey); + + $ckeys[] = $this->cacheKey($this->tableName(), $pkeys, $pvals); + + return $ckeys; } function multicache($cls, $kv) -- cgit v1.2.3-54-g00ecf From bcddcb38cebc5ffbffd3df61fb9acaf6f674b133 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 09:09:59 -1000 Subject: make compression threshold and min savings config attrs for MemcachePlugin --- plugins/MemcachePlugin.php | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/plugins/MemcachePlugin.php b/plugins/MemcachePlugin.php index 78e2b2406..998766313 100644 --- a/plugins/MemcachePlugin.php +++ b/plugins/MemcachePlugin.php @@ -54,6 +54,9 @@ class MemcachePlugin extends Plugin private $_conn = null; public $servers = array('127.0.0.1;11211'); + public $compressThreshold = 20480; + public $compressMinSaving = 0.2; + /** * Initialize the plugin * @@ -156,11 +159,16 @@ class MemcachePlugin extends Plugin } $this->_conn->addServer($host, $port); } - //Compress items stored in the cache if they're over 2k in size - //and the compression would save more than 20%. - //Allows the cache to store objects larger than 1MB (if they - //compress to less than 1MB), and improves cache memory efficiency. - $this->_conn->setCompressThreshold(20000, 0.2); + + // Compress items stored in the cache if they're over threshold in size + // (default 2KiB) and the compression would save more than min savings + // ratio (default 0.2). + + // Allows the cache to store objects larger than 1MB (if they + // compress to less than 1MB), and improves cache memory efficiency. + + $this->_conn->setCompressThreshold($this->compressThreshold, + $this->compressMinSaving); } } } -- cgit v1.2.3-54-g00ecf From c0e4d7bfa2181090f05d3b42ed9f9b135b43aea1 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 4 Jan 2010 14:43:05 -0500 Subject: Add 'takeOverLogin' parameter for a real SSO feel --- plugins/CasAuthentication/CasAuthenticationPlugin.php | 9 +++++++++ plugins/CasAuthentication/README | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/CasAuthentication/CasAuthenticationPlugin.php b/plugins/CasAuthentication/CasAuthenticationPlugin.php index 8b6ef5462..8f29c7d2a 100644 --- a/plugins/CasAuthentication/CasAuthenticationPlugin.php +++ b/plugins/CasAuthentication/CasAuthenticationPlugin.php @@ -40,6 +40,7 @@ class CasAuthenticationPlugin extends AuthenticationPlugin public $server; public $port = 443; public $path = ''; + public $takeOverLogin = false; function checkPassword($username, $password) { @@ -62,6 +63,14 @@ class CasAuthenticationPlugin extends AuthenticationPlugin } } + function onArgsInitialize(&$args) + { + if($this->takeOverLogin && $args['action'] == 'login') + { + $args['action'] = 'caslogin'; + } + } + function onStartInitializeRouter($m) { $m->connect('main/cas', array('action' => 'caslogin')); diff --git a/plugins/CasAuthentication/README b/plugins/CasAuthentication/README index 2ee54dc05..c17a28e54 100644 --- a/plugins/CasAuthentication/README +++ b/plugins/CasAuthentication/README @@ -21,6 +21,9 @@ password_changeable*: must be set to false. This plugin does not support changin server*: CAS server to authentication against port (443): Port the CAS server listens on. Almost always 443 path (): Path on the server to CAS. Usually blank. +takeOverLogin (false): Take over the main login action. If takeOverLogin is + set, anytime the standard username/password login form would be shown, + a CAS login will be done instead. * required default values are in (parenthesis) @@ -33,6 +36,7 @@ addPlugin('casAuthentication', array( 'autoregistration'=>true, 'server'=>'sso-cas.univ-rennes1.fr', 'port'=>443, - 'path'=>'' + 'path'=>'', + 'takeOverLogin'=>true )); -- cgit v1.2.3-54-g00ecf From 6911e1c7972c3adec53d0fe04ebdd7da0fbd8b12 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 11:55:27 -0800 Subject: Ticket 2141: bugs with weighted popularity lists across year boundary. Consolidated several separate implementations of the same weighting algorithm into common_sql_weight() and fixed some bugs... For MySQL, now using timestampdiff() instead of subtraction for the comparison, so we get sane results when the year doesn't match, and utc_timestamp() rather than now() so we don't get negative ages for recent items with local server timezone. Unknown whether the same problems affect PostgreSQL, but note that it lacks the timestampdiff() SQL function. --- actions/favorited.php | 8 ++------ actions/publictagcloud.php | 11 ++++------- lib/grouptagcloudsection.php | 7 +------ lib/personaltagcloudsection.php | 11 +++-------- lib/popularnoticesection.php | 6 +++--- lib/util.php | 20 ++++++++++++++++++++ 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/actions/favorited.php b/actions/favorited.php index 150b67b0b..9ffa5b844 100644 --- a/actions/favorited.php +++ b/actions/favorited.php @@ -185,11 +185,7 @@ class FavoritedAction extends Action function showContent() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - fave.modified)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - fave.modified) / %s))'; - } + $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); $qry = 'SELECT notice.*, '. $weightexpr . ' as weight ' . @@ -207,7 +203,7 @@ class FavoritedAction extends Action } $notice = Memcached_DataObject::cachedQuery('Notice', - sprintf($qry, common_config('popular', 'dropoff')), + $qry, 600); $nl = new NoticeList($notice, $this); diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index 5c7074029..b5b474f13 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -105,12 +105,8 @@ class PublictagcloudAction extends Action #Add the aggregated columns... $tags->selectAdd('max(notice_id) as last_notice_id'); - if(common_config('db','type')=='pgsql') { - $calc='sum(exp(-extract(epoch from (now()-created))/%s)) as weight'; - } else { - $calc='sum(exp(-(now() - created)/%s)) as weight'; - } - $tags->selectAdd(sprintf($calc, common_config('tag', 'dropoff'))); + $calc = common_sql_weight('created', common_config('tag', 'dropoff')); + $tags->selectAdd($calc . ' as weight'); $tags->groupBy('tag'); $tags->orderBy('weight DESC'); @@ -136,10 +132,11 @@ class PublictagcloudAction extends Action $this->elementStart('dd'); $this->elementStart('ul', 'tags xoxo tag-cloud'); foreach ($tw as $tag => $weight) { + common_log(LOG_DEBUG, "$weight/$sum"); if ($sum) { $weightedSum = $weight/$sum; } else { - $weightedSum = 1; + $weightedSum = 0.5; } $this->showTag($tag, $weight, $weightedSum); } diff --git a/lib/grouptagcloudsection.php b/lib/grouptagcloudsection.php index 091cf4845..14ceda085 100644 --- a/lib/grouptagcloudsection.php +++ b/lib/grouptagcloudsection.php @@ -58,11 +58,7 @@ class GroupTagCloudSection extends TagCloudSection function getTags() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - notice_tag.created)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - notice_tag.created) / %s))'; - } + $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); $names = $this->group->getAliases(); @@ -99,7 +95,6 @@ class GroupTagCloudSection extends TagCloudSection $tag = Memcached_DataObject::cachedQuery('Notice_tag', sprintf($qry, - common_config('tag', 'dropoff'), $this->group->id, $namestring), 3600); diff --git a/lib/personaltagcloudsection.php b/lib/personaltagcloudsection.php index 0b29d58ca..091425f92 100644 --- a/lib/personaltagcloudsection.php +++ b/lib/personaltagcloudsection.php @@ -58,13 +58,9 @@ class PersonalTagCloudSection extends TagCloudSection function getTags() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - notice_tag.created)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - notice_tag.created) / %s))'; - } - - $qry = 'SELECT notice_tag.tag, '. + $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); + + $qry = 'SELECT notice_tag.tag, '. $weightexpr . ' as weight ' . 'FROM notice_tag JOIN notice ' . 'ON notice_tag.notice_id = notice.id ' . @@ -83,7 +79,6 @@ class PersonalTagCloudSection extends TagCloudSection $tag = Memcached_DataObject::cachedQuery('Notice_tag', sprintf($qry, - common_config('tag', 'dropoff'), $this->user->id), 3600); return $tag; diff --git a/lib/popularnoticesection.php b/lib/popularnoticesection.php index 9fbc9d2dd..fbf9a60ab 100644 --- a/lib/popularnoticesection.php +++ b/lib/popularnoticesection.php @@ -48,17 +48,17 @@ class PopularNoticeSection extends NoticeSection { function getNotices() { + // @fixme there should be a common func for this if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - fave.modified)) / %s))'; if (!empty($this->out->tag)) { $tag = pg_escape_string($this->out->tag); } } else { - $weightexpr='sum(exp(-(now() - fave.modified) / %s))'; if (!empty($this->out->tag)) { $tag = mysql_escape_string($this->out->tag); } } + $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); $qry = "SELECT notice.*, $weightexpr as weight "; if(isset($tag)) { $qry .= 'FROM notice_tag, notice JOIN fave ON notice.id = fave.notice_id ' . @@ -78,7 +78,7 @@ class PopularNoticeSection extends NoticeSection $qry .= ' LIMIT ' . $limit . ' OFFSET ' . $offset; $notice = Memcached_DataObject::cachedQuery('Notice', - sprintf($qry, common_config('popular', 'dropoff')), + $qry, 1200); return $notice; } diff --git a/lib/util.php b/lib/util.php index 63656b604..50bd0e2ac 100644 --- a/lib/util.php +++ b/lib/util.php @@ -908,6 +908,26 @@ function common_sql_date($datetime) return strftime('%Y-%m-%d %H:%M:%S', $datetime); } +/** + * Return an SQL fragment to calculate an age-based weight from a given + * timestamp or datetime column. + * + * @param string $column name of field we're comparing against current time + * @param integer $dropoff divisor for age in seconds before exponentiation + * @return string SQL fragment + */ +function common_sql_weight($column, $dropoff) +{ + if (common_config('db', 'type') == 'pgsql') { + // PostgreSQL doesn't support timestampdiff function. + // @fixme will this use the right time zone? + // @fixme does this handle cross-year subtraction correctly? + return "sum(exp(-extract(epoch from (now() - $column)) / $dropoff))"; + } else { + return "sum(exp(timestampdiff(second, utc_timestamp(), $column) / $dropoff))"; + } +} + function common_redirect($url, $code=307) { static $status = array(301 => "Moved Permanently", -- cgit v1.2.3-54-g00ecf From 928b5f8f2be554c37b0a435e76e88e92260e667b Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 09:57:48 -1000 Subject: Differentiate between empty values and cache misses in CacheLogPlugin --- plugins/CacheLogPlugin.php | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/plugins/CacheLogPlugin.php b/plugins/CacheLogPlugin.php index 9eb04641e..f1e5dd83a 100644 --- a/plugins/CacheLogPlugin.php +++ b/plugins/CacheLogPlugin.php @@ -61,7 +61,7 @@ class CacheLogPlugin extends Plugin function onEndCacheGet($key, &$value) { - if (is_null($value)) { + if ($value === false) { $this->log(LOG_INFO, "Cache MISS for key '$key'"); } else { $this->log(LOG_INFO, "Cache HIT for key '$key'"); @@ -71,7 +71,21 @@ class CacheLogPlugin extends Plugin function onStartCacheSet(&$key, &$value, &$flag, &$expiry, &$success) { - $this->log(LOG_INFO, "Setting cache value for key '$key'"); + if (empty($value)) { + if (is_array($value)) { + $this->log(LOG_INFO, "Setting empty array for key '$key'"); + } else if (is_null($value)) { + $this->log(LOG_INFO, "Setting null value for key '$key'"); + } else if (is_string($value)) { + $this->log(LOG_INFO, "Setting empty string for key '$key'"); + } else if (is_integer($value)) { + $this->log(LOG_INFO, "Setting integer 0 for key '$key'"); + } else { + $this->log(LOG_INFO, "Setting empty value '$value' for key '$key'"); + } + } else { + $this->log(LOG_INFO, "Setting non-empty value for key '$key'"); + } return true; } -- cgit v1.2.3-54-g00ecf From a1821ec8afd40c3ab88546a8fa2e6fc03a19ad25 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 09:59:47 -1000 Subject: default value for cache::get() changed from null to false --- lib/cache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cache.php b/lib/cache.php index 253839fb1..bac3499e5 100644 --- a/lib/cache.php +++ b/lib/cache.php @@ -116,7 +116,7 @@ class Cache function get($key) { - $value = null; + $value = false; if (Event::handle('StartCacheGet', array(&$key, &$value))) { if (array_key_exists($key, $this->_items)) { -- cgit v1.2.3-54-g00ecf From abc9b332412921e7ae7f730b23e186bcba412486 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:00:17 -1000 Subject: Memcached_DataObject stores empty values in the cache There's great value in knowing that something doesn't exist. We now cache this information, and carefully compare the results from cache as $results !== false instead of !empty($results), since some empty values (null, 0, empty array, empty string) are stored in the cache. Caching staticGet() and pkeyGet() now store DB misses in the cache, and cachedQuery() checks for empty results from the cache. --- classes/Memcached_DataObject.php | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 020d813b2..a77e43d38 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -19,8 +19,6 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } -require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; - class Memcached_DataObject extends DB_DataObject { /** @@ -57,7 +55,7 @@ class Memcached_DataObject extends DB_DataObject unset($i); } $i = Memcached_DataObject::getcached($cls, $k, $v); - if ($i) { + if ($i !== false) { // false == cache miss return $i; } else { $i = DB_DataObject::factory($cls); @@ -69,6 +67,12 @@ class Memcached_DataObject extends DB_DataObject $i->encache(); return $i; } else { + // save the fact that no such row exists + $c = self::memcache(); + if (!empty($c)) { + $ck = self::cachekey($cls, $k, $v); + $c->set($ck, null); + } return false; } } @@ -77,10 +81,13 @@ class Memcached_DataObject extends DB_DataObject function &pkeyGet($cls, $kv) { $i = Memcached_DataObject::multicache($cls, $kv); - if ($i) { + if ($i !== false) { // false == cache miss return $i; } else { - $i = new $cls(); + $i = DB_DataObject::factory($cls); + if (empty($i)) { + return false; + } foreach ($kv as $k => $v) { $i->$k = $v; } @@ -88,6 +95,11 @@ class Memcached_DataObject extends DB_DataObject $i->encache(); } else { $i = null; + $c = self::memcache(); + if (!empty($c)) { + $ck = self::multicacheKey($cls, $kv); + $c->set($ck, null); + } } return $i; } @@ -220,16 +232,22 @@ class Memcached_DataObject extends DB_DataObject function multicache($cls, $kv) { ksort($kv); - $c = Memcached_DataObject::memcache(); + $c = self::memcache(); if (!$c) { return false; } else { - $pkeys = implode(',', array_keys($kv)); - $pvals = implode(',', array_values($kv)); - return $c->get(Memcached_DataObject::cacheKey($cls, $pkeys, $pvals)); + return $c->get(self::multicacheKey($cls, $kv)); } } + static function multicacheKey($cls, $kv) + { + ksort($kv); + $pkeys = implode(',', array_keys($kv)); + $pvals = implode(',', array_values($kv)); + return self::cacheKey($cls, $pkeys, $pvals); + } + function getSearchEngine($table) { require_once INSTALLDIR.'/lib/search_engines.php'; @@ -264,7 +282,8 @@ class Memcached_DataObject extends DB_DataObject $key_part = common_keyize($cls).':'.md5($qry); $ckey = common_cache_key($key_part); $stored = $c->get($ckey); - if ($stored) { + + if ($stored !== false) { return new ArrayWrapper($stored); } -- cgit v1.2.3-54-g00ecf From 254ea279d8c86ed25ed74bff036b16e6293571f7 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:02:59 -1000 Subject: carefully compare cached settings against false for Config --- classes/Config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/Config.php b/classes/Config.php index 390d75381..6d914ca1f 100644 --- a/classes/Config.php +++ b/classes/Config.php @@ -59,7 +59,7 @@ class Config extends Memcached_DataObject if (!empty($c)) { $settings = $c->get(common_cache_key(self::settingsKey)); - if (!empty($settings)) { + if ($settings !== false) { return $settings; } } -- cgit v1.2.3-54-g00ecf From 06b6a27d7d31cb0680dffff70c498e27ece56762 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:03:57 -1000 Subject: cached id streams can be empty, compare against false --- classes/Notice.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 3e55bd6fa..e8bc509a6 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -1207,7 +1207,7 @@ class Notice extends Memcached_DataObject $idstr = $cache->get($idkey); - if (!empty($idstr)) { + if ($idstr !== false) { // Cache hit! Woohoo! $window = explode(',', $idstr); $ids = array_slice($window, $offset, $limit); @@ -1216,7 +1216,7 @@ class Notice extends Memcached_DataObject $laststr = $cache->get($idkey.';last'); - if (!empty($laststr)) { + if ($laststr !== false) { $window = explode(',', $laststr); $last_id = $window[0]; $new_ids = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, @@ -1376,7 +1376,7 @@ class Notice extends Memcached_DataObject $ids = $this->_repeatStreamDirect($limit); } else { $idstr = $cache->get(common_cache_key('notice:repeats:'.$this->id)); - if (!empty($idstr)) { + if ($idstr !== false) { $ids = explode(',', $idstr); } else { $ids = $this->_repeatStreamDirect(100); -- cgit v1.2.3-54-g00ecf From 96480aa6c1abda95db272f7c0a2b0e96f17acc70 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:12:19 -1000 Subject: XCachePlugin returns false value for cache miss --- plugins/XCachePlugin.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/XCachePlugin.php b/plugins/XCachePlugin.php index 8eed12cbc..03cb0c06e 100644 --- a/plugins/XCachePlugin.php +++ b/plugins/XCachePlugin.php @@ -63,8 +63,10 @@ class XCachePlugin extends Plugin function onStartCacheGet(&$key, &$value) { - $value = xcache_get($key); - if (!is_null($value)) { + if (!xcache_isset($key)) { + $value = false; + } else { + $value = xcache_get($key); $value = unserialize($value); } Event::handle('EndCacheGet', array($key, &$value)); -- cgit v1.2.3-54-g00ecf From e668709ce48f366b4477bc62434db28b3e211a33 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 13:02:41 -0800 Subject: drop debug statement --- actions/publictagcloud.php | 1 - 1 file changed, 1 deletion(-) diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index b5b474f13..9e4478dbb 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -132,7 +132,6 @@ class PublictagcloudAction extends Action $this->elementStart('dd'); $this->elementStart('ul', 'tags xoxo tag-cloud'); foreach ($tw as $tag => $weight) { - common_log(LOG_DEBUG, "$weight/$sum"); if ($sum) { $weightedSum = $weight/$sum; } else { -- cgit v1.2.3-54-g00ecf From 440b9957f9ae6fdfdb44c39074803fcdbc64112d Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 14:24:16 -0800 Subject: Exclude process-specific link & result cache references from serialized Memcached_Data_Object instances. Should fix seemingly-random bugs due to destructor free()ing local resources by mistake. --- classes/Memcached_DataObject.php | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index a77e43d38..d830884b6 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -21,7 +21,7 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } class Memcached_DataObject extends DB_DataObject { - /** + /** * Destructor to free global memory resources associated with * this data object when it's unset or goes out of scope. * DB_DataObject doesn't do this yet by itself. @@ -35,6 +35,42 @@ class Memcached_DataObject extends DB_DataObject } } + /** + * Magic function called at serialize() time. + * + * We use this to drop a couple process-specific references + * from DB_DataObject which can cause trouble in future + * processes. + * + * @return array of variable names to include in serialization. + */ + function __sleep() + { + $vars = array_keys(get_object_vars($this)); + $skip = array('_DB_resultid', '_link_loaded'); + return array_diff($vars, $skip); + } + + /** + * Magic function called at unserialize() time. + * + * Clean out some process-specific variables which might + * be floating around from a previous process's cached + * objects. + * + * Old cached objects may still have them. + */ + function __wakeup() + { + // Refers to global state info from a previous process. + // Clear this out so we don't accidentally break global + // state in *this* process. + $this->_DB_resultid = null; + + // We don't have any local DBO refs, so clear these out. + $this->_link_loaded = false; + } + /** * Wrapper for DB_DataObject's static lookup using memcached * as backing instead of an in-process cache array. -- cgit v1.2.3-54-g00ecf