From ce0e6cb50d88c593db62edd8375c4414e8a8ebf8 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 22 Mar 2010 00:25:49 -0400 Subject: user sitemap --- plugins/Sitemap/SitemapPlugin.php | 111 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 plugins/Sitemap/SitemapPlugin.php (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php new file mode 100644 index 000000000..42ea1dbe6 --- /dev/null +++ b/plugins/Sitemap/SitemapPlugin.php @@ -0,0 +1,111 @@ +. + * + * @category Sample + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0 + * @link http://status.net/ + */ + +if (!defined('STATUSNET')) { + // This check helps protect against security problems; + // your code file can't be executed directly from the web. + exit(1); +} + +/** + * Sitemap plugin + * + * @category Sample + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0 + * @link http://status.net/ + */ + +class SitemapPlugin extends Plugin +{ + /** + * Load related modules when needed + * + * Most non-trivial plugins will require extra modules to do their work. Typically + * these include data classes, action classes, widget classes, or external libraries. + * + * This method receives a class name and loads the PHP file related to that class. By + * tradition, action classes typically have files named for the action, all lower-case. + * Data classes are in files with the data class name, initial letter capitalized. + * + * Note that this method will be called for *all* overloaded classes, not just ones + * in this plugin! So, make sure to return true by default to let other plugins, and + * the core code, get a chance. + * + * @param string $cls Name of the class to be loaded + * + * @return boolean hook value; true means continue processing, false means stop. + */ + + function onAutoload($cls) + { + $dir = dirname(__FILE__); + + switch ($cls) + { + case 'SitemapindexAction': + case 'NoticesitemapAction': + case 'UsersitemapAction': + require_once $dir . '/' . strtolower(mb_substr($cls, 0, -6)) . '.php'; + return false; + case 'SitemapAction': + require_once $dir . '/' . strtolower($cls) . '.php'; + default: + return true; + } + } + + /** + * Map URLs to actions + * + * @param Net_URL_Mapper $m path-to-action mapper + * + * @return boolean hook value; true means continue processing, false means stop. + */ + + function onRouterInitialized($m) + { + $m->connect('sitemapindex.xml', + array('action' => 'sitemapindex')); + $m->connect('/sitemaps/notice/:year/:month/:day.xml', + array('action' => 'noticesitemap'), + array('year' => '[0-9]{4}', + 'month' => '[1]?[0-9]', + 'day' => '[123]?[0-9]')); + $m->connect('/sitemaps/user/:index.xml', + array('action' => 'usersitemap'), + array('index' => '[0-9]+', + 'month' => '[1]?[0-9]', + 'day' => '[123]?[0-9]')); + return true; + } +} -- cgit v1.2.3-54-g00ecf From 524cd9df936bd27d5b64fbf08b219b56cb071122 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 22 Mar 2010 08:09:15 -0400 Subject: Add a Notice sitemap --- plugins/Sitemap/SitemapPlugin.php | 6 ++- plugins/Sitemap/noticesitemap.php | 94 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 plugins/Sitemap/noticesitemap.php (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 42ea1dbe6..8889c8930 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -96,11 +96,13 @@ class SitemapPlugin extends Plugin { $m->connect('sitemapindex.xml', array('action' => 'sitemapindex')); - $m->connect('/sitemaps/notice/:year/:month/:day.xml', + $m->connect('/sitemaps/notice/:year/:month/:day/:index.xml', array('action' => 'noticesitemap'), array('year' => '[0-9]{4}', 'month' => '[1]?[0-9]', - 'day' => '[123]?[0-9]')); + 'day' => '[123]?[0-9]', + 'index' => '[0-9]+')); + $m->connect('/sitemaps/user/:index.xml', array('action' => 'usersitemap'), array('index' => '[0-9]+', diff --git a/plugins/Sitemap/noticesitemap.php b/plugins/Sitemap/noticesitemap.php new file mode 100644 index 000000000..7eec88636 --- /dev/null +++ b/plugins/Sitemap/noticesitemap.php @@ -0,0 +1,94 @@ +. + * + * @category Sitemap + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +/** + * sitemap for users + * + * @category Sitemap + * @package StatusNet + * @author Evan Prodromou + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +class NoticesitemapAction extends SitemapAction +{ + const NOTICES_PER_MAP = 25000; + + var $notice = null; + + function prepare($args) + { + parent::prepare($args); + + $y = $this->trimmed('year'); + + $m = $this->trimmed('month'); + $d = $this->trimmed('day'); + + $i = $this->trimmed('index'); + + $y += 0; + $m += 0; + $d += 0; + $i += 0; + + $offset = ($i-1) * self::NOTICES_PER_MAP; + $limit = self::NOTICES_PER_MAP; + + $this->notice = new Notice(); + + $this->notice->whereAdd("created > '$y-$m-$d 00:00:00'"); + $this->notice->whereAdd("created <= '$y-$m-$d 11:59:59'"); + $this->notice->whereAdd('is_local = 1'); + + $this->notice->orderBy('id'); + $this->notice->limit($offset, $limit); + + $this->notice->find(); + + return true; + } + + function nextUrl() + { + if ($this->notice->fetch()) { + return array(common_local_url('shownotice', array('notice' => $this->notice->id)), + common_date_w3dtf($this->notice->created), + null, + null); + } else { + return null; + } + } +} -- cgit v1.2.3-54-g00ecf From 0a04f9d49db4f28ace3f3d94ec0763a926296b44 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 25 Mar 2010 23:56:09 -0400 Subject: better query for notices by date --- plugins/Sitemap/SitemapPlugin.php | 17 +++-------------- plugins/Sitemap/noticesitemap.php | 9 ++++++--- 2 files changed, 9 insertions(+), 17 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 8889c8930..bb404cd25 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -50,17 +50,6 @@ class SitemapPlugin extends Plugin /** * Load related modules when needed * - * Most non-trivial plugins will require extra modules to do their work. Typically - * these include data classes, action classes, widget classes, or external libraries. - * - * This method receives a class name and loads the PHP file related to that class. By - * tradition, action classes typically have files named for the action, all lower-case. - * Data classes are in files with the data class name, initial letter capitalized. - * - * Note that this method will be called for *all* overloaded classes, not just ones - * in this plugin! So, make sure to return true by default to let other plugins, and - * the core code, get a chance. - * * @param string $cls Name of the class to be loaded * * @return boolean hook value; true means continue processing, false means stop. @@ -99,9 +88,9 @@ class SitemapPlugin extends Plugin $m->connect('/sitemaps/notice/:year/:month/:day/:index.xml', array('action' => 'noticesitemap'), array('year' => '[0-9]{4}', - 'month' => '[1]?[0-9]', - 'day' => '[123]?[0-9]', - 'index' => '[0-9]+')); + 'month' => '[01][0-9]', + 'day' => '[0123][0-9]', + 'index' => '[1-9][0-9]*')); $m->connect('/sitemaps/user/:index.xml', array('action' => 'usersitemap'), diff --git a/plugins/Sitemap/noticesitemap.php b/plugins/Sitemap/noticesitemap.php index 7eec88636..002408486 100644 --- a/plugins/Sitemap/noticesitemap.php +++ b/plugins/Sitemap/noticesitemap.php @@ -68,9 +68,12 @@ class NoticesitemapAction extends SitemapAction $this->notice = new Notice(); - $this->notice->whereAdd("created > '$y-$m-$d 00:00:00'"); - $this->notice->whereAdd("created <= '$y-$m-$d 11:59:59'"); - $this->notice->whereAdd('is_local = 1'); + $dt = sprintf('%04d-%02d-%02d', $y, $m, $d); + + $this->notice->whereAdd("created > '$dt 00:00:00'"); + $this->notice->whereAdd("created <= '$dt 23:59:59'"); + + $this->notice->whereAdd('is_local != 0'); $this->notice->orderBy('id'); $this->notice->limit($offset, $limit); -- cgit v1.2.3-54-g00ecf From 9e592baa39dfab0c1e09c4356fb8d434379797c3 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sat, 10 Apr 2010 10:03:57 -0400 Subject: move USERS_PER_MAP to plugin --- plugins/Sitemap/SitemapPlugin.php | 3 +++ plugins/Sitemap/usersitemap.php | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index bb404cd25..40263aaee 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -47,6 +47,8 @@ if (!defined('STATUSNET')) { class SitemapPlugin extends Plugin { + const USERS_PER_MAP = 25000; + /** * Load related modules when needed * @@ -85,6 +87,7 @@ class SitemapPlugin extends Plugin { $m->connect('sitemapindex.xml', array('action' => 'sitemapindex')); + $m->connect('/sitemaps/notice/:year/:month/:day/:index.xml', array('action' => 'noticesitemap'), array('year' => '[0-9]{4}', diff --git a/plugins/Sitemap/usersitemap.php b/plugins/Sitemap/usersitemap.php index 582a13b66..b7cc939a9 100644 --- a/plugins/Sitemap/usersitemap.php +++ b/plugins/Sitemap/usersitemap.php @@ -43,8 +43,6 @@ if (!defined('STATUSNET')) { class UsersitemapAction extends SitemapAction { - const USERS_PER_MAP = 25000; - var $user = null; function prepare($args) @@ -55,8 +53,8 @@ class UsersitemapAction extends SitemapAction $i += 0; - $offset = ($i-1) * self::USERS_PER_MAP; - $limit = self::USERS_PER_MAP; + $offset = ($i-1) * SitemapPlugin::USERS_PER_MAP; + $limit = SitemapPlugin::USERS_PER_MAP; $this->user = new User(); -- cgit v1.2.3-54-g00ecf From 63c4eef64322da6a360c9ef3d7e1a20de9ca9cdd Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sat, 10 Apr 2010 10:21:19 -0400 Subject: change URLs for user sitemap --- plugins/Sitemap/SitemapPlugin.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 40263aaee..5b2af4879 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -95,11 +95,12 @@ class SitemapPlugin extends Plugin 'day' => '[0123][0-9]', 'index' => '[1-9][0-9]*')); - $m->connect('/sitemaps/user/:index.xml', + $m->connect('/sitemaps/user/:year/:month/:day/:index.xml', array('action' => 'usersitemap'), - array('index' => '[0-9]+', - 'month' => '[1]?[0-9]', - 'day' => '[123]?[0-9]')); + array('year' => '[0-9]{4}', + 'month' => '[01][0-9]', + 'day' => '[0123][0-9]', + 'index' => '[1-9][0-9]*')); return true; } } -- cgit v1.2.3-54-g00ecf From b73c8ff441008a4fa09eef66871f2cfebc0569b5 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 12 Apr 2010 10:23:32 -0400 Subject: Move NOTICES_PER_MAP to SitemapPlugin --- plugins/Sitemap/SitemapPlugin.php | 3 ++- plugins/Sitemap/noticesitemap.php | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 5b2af4879..fa9c9a76d 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -47,7 +47,8 @@ if (!defined('STATUSNET')) { class SitemapPlugin extends Plugin { - const USERS_PER_MAP = 25000; + const USERS_PER_MAP = 25000; + const NOTICES_PER_MAP = 25000; /** * Load related modules when needed diff --git a/plugins/Sitemap/noticesitemap.php b/plugins/Sitemap/noticesitemap.php index 12a22dbb2..c8db24efe 100644 --- a/plugins/Sitemap/noticesitemap.php +++ b/plugins/Sitemap/noticesitemap.php @@ -43,8 +43,6 @@ if (!defined('STATUSNET')) { class NoticesitemapAction extends SitemapAction { - const NOTICES_PER_MAP = 25000; - var $notice = null; function prepare($args) @@ -63,8 +61,8 @@ class NoticesitemapAction extends SitemapAction $d += 0; $i += 0; - $offset = ($i-1) * self::NOTICES_PER_MAP; - $limit = self::NOTICES_PER_MAP; + $offset = ($i-1) * SitemapPlugin::NOTICES_PER_MAP; + $limit = SitemapPlugin::NOTICES_PER_MAP; $this->notice = new Notice(); -- cgit v1.2.3-54-g00ecf From 8b9ce731f4707f6939497d139521acee56596dea Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 12 Apr 2010 11:05:19 -0400 Subject: max users, notices per sitemap = 50K --- plugins/Sitemap/SitemapPlugin.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index fa9c9a76d..29c32a624 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -47,8 +47,8 @@ if (!defined('STATUSNET')) { class SitemapPlugin extends Plugin { - const USERS_PER_MAP = 25000; - const NOTICES_PER_MAP = 25000; + const USERS_PER_MAP = 50000; + const NOTICES_PER_MAP = 50000; /** * Load related modules when needed -- cgit v1.2.3-54-g00ecf From 5ff9c0242b2ad1df22af5630a7ebfdcce8177212 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 12 Apr 2010 12:06:08 -0400 Subject: make sure notice and user sitemap are 'in' top level directory --- plugins/Sitemap/SitemapPlugin.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 29c32a624..ed876d94f 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -89,14 +89,14 @@ class SitemapPlugin extends Plugin $m->connect('sitemapindex.xml', array('action' => 'sitemapindex')); - $m->connect('/sitemaps/notice/:year/:month/:day/:index.xml', + $m->connect('/notice-sitemap-:year-:month-:day-:index.xml', array('action' => 'noticesitemap'), array('year' => '[0-9]{4}', 'month' => '[01][0-9]', 'day' => '[0123][0-9]', 'index' => '[1-9][0-9]*')); - $m->connect('/sitemaps/user/:year/:month/:day/:index.xml', + $m->connect('/user-sitemap-:year-:month-:day-:index.xml', array('action' => 'usersitemap'), array('year' => '[0-9]{4}', 'month' => '[01][0-9]', -- cgit v1.2.3-54-g00ecf From 1c858e17eaf0e9509b7e165ea1b0d45f7e923361 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 12 Apr 2010 14:32:01 -0400 Subject: add sitemap statement to robots.txt --- plugins/Sitemap/SitemapPlugin.php | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index ed876d94f..6fc702104 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -76,6 +76,23 @@ class SitemapPlugin extends Plugin } } + /** + * Add sitemap-related information at the end of robots.txt + * + * @param Action $action Action being run + * + * @return boolean hook value. + */ + + function onEndRobotsTxt($action) + { + $url = common_local_url('sitemapindex'); + + print "\nSitemap: $url\n"; + + return true; + } + /** * Map URLs to actions * -- cgit v1.2.3-54-g00ecf From 7d85b79814e7e91a88f23d7c7e752a0bcfc83ff4 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 3 May 2010 14:00:12 -0400 Subject: Database tables to cache expensive query data We need to bundle counts of notices and users by date. This can be expensive for large sites. So, new tables are added to cache the results of these queries, which don't change after the date is over. --- plugins/Sitemap/SitemapPlugin.php | 36 +++++++++ plugins/Sitemap/Sitemap_notice_count.php | 125 +++++++++++++++++++++++++++++++ plugins/Sitemap/Sitemap_user_count.php | 121 ++++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 plugins/Sitemap/Sitemap_notice_count.php create mode 100644 plugins/Sitemap/Sitemap_user_count.php (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 6fc702104..831694efc 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -121,4 +121,40 @@ class SitemapPlugin extends Plugin 'index' => '[1-9][0-9]*')); return true; } + + /** + * Database schema setup + * + * We cache some data persistently to avoid overlong queries. + * + * @see Sitemap_user_count + * @see Sitemap_notice_count + * + * @return boolean hook value; true means continue processing, false means stop. + */ + + function onCheckSchema() + { + $schema = Schema::get(); + + // For storing user-submitted flags on profiles + + $schema->ensureTable('sitemap_user_count', + array(new ColumnDef('registration_date', 'date', null, + true, 'PRI'), + new ColumnDef('user_count', 'integer'), + new ColumnDef('created', 'datetime', + null, false), + new ColumnDef('modified', 'timestamp'))); + + $schema->ensureTable('sitemap_notice_count', + array(new ColumnDef('notice_date', 'date', null, + true, 'PRI'), + new ColumnDef('notice_count', 'integer'), + new ColumnDef('created', 'datetime', + null, false), + new ColumnDef('modified', 'timestamp'))); + + return true; + } } diff --git a/plugins/Sitemap/Sitemap_notice_count.php b/plugins/Sitemap/Sitemap_notice_count.php new file mode 100644 index 000000000..72bb2b9d4 --- /dev/null +++ b/plugins/Sitemap/Sitemap_notice_count.php @@ -0,0 +1,125 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2010, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +require_once INSTALLDIR . '/classes/Memcached_DataObject.php'; + +/** + * Data class for counting notices by date + * + * We make a separate sitemap for each notice posted by date. + * To save ourselves some (not inconsiderable) processing effort, + * we cache this data in the sitemap_notice_count table. Each + * row represents a day since the site has been started, with a count + * of notices posted on that day. Since, after the end of the day, + * this number doesn't change, it's a good candidate for persistent caching. + * + * @category Data + * @package StatusNet + * @author Evan Prodromou + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * @see DB_DataObject + */ + +class Sitemap_notice_count extends Memcached_DataObject +{ + public $__table = 'sitemap_notice_count'; // table name + + public $notice_date; // date primary_key not_null + public $notice_count; // int(4) + public $created; + public $modified; + + /** + * Get an instance by key + * + * This is a utility method to get a single instance with a given key value. + * + * @param string $k Key to use to lookup (usually 'notice_id' for this class) + * @param mixed $v Value to lookup + * + * @return Sitemap_notice_count object found, or null for no hits + * + */ + + function staticGet($k, $v=null) + { + return Memcached_DataObject::staticGet('Sitemap_notice_count', $k, $v); + } + + /** + * return table definition for DB_DataObject + * + * DB_DataObject needs to know something about the table to manipulate + * instances. This method provides all the DB_DataObject needs to know. + * + * @return array array of column definitions + */ + + function table() + { + return array('notice_date' => DB_DATAOBJECT_DATE + DB_DATAOBJECT_NOTNULL, + 'notice_count' => DB_DATAOBJECT_INT, + 'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL, + 'modified' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL); + } + + /** + * return key definitions for DB_DataObject + * + * DB_DataObject needs to know about keys that the table has; this function + * defines them. + * + * @return array key definitions + */ + + function keys() + { + return array('notice_date' => 'K'); + } + + /** + * return key definitions for Memcached_DataObject + * + * Our caching system uses the same key definitions, but uses a different + * method to get them. + * + * @return array key definitions + */ + + function keyTypes() + { + return $this->keys(); + } +} diff --git a/plugins/Sitemap/Sitemap_user_count.php b/plugins/Sitemap/Sitemap_user_count.php new file mode 100644 index 000000000..1a7a6577d --- /dev/null +++ b/plugins/Sitemap/Sitemap_user_count.php @@ -0,0 +1,121 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2010, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +require_once INSTALLDIR . '/classes/Memcached_DataObject.php'; + +/** + * Data class for counting users by date + * + * We make a separate sitemap for each user registered by date. + * To save ourselves some processing effort, we cache this data + * + * @category Action + * @package StatusNet + * @author Evan Prodromou + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * @see DB_DataObject + */ + +class Sitemap_user_count extends Memcached_DataObject +{ + public $__table = 'sitemap_user_count'; // table name + + public $registration_date; // date primary_key not_null + public $user_count; // int(4) + public $created; + public $modified; + + /** + * Get an instance by key + * + * This is a utility method to get a single instance with a given key value. + * + * @param string $k Key to use to lookup (usually 'user_id' for this class) + * @param mixed $v Value to lookup + * + * @return Sitemap_user_count object found, or null for no hits + * + */ + + function staticGet($k, $v=null) + { + return Memcached_DataObject::staticGet('Sitemap_user_count', $k, $v); + } + + /** + * return table definition for DB_DataObject + * + * DB_DataObject needs to know something about the table to manipulate + * instances. This method provides all the DB_DataObject needs to know. + * + * @return array array of column definitions + */ + + function table() + { + return array('registration_date' => DB_DATAOBJECT_DATE + DB_DATAOBJECT_NOTNULL, + 'user_count' => DB_DATAOBJECT_INT, + 'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL, + 'modified' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL); + } + + /** + * return key definitions for DB_DataObject + * + * DB_DataObject needs to know about keys that the table has; this function + * defines them. + * + * @return array key definitions + */ + + function keys() + { + return array('registration_date' => 'K'); + } + + /** + * return key definitions for Memcached_DataObject + * + * Our caching system uses the same key definitions, but uses a different + * method to get them. + * + * @return array key definitions + */ + + function keyTypes() + { + return $this->keys(); + } +} -- cgit v1.2.3-54-g00ecf From 416161c94366292a623aecf8fe79b0d73c337e98 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 21 May 2010 16:47:30 -0400 Subject: make user counts use the database table --- plugins/Sitemap/SitemapPlugin.php | 4 + plugins/Sitemap/Sitemap_user_count.php | 166 +++++++++++++++++++++++++++++++++ plugins/Sitemap/sitemapindex.php | 35 ++----- 3 files changed, 180 insertions(+), 25 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 831694efc..82c007d66 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -64,6 +64,9 @@ class SitemapPlugin extends Plugin switch ($cls) { + case 'Sitemap_user_count': + require_once $dir . '/' . $cls . '.php'; + return false; case 'SitemapindexAction': case 'NoticesitemapAction': case 'UsersitemapAction': @@ -71,6 +74,7 @@ class SitemapPlugin extends Plugin return false; case 'SitemapAction': require_once $dir . '/' . strtolower($cls) . '.php'; + return false; default: return true; } diff --git a/plugins/Sitemap/Sitemap_user_count.php b/plugins/Sitemap/Sitemap_user_count.php index 1a7a6577d..7743b0532 100644 --- a/plugins/Sitemap/Sitemap_user_count.php +++ b/plugins/Sitemap/Sitemap_user_count.php @@ -105,6 +105,11 @@ class Sitemap_user_count extends Memcached_DataObject return array('registration_date' => 'K'); } + function sequenceKey() + { + return array(false, false, false); + } + /** * return key definitions for Memcached_DataObject * @@ -118,4 +123,165 @@ class Sitemap_user_count extends Memcached_DataObject { return $this->keys(); } + + static function getAll() + { + $userCounts = self::cacheGet('sitemap:user:counts'); + + if ($userCounts === false) { + + $suc = new Sitemap_user_count(); + $suc->orderBy('registration_date DESC'); + + // Fetch the first one to check up-to-date-itude + + $n = $suc->find(true); + + $today = self::today(); + $userCounts = array(); + + if (!$n) { // No counts saved yet + $userCounts = self::initializeCounts(); + } else if ($suc->registration_date < $today) { // There are counts but not up to today + $userCounts = self::fillInCounts($suc->registration_date); + } else if ($suc->registration_date == $today) { // Refresh today's + $userCounts[$today] = self::updateToday(); + } + + // starts with second-to-last date + + while ($suc->fetch()) { + $userCounts[$suc->registration_date] = $suc->user_count; + } + + self::cacheSet('sitemap:user:counts', $userCounts); + } + + return $userCounts; + } + + static function initializeCounts() + { + $firstDate = self::getFirstDate(); // awww + $today = self::today(); + + $counts = array(); + + for ($d = $firstDate; $d <= $today; $d = self::incrementDay($d)) { + common_debug("Date = '$d'"); + $n = self::getCount($d); + self::insertCount($d, $n); + $counts[$d] = $n; + } + + return $counts; + } + + static function fillInCounts($lastDate) + { + $today = self::today(); + + $counts = array(); + + $n = self::getCount($lastDate); + self::updateCount($lastDate, $n); + + $counts[$lastDate] = $n; + + for ($d = self::incrementDay($lastDate); $d <= $today; $d = self::incrementDay($d)) { + $n = self::getCount($d); + self::insertCount($d, $n); + } + + return $counts; + } + + static function updateToday() + { + $today = self::today(); + + $n = self::getCount($today); + self::updateCount($today, $n); + + return $n; + } + + static function getCount($d) + { + $user = new User(); + $user->whereAdd('created BETWEEN "'.$d.' 00:00:00" AND "'.self::incrementDay($d).' 00:00:00"'); + $n = $user->count(); + + return $n; + } + + static function insertCount($d, $n) + { + common_debug("Inserting count '$n' for '$d'"); + + $suc = new Sitemap_user_count(); + + $suc->registration_date = DB_DataObject_Cast::date($d); + $suc->user_count = $n; + $suc->created = common_sql_now(); + $suc->modified = $suc->created; + + if (!$suc->insert()) { + common_log(LOG_WARNING, "Could not save user counts for '$d'"); + } + } + + static function updateCount($d, $n) + { + $suc = Sitemap_user_count::staticGet('registration_date', DB_DataObject_Cast::date($d)); + + if (empty($suc)) { + throw new Exception("No such registration date: $d"); + } + + $orig = clone($suc); + + $suc->registration_date = DB_DataObject_Cast::date($d); + $suc->user_count = $n; + $suc->created = common_sql_now(); + $suc->modified = $suc->created; + + if (!$suc->update($orig)) { + common_log(LOG_WARNING, "Could not save user counts for '$d'"); + } + } + + static function incrementDay($d) + { + $dt = self::dateStrToInt($d); + return self::dateIntToStr($dt + 24 * 60 * 60); + } + + static function dateStrToInt($d) + { + return strtotime($d.' 00:00:00'); + } + + static function dateIntToStr($dt) + { + return date('Y-m-d', $dt); + } + + static function getFirstDate() + { + $u = new User(); + $u->selectAdd(); + $u->selectAdd('date(min(created)) as first_date'); + if ($u->find(true)) { + return $u->first_date; + } else { + // Is this right? + return self::dateIntToStr(time()); + } + } + + static function today() + { + return self::dateIntToStr(time()); + } } diff --git a/plugins/Sitemap/sitemapindex.php b/plugins/Sitemap/sitemapindex.php index 2055dd7f0..a3328340f 100644 --- a/plugins/Sitemap/sitemapindex.php +++ b/plugins/Sitemap/sitemapindex.php @@ -68,11 +68,15 @@ class SitemapindexAction extends Action function showUserSitemaps() { - $userCounts = $this->getUserCounts(); + $userCounts = Sitemap_user_count::getAll(); foreach ($userCounts as $dt => $cnt) { $cnt = $cnt+0; - assert($cnt != 0); + + if ($cnt == 0) { + continue; + } + $n = (int)$cnt / (int)SitemapPlugin::USERS_PER_MAP; if (($cnt % SitemapPlugin::USERS_PER_MAP) != 0) { $n++; @@ -88,7 +92,9 @@ class SitemapindexAction extends Action $noticeCounts = $this->getNoticeCounts(); foreach ($noticeCounts as $dt => $cnt) { - assert($cnt != 0); + if ($cnt == 0) { + continue; + } $n = $cnt / SitemapPlugin::NOTICES_PER_MAP; if ($cnt % SitemapPlugin::NOTICES_PER_MAP) { $n++; @@ -101,28 +107,7 @@ class SitemapindexAction extends Action function getUserCounts() { - $userCounts = User::cacheGet('sitemap:user:counts'); - - if ($userCounts === false) { - - $user = new User(); - - $user->selectAdd(); - $user->selectAdd('date(created) as regdate, count(*) as regcount'); - $user->groupBy('regdate'); - - $user->find(); - - $userCounts = array(); - - while ($user->fetch()) { - $userCounts[$user->regdate] = $user->regcount; - } - - User::cacheSet('sitemap:user:counts', $userCounts); - } - - return $userCounts; + return Sitemap_user_count::getAll(); } function getNoticeCounts() -- cgit v1.2.3-54-g00ecf From 271d7dd8509537aee7e10fd6c8f493e62e6c75b2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 31 May 2010 07:48:14 -0700 Subject: load Sitemap_notice_count --- plugins/Sitemap/SitemapPlugin.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'plugins/Sitemap/SitemapPlugin.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 82c007d66..7ef5f1aa9 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -65,6 +65,7 @@ class SitemapPlugin extends Plugin switch ($cls) { case 'Sitemap_user_count': + case 'Sitemap_notice_count': require_once $dir . '/' . $cls . '.php'; return false; case 'SitemapindexAction': @@ -141,8 +142,6 @@ class SitemapPlugin extends Plugin { $schema = Schema::get(); - // For storing user-submitted flags on profiles - $schema->ensureTable('sitemap_user_count', array(new ColumnDef('registration_date', 'date', null, true, 'PRI'), -- cgit v1.2.3-54-g00ecf