From 35272f638c0f162f43c951e1ffcef55c8f54787e Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sat, 10 Apr 2010 10:24:58 -0400 Subject: Start of an action for sitemap index --- plugins/Sitemap/sitemapindex.php | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 plugins/Sitemap/sitemapindex.php (limited to 'plugins/Sitemap/sitemapindex.php') diff --git a/plugins/Sitemap/sitemapindex.php b/plugins/Sitemap/sitemapindex.php new file mode 100644 index 000000000..09aebe0d8 --- /dev/null +++ b/plugins/Sitemap/sitemapindex.php @@ -0,0 +1,75 @@ +. + * + * @category Sitemap + * @package StatusNet + * @author Evan Prodromou + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +/** + * Show the sitemap index + * + * @category Sitemap + * @package StatusNet + * @author Evan Prodromou + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +class SitemapAction extends Action +{ + /** + * handle the action + * + * @param array $args unused. + * + * @return void + */ + + function handle($args) + { + header('Content-Type: text/xml; charset=UTF-8'); + $this->startXML(); + + $this->elementStart('sitemapindex', array('xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9')); + + $this->showUserSitemaps(); + $this->showNoticeSitemaps(); + + $this->elementEnd('sitemapindex'); + + $this->endXML(); + } + + function showUserSitemaps() + { + $user = new User(); + $cnt = $user->count(); + + } +} -- cgit v1.2.3-54-g00ecf From 4b321f96fc9b45ae3000088b8cfd856f9ffe1529 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 12 Apr 2010 11:04:56 -0400 Subject: show sitemapindex with user and notice sitemaps --- plugins/Sitemap/sitemapindex.php | 93 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) (limited to 'plugins/Sitemap/sitemapindex.php') diff --git a/plugins/Sitemap/sitemapindex.php b/plugins/Sitemap/sitemapindex.php index 09aebe0d8..7942bc3bd 100644 --- a/plugins/Sitemap/sitemapindex.php +++ b/plugins/Sitemap/sitemapindex.php @@ -41,7 +41,7 @@ if (!defined('STATUSNET')) { * @link http://status.net/ */ -class SitemapAction extends Action +class SitemapindexAction extends Action { /** * handle the action @@ -68,8 +68,97 @@ class SitemapAction extends Action function showUserSitemaps() { + $userCounts = $this->getUserCounts(); + + foreach ($userCounts as $dt => $cnt) { + $cnt = $cnt+0; + assert($cnt != 0); + $n = (int)$cnt / (int)SitemapPlugin::USERS_PER_MAP; + if (($cnt % SitemapPlugin::USERS_PER_MAP) != 0) { + $n++; + } + for ($i = 1; $i <= $n; $i++) { + $this->showSitemap('user', $dt, $i); + } + } + } + + function showNoticeSitemaps() + { + $noticeCounts = $this->getNoticeCounts(); + + foreach ($noticeCounts as $dt => $cnt) { + assert($cnt != 0); + $n = $cnt / SitemapPlugin::NOTICES_PER_MAP; + if ($cnt % SitemapPlugin::NOTICES_PER_MAP) { + $n++; + } + for ($i = 1; $i <= $n; $i++) { + $this->showSitemap('notice', $dt, $i); + } + } + } + + function getUserCounts() + { + // XXX: cachemeplease + $user = new User(); - $cnt = $user->count(); + $user->selectAdd(); + $user->selectAdd('date(created) as regdate, count(*) as regcount'); + $user->groupBy('regdate'); + + $user->find(); + + $userCounts = array(); + + while ($user->fetch()) { + $userCounts[$user->regdate] = $user->regcount; + } + + return $userCounts; + } + + function getNoticeCounts() + { + // XXX: cachemeplease + + $notice = new Notice(); + + $notice->selectAdd(); + $notice->selectAdd('date(created) as postdate, count(*) as postcount'); + $notice->groupBy('postdate'); + + $notice->find(); + + $noticeCounts = array(); + + while ($notice->fetch()) { + $noticeCounts[$notice->postdate] = $notice->postcount; + } + + return $noticeCounts; + } + + function showSitemap($prefix, $dt, $i) + { + list($y, $m, $d) = explode('-', $dt); + + $this->elementStart('sitemap'); + $this->element('loc', null, common_local_url($prefix.'sitemap', + array('year' => $y, + 'month' => $m, + 'day' => $d, + 'index' => $i))); + + $begdate = strtotime("$y-$m-$d 00:00:00"); + $enddate = $begdate + (24 * 60 * 60); + + if ($enddate < time()) { + $this->element('lastmod', null, date(DATE_W3C, $enddate)); + } + + $this->elementEnd('sitemap'); } } -- cgit v1.2.3-54-g00ecf From e363b724b96d0509e56edabcb7fb199698e158b7 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 12 Apr 2010 12:13:48 -0400 Subject: cache notice and user counts in sitemap index --- plugins/Sitemap/sitemapindex.php | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) (limited to 'plugins/Sitemap/sitemapindex.php') diff --git a/plugins/Sitemap/sitemapindex.php b/plugins/Sitemap/sitemapindex.php index 7942bc3bd..2055dd7f0 100644 --- a/plugins/Sitemap/sitemapindex.php +++ b/plugins/Sitemap/sitemapindex.php @@ -101,20 +101,25 @@ class SitemapindexAction extends Action function getUserCounts() { - // XXX: cachemeplease + $userCounts = User::cacheGet('sitemap:user:counts'); - $user = new User(); + if ($userCounts === false) { - $user->selectAdd(); - $user->selectAdd('date(created) as regdate, count(*) as regcount'); - $user->groupBy('regdate'); + $user = new User(); - $user->find(); + $user->selectAdd(); + $user->selectAdd('date(created) as regdate, count(*) as regcount'); + $user->groupBy('regdate'); - $userCounts = array(); + $user->find(); - while ($user->fetch()) { - $userCounts[$user->regdate] = $user->regcount; + $userCounts = array(); + + while ($user->fetch()) { + $userCounts[$user->regdate] = $user->regcount; + } + + User::cacheSet('sitemap:user:counts', $userCounts); } return $userCounts; @@ -122,20 +127,25 @@ class SitemapindexAction extends Action function getNoticeCounts() { - // XXX: cachemeplease + $noticeCounts = Notice::cacheGet('sitemap:notice:counts'); - $notice = new Notice(); + if ($noticeCounts === false) { - $notice->selectAdd(); - $notice->selectAdd('date(created) as postdate, count(*) as postcount'); - $notice->groupBy('postdate'); + $notice = new Notice(); - $notice->find(); + $notice->selectAdd(); + $notice->selectAdd('date(created) as postdate, count(*) as postcount'); + $notice->groupBy('postdate'); - $noticeCounts = array(); + $notice->find(); + + $noticeCounts = array(); + + while ($notice->fetch()) { + $noticeCounts[$notice->postdate] = $notice->postcount; + } - while ($notice->fetch()) { - $noticeCounts[$notice->postdate] = $notice->postcount; + Notice::cacheSet('sitemap:notice:counts', $noticeCounts); } return $noticeCounts; -- cgit v1.2.3-54-g00ecf From 416161c94366292a623aecf8fe79b0d73c337e98 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 21 May 2010 16:47:30 -0400 Subject: make user counts use the database table --- plugins/Sitemap/SitemapPlugin.php | 4 + plugins/Sitemap/Sitemap_user_count.php | 166 +++++++++++++++++++++++++++++++++ plugins/Sitemap/sitemapindex.php | 35 ++----- 3 files changed, 180 insertions(+), 25 deletions(-) (limited to 'plugins/Sitemap/sitemapindex.php') diff --git a/plugins/Sitemap/SitemapPlugin.php b/plugins/Sitemap/SitemapPlugin.php index 831694efc..82c007d66 100644 --- a/plugins/Sitemap/SitemapPlugin.php +++ b/plugins/Sitemap/SitemapPlugin.php @@ -64,6 +64,9 @@ class SitemapPlugin extends Plugin switch ($cls) { + case 'Sitemap_user_count': + require_once $dir . '/' . $cls . '.php'; + return false; case 'SitemapindexAction': case 'NoticesitemapAction': case 'UsersitemapAction': @@ -71,6 +74,7 @@ class SitemapPlugin extends Plugin return false; case 'SitemapAction': require_once $dir . '/' . strtolower($cls) . '.php'; + return false; default: return true; } diff --git a/plugins/Sitemap/Sitemap_user_count.php b/plugins/Sitemap/Sitemap_user_count.php index 1a7a6577d..7743b0532 100644 --- a/plugins/Sitemap/Sitemap_user_count.php +++ b/plugins/Sitemap/Sitemap_user_count.php @@ -105,6 +105,11 @@ class Sitemap_user_count extends Memcached_DataObject return array('registration_date' => 'K'); } + function sequenceKey() + { + return array(false, false, false); + } + /** * return key definitions for Memcached_DataObject * @@ -118,4 +123,165 @@ class Sitemap_user_count extends Memcached_DataObject { return $this->keys(); } + + static function getAll() + { + $userCounts = self::cacheGet('sitemap:user:counts'); + + if ($userCounts === false) { + + $suc = new Sitemap_user_count(); + $suc->orderBy('registration_date DESC'); + + // Fetch the first one to check up-to-date-itude + + $n = $suc->find(true); + + $today = self::today(); + $userCounts = array(); + + if (!$n) { // No counts saved yet + $userCounts = self::initializeCounts(); + } else if ($suc->registration_date < $today) { // There are counts but not up to today + $userCounts = self::fillInCounts($suc->registration_date); + } else if ($suc->registration_date == $today) { // Refresh today's + $userCounts[$today] = self::updateToday(); + } + + // starts with second-to-last date + + while ($suc->fetch()) { + $userCounts[$suc->registration_date] = $suc->user_count; + } + + self::cacheSet('sitemap:user:counts', $userCounts); + } + + return $userCounts; + } + + static function initializeCounts() + { + $firstDate = self::getFirstDate(); // awww + $today = self::today(); + + $counts = array(); + + for ($d = $firstDate; $d <= $today; $d = self::incrementDay($d)) { + common_debug("Date = '$d'"); + $n = self::getCount($d); + self::insertCount($d, $n); + $counts[$d] = $n; + } + + return $counts; + } + + static function fillInCounts($lastDate) + { + $today = self::today(); + + $counts = array(); + + $n = self::getCount($lastDate); + self::updateCount($lastDate, $n); + + $counts[$lastDate] = $n; + + for ($d = self::incrementDay($lastDate); $d <= $today; $d = self::incrementDay($d)) { + $n = self::getCount($d); + self::insertCount($d, $n); + } + + return $counts; + } + + static function updateToday() + { + $today = self::today(); + + $n = self::getCount($today); + self::updateCount($today, $n); + + return $n; + } + + static function getCount($d) + { + $user = new User(); + $user->whereAdd('created BETWEEN "'.$d.' 00:00:00" AND "'.self::incrementDay($d).' 00:00:00"'); + $n = $user->count(); + + return $n; + } + + static function insertCount($d, $n) + { + common_debug("Inserting count '$n' for '$d'"); + + $suc = new Sitemap_user_count(); + + $suc->registration_date = DB_DataObject_Cast::date($d); + $suc->user_count = $n; + $suc->created = common_sql_now(); + $suc->modified = $suc->created; + + if (!$suc->insert()) { + common_log(LOG_WARNING, "Could not save user counts for '$d'"); + } + } + + static function updateCount($d, $n) + { + $suc = Sitemap_user_count::staticGet('registration_date', DB_DataObject_Cast::date($d)); + + if (empty($suc)) { + throw new Exception("No such registration date: $d"); + } + + $orig = clone($suc); + + $suc->registration_date = DB_DataObject_Cast::date($d); + $suc->user_count = $n; + $suc->created = common_sql_now(); + $suc->modified = $suc->created; + + if (!$suc->update($orig)) { + common_log(LOG_WARNING, "Could not save user counts for '$d'"); + } + } + + static function incrementDay($d) + { + $dt = self::dateStrToInt($d); + return self::dateIntToStr($dt + 24 * 60 * 60); + } + + static function dateStrToInt($d) + { + return strtotime($d.' 00:00:00'); + } + + static function dateIntToStr($dt) + { + return date('Y-m-d', $dt); + } + + static function getFirstDate() + { + $u = new User(); + $u->selectAdd(); + $u->selectAdd('date(min(created)) as first_date'); + if ($u->find(true)) { + return $u->first_date; + } else { + // Is this right? + return self::dateIntToStr(time()); + } + } + + static function today() + { + return self::dateIntToStr(time()); + } } diff --git a/plugins/Sitemap/sitemapindex.php b/plugins/Sitemap/sitemapindex.php index 2055dd7f0..a3328340f 100644 --- a/plugins/Sitemap/sitemapindex.php +++ b/plugins/Sitemap/sitemapindex.php @@ -68,11 +68,15 @@ class SitemapindexAction extends Action function showUserSitemaps() { - $userCounts = $this->getUserCounts(); + $userCounts = Sitemap_user_count::getAll(); foreach ($userCounts as $dt => $cnt) { $cnt = $cnt+0; - assert($cnt != 0); + + if ($cnt == 0) { + continue; + } + $n = (int)$cnt / (int)SitemapPlugin::USERS_PER_MAP; if (($cnt % SitemapPlugin::USERS_PER_MAP) != 0) { $n++; @@ -88,7 +92,9 @@ class SitemapindexAction extends Action $noticeCounts = $this->getNoticeCounts(); foreach ($noticeCounts as $dt => $cnt) { - assert($cnt != 0); + if ($cnt == 0) { + continue; + } $n = $cnt / SitemapPlugin::NOTICES_PER_MAP; if ($cnt % SitemapPlugin::NOTICES_PER_MAP) { $n++; @@ -101,28 +107,7 @@ class SitemapindexAction extends Action function getUserCounts() { - $userCounts = User::cacheGet('sitemap:user:counts'); - - if ($userCounts === false) { - - $user = new User(); - - $user->selectAdd(); - $user->selectAdd('date(created) as regdate, count(*) as regcount'); - $user->groupBy('regdate'); - - $user->find(); - - $userCounts = array(); - - while ($user->fetch()) { - $userCounts[$user->regdate] = $user->regcount; - } - - User::cacheSet('sitemap:user:counts', $userCounts); - } - - return $userCounts; + return Sitemap_user_count::getAll(); } function getNoticeCounts() -- cgit v1.2.3-54-g00ecf From 1066b264247811ec9371ff2f473f5d7f2a6dd98a Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 31 May 2010 07:50:27 -0700 Subject: use sitemap_notice_count in sitemap index --- plugins/Sitemap/sitemapindex.php | 37 ++++--------------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) (limited to 'plugins/Sitemap/sitemapindex.php') diff --git a/plugins/Sitemap/sitemapindex.php b/plugins/Sitemap/sitemapindex.php index a3328340f..5150b1aeb 100644 --- a/plugins/Sitemap/sitemapindex.php +++ b/plugins/Sitemap/sitemapindex.php @@ -58,8 +58,8 @@ class SitemapindexAction extends Action $this->elementStart('sitemapindex', array('xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9')); - $this->showUserSitemaps(); $this->showNoticeSitemaps(); + $this->showUserSitemaps(); $this->elementEnd('sitemapindex'); @@ -89,7 +89,9 @@ class SitemapindexAction extends Action function showNoticeSitemaps() { - $noticeCounts = $this->getNoticeCounts(); + $noticeCounts = Sitemap_notice_count::getAll(); + + common_debug(sprintf("Got %d notice counts", count($noticeCounts))); foreach ($noticeCounts as $dt => $cnt) { if ($cnt == 0) { @@ -105,37 +107,6 @@ class SitemapindexAction extends Action } } - function getUserCounts() - { - return Sitemap_user_count::getAll(); - } - - function getNoticeCounts() - { - $noticeCounts = Notice::cacheGet('sitemap:notice:counts'); - - if ($noticeCounts === false) { - - $notice = new Notice(); - - $notice->selectAdd(); - $notice->selectAdd('date(created) as postdate, count(*) as postcount'); - $notice->groupBy('postdate'); - - $notice->find(); - - $noticeCounts = array(); - - while ($notice->fetch()) { - $noticeCounts[$notice->postdate] = $notice->postcount; - } - - Notice::cacheSet('sitemap:notice:counts', $noticeCounts); - } - - return $noticeCounts; - } - function showSitemap($prefix, $dt, $i) { list($y, $m, $d) = explode('-', $dt); -- cgit v1.2.3-54-g00ecf From 1100831bc0271574552e9757aff09d8a9c916038 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 3 Jun 2010 15:19:46 -0400 Subject: remove debug statements from Sitemap plugin --- plugins/Sitemap/Sitemap_user_count.php | 3 --- plugins/Sitemap/sitemapindex.php | 2 -- 2 files changed, 5 deletions(-) (limited to 'plugins/Sitemap/sitemapindex.php') diff --git a/plugins/Sitemap/Sitemap_user_count.php b/plugins/Sitemap/Sitemap_user_count.php index 7743b0532..64b4c3442 100644 --- a/plugins/Sitemap/Sitemap_user_count.php +++ b/plugins/Sitemap/Sitemap_user_count.php @@ -168,7 +168,6 @@ class Sitemap_user_count extends Memcached_DataObject $counts = array(); for ($d = $firstDate; $d <= $today; $d = self::incrementDay($d)) { - common_debug("Date = '$d'"); $n = self::getCount($d); self::insertCount($d, $n); $counts[$d] = $n; @@ -217,8 +216,6 @@ class Sitemap_user_count extends Memcached_DataObject static function insertCount($d, $n) { - common_debug("Inserting count '$n' for '$d'"); - $suc = new Sitemap_user_count(); $suc->registration_date = DB_DataObject_Cast::date($d); diff --git a/plugins/Sitemap/sitemapindex.php b/plugins/Sitemap/sitemapindex.php index 5150b1aeb..169e3031c 100644 --- a/plugins/Sitemap/sitemapindex.php +++ b/plugins/Sitemap/sitemapindex.php @@ -91,8 +91,6 @@ class SitemapindexAction extends Action { $noticeCounts = Sitemap_notice_count::getAll(); - common_debug(sprintf("Got %d notice counts", count($noticeCounts))); - foreach ($noticeCounts as $dt => $cnt) { if ($cnt == 0) { continue; -- cgit v1.2.3-54-g00ecf