From c4040303cecc48aed66d0102de6b9f5ffdc64b22 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 10:18:16 -0400 Subject: initial snapshot stuff --- lib/common.php | 3 ++ lib/snapshot.php | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 lib/snapshot.php (limited to 'lib') diff --git a/lib/common.php b/lib/common.php index dd2570e75..f81c3dc76 100644 --- a/lib/common.php +++ b/lib/common.php @@ -157,6 +157,9 @@ $config = 'newuser' => array('subscribe' => null, 'welcome' => null), + 'snapshot' => + array('run' => 'web', + 'frequency' => 10000), ); $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); diff --git a/lib/snapshot.php b/lib/snapshot.php new file mode 100644 index 000000000..4f9bb3f62 --- /dev/null +++ b/lib/snapshot.php @@ -0,0 +1,101 @@ +. + * + * @category Stats + * @package Laconica + * @author Evan Prodromou + * @copyright 2009 Control Yourself, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + */ + +if (!defined('LACONICA')) { + exit(1); +} + +/** + * A snapshot of site stats that can report itself to headquarters + * + * This class will collect statistics on the site and report them to + * a statistics server of the admin's choice. (Default is the big one + * at laconi.ca.) + * + * It can either be called from a cron job, or run occasionally by the + * Web site. + * + * @category Stats + * @package Laconica + * @author Evan Prodromou + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://laconi.ca/ + * + */ + +class Snapshot { + + function __construct() + { + } + + function take() + { + } + + function report() + { + } + + static function check() + { + switch (common_config('snapshot', 'run')) { + case 'web': + // skip if we're not running on the Web. + if (!isset($_SERVER) || !array_key_exists('REQUEST_METHOD', $_SERVER)) { + break; + } + // Run once every frequency hits + // XXX: do frequency by time (once a week, etc.) rather than + // hits + if (rand() % common_config('snapshot', 'frequency') == 0) { + $snapshot = new Snapshot(); + if ($snapshot->take()) { + $snapshot->report(); + } + } + break; + case 'cron': + // skip if we're running on the Web + if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + break; + } + // We're running from the command line; assume + $snapshot = new Snapshot(); + if ($snapshot->take()) { + $snapshot->report(); + } + break; + case 'never': + break; + default: + common_log(LOG_WARNING, "Unrecognized value for snapshot run config."); + } + } +} -- cgit v1.2.3-54-g00ecf From 5128448c777e74c0e294c21ce80711c04395d6d4 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 12:41:30 -0400 Subject: code complete on snapshot.php --- lib/snapshot.php | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/snapshot.php b/lib/snapshot.php index 4f9bb3f62..338c8d559 100644 --- a/lib/snapshot.php +++ b/lib/snapshot.php @@ -51,15 +51,9 @@ if (!defined('LACONICA')) { class Snapshot { - function __construct() - { - } + var $stats = null; - function take() - { - } - - function report() + function __construct() { } @@ -98,4 +92,82 @@ class Snapshot { common_log(LOG_WARNING, "Unrecognized value for snapshot run config."); } } + + function take() + { + $this->stats = array(); + + // Some basic identification stuff + + $this->stats['version'] = LACONICA_VERSION; + $this->stats['phpversion'] = phpversion(); + $this->stats['name'] = common_config('site', 'name'); + $this->stats['root'] = common_root_url(); + + // non-identifying stats on various tables. Primary + // interest is size and rate of activity of service. + + $tables = array('user', + 'notice', + 'subscription', + 'remote_profile', + 'user_group'); + + foreach ($tables as $table) { + $this->tableStats($table); + } + + // stats on some important config options + + $this->stats['theme'] = common_config('site', 'theme'); + $this->stats['dbtype'] = common_config('db', 'type'); + $this->stats['xmpp'] = common_config('xmpp', 'enabled'); + $this->stats['inboxes'] = common_config('inboxes', 'enabled'); + $this->stats['queue'] = common_config('queue', 'enabled'); + $this->stats['license'] = common_config('license', 'url'); + $this->stats['fancy'] = common_config('site', 'fancy'); + $this->stats['private'] = common_config('site', 'private'); + $this->stats['closed'] = common_config('site', 'closed'); + $this->stats['memcached'] = common_config('memcached', 'enabled'); + $this->stats['language'] = common_config('site', 'language'); + $this->stats['timezone'] = common_config('site', 'timezone'); + } + + function report() + { + // XXX: Use OICU2 and OAuth to make authorized requests + + $postdata = http_build_query($this->stats); + + $opts = array('http' => + array( + 'method' => 'POST', + 'header' => 'Content-type: application/x-www-form-urlencoded', + 'content' => $postdata, + 'user_agent' => 'Laconica/'.LACONICA_VERSION + ) + ); + + $context = stream_context_create($opts); + + $reporturl = common_config('snapshot', 'reporturl'); + + $result = file_get_contents($reporturl, false, $context); + } + + function tableStats($table) + { + $inst = DB_DataObject::Factory($table); + $res = $inst->query('SELECT count(*) as cnt, '. + 'min(created) as first, '. + 'max(created) as last '. + 'from ' . $table); + if ($res) { + $this->stats[$table.'count'] = $inst->cnt; + $this->stats[$table.'first'] = $inst->first; + $this->stats[$table.'last'] = $inst->last; + } + $inst->free(); + unset($inst); + } } -- cgit v1.2.3-54-g00ecf From 415abdfdef64608d6c4ba081dd21575e1920770b Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 12:50:13 -0400 Subject: config options for snapshots --- lib/common.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/common.php b/lib/common.php index f81c3dc76..05539af83 100644 --- a/lib/common.php +++ b/lib/common.php @@ -159,7 +159,8 @@ $config = 'welcome' => null), 'snapshot' => array('run' => 'web', - 'frequency' => 10000), + 'frequency' => 10000, + 'reporturl' => 'http://laconi.ca/stats/report'), ); $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); -- cgit v1.2.3-54-g00ecf From 6a247acb45198c3748df48d8b14b71c8f8b839de Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 12:50:29 -0400 Subject: Do some phpcs reformatting for snapshot --- lib/snapshot.php | 107 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/snapshot.php b/lib/snapshot.php index 338c8d559..a014d3435 100644 --- a/lib/snapshot.php +++ b/lib/snapshot.php @@ -49,18 +49,32 @@ if (!defined('LACONICA')) { * */ -class Snapshot { - +class Snapshot +{ var $stats = null; + /** + * Constructor for a snapshot + */ + function __construct() { } + /** + * Static function for reporting statistics + * + * This function checks whether it should report statistics, based on + * the current configuation settings. If it should, it creates a new + * Snapshot object, takes a snapshot, and reports it to headquarters. + * + * @return void + */ + static function check() { switch (common_config('snapshot', 'run')) { - case 'web': + case 'web': // skip if we're not running on the Web. if (!isset($_SERVER) || !array_key_exists('REQUEST_METHOD', $_SERVER)) { break; @@ -75,7 +89,7 @@ class Snapshot { } } break; - case 'cron': + case 'cron': // skip if we're running on the Web if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { break; @@ -86,23 +100,33 @@ class Snapshot { $snapshot->report(); } break; - case 'never': + case 'never': break; - default: + default: common_log(LOG_WARNING, "Unrecognized value for snapshot run config."); } } + /** + * Take a snapshot of the server + * + * Builds an array of statistical and configuration data based + * on the local database and config files. We avoid grabbing any + * information that could be personal or private. + * + * @return void + */ + function take() { $this->stats = array(); // Some basic identification stuff - $this->stats['version'] = LACONICA_VERSION; + $this->stats['version'] = LACONICA_VERSION; $this->stats['phpversion'] = phpversion(); - $this->stats['name'] = common_config('site', 'name'); - $this->stats['root'] = common_root_url(); + $this->stats['name'] = common_config('site', 'name'); + $this->stats['root'] = common_root_url(); // non-identifying stats on various tables. Primary // interest is size and rate of activity of service. @@ -119,34 +143,44 @@ class Snapshot { // stats on some important config options - $this->stats['theme'] = common_config('site', 'theme'); - $this->stats['dbtype'] = common_config('db', 'type'); - $this->stats['xmpp'] = common_config('xmpp', 'enabled'); - $this->stats['inboxes'] = common_config('inboxes', 'enabled'); - $this->stats['queue'] = common_config('queue', 'enabled'); - $this->stats['license'] = common_config('license', 'url'); - $this->stats['fancy'] = common_config('site', 'fancy'); - $this->stats['private'] = common_config('site', 'private'); - $this->stats['closed'] = common_config('site', 'closed'); + $this->stats['theme'] = common_config('site', 'theme'); + $this->stats['dbtype'] = common_config('db', 'type'); + $this->stats['xmpp'] = common_config('xmpp', 'enabled'); + $this->stats['inboxes'] = common_config('inboxes', 'enabled'); + $this->stats['queue'] = common_config('queue', 'enabled'); + $this->stats['license'] = common_config('license', 'url'); + $this->stats['fancy'] = common_config('site', 'fancy'); + $this->stats['private'] = common_config('site', 'private'); + $this->stats['closed'] = common_config('site', 'closed'); $this->stats['memcached'] = common_config('memcached', 'enabled'); - $this->stats['language'] = common_config('site', 'language'); - $this->stats['timezone'] = common_config('site', 'timezone'); + $this->stats['language'] = common_config('site', 'language'); + $this->stats['timezone'] = common_config('site', 'timezone'); } + /** + * Reports statistics to headquarters + * + * Posts statistics to a reporting server. + * + * @return void + */ + function report() { // XXX: Use OICU2 and OAuth to make authorized requests $postdata = http_build_query($this->stats); - $opts = array('http' => - array( - 'method' => 'POST', - 'header' => 'Content-type: application/x-www-form-urlencoded', - 'content' => $postdata, - 'user_agent' => 'Laconica/'.LACONICA_VERSION - ) - ); + $opts = + array('http' => + array( + 'method' => 'POST', + 'header' => 'Content-type: '. + 'application/x-www-form-urlencoded', + 'content' => $postdata, + 'user_agent' => 'Laconica/'.LACONICA_VERSION + ) + ); $context = stream_context_create($opts); @@ -155,18 +189,33 @@ class Snapshot { $result = file_get_contents($reporturl, false, $context); } + /** + * Updates statistics for a single table + * + * Determines the size of a table and its oldest and newest rows. + * Goal here is to see how active a site is. Note that it + * fills up the instance stats variable. + * + * @param string $table name of table to check + * + * @return void + */ + function tableStats($table) { $inst = DB_DataObject::Factory($table); + $res = $inst->query('SELECT count(*) as cnt, '. 'min(created) as first, '. 'max(created) as last '. 'from ' . $table); + if ($res) { $this->stats[$table.'count'] = $inst->cnt; $this->stats[$table.'first'] = $inst->first; - $this->stats[$table.'last'] = $inst->last; + $this->stats[$table.'last'] = $inst->last; } + $inst->free(); unset($inst); } -- cgit v1.2.3-54-g00ecf From 0838143d5855bb013b355d4abb7ec7a643ef37dc Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 12:53:17 -0400 Subject: allow snapshots to be disabled easily --- lib/common.php | 3 ++- lib/snapshot.php | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/common.php b/lib/common.php index 05539af83..2c60b0442 100644 --- a/lib/common.php +++ b/lib/common.php @@ -158,7 +158,8 @@ $config = array('subscribe' => null, 'welcome' => null), 'snapshot' => - array('run' => 'web', + array('enabled' => true, + 'run' => 'web', 'frequency' => 10000, 'reporturl' => 'http://laconi.ca/stats/report'), ); diff --git a/lib/snapshot.php b/lib/snapshot.php index a014d3435..75850a301 100644 --- a/lib/snapshot.php +++ b/lib/snapshot.php @@ -73,6 +73,10 @@ class Snapshot static function check() { + if (!common_config('snapshot', 'enabled')) { + return; + } + switch (common_config('snapshot', 'run')) { case 'web': // skip if we're not running on the Web. -- cgit v1.2.3-54-g00ecf From d76bb2fd115f120d799335069d6bc59203650d55 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 16 Apr 2009 13:05:35 -0400 Subject: Revert "allow snapshots to be disabled easily" This reverts commit 0838143d5855bb013b355d4abb7ec7a643ef37dc. Already handled with the 'never' option. --- lib/common.php | 3 +-- lib/snapshot.php | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/common.php b/lib/common.php index 37744ebd4..e64ca34da 100644 --- a/lib/common.php +++ b/lib/common.php @@ -158,8 +158,7 @@ $config = array('subscribe' => null, 'welcome' => null), 'snapshot' => - array('enabled' => true, - 'run' => 'web', + array('run' => 'web', 'frequency' => 10000, 'reporturl' => 'http://laconi.ca/stats/report'), ); diff --git a/lib/snapshot.php b/lib/snapshot.php index 75850a301..a014d3435 100644 --- a/lib/snapshot.php +++ b/lib/snapshot.php @@ -73,10 +73,6 @@ class Snapshot static function check() { - if (!common_config('snapshot', 'enabled')) { - return; - } - switch (common_config('snapshot', 'run')) { case 'web': // skip if we're not running on the Web. -- cgit v1.2.3-54-g00ecf From da035331d81bc73f488968d835e4d05a2da975f3 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 26 May 2009 17:26:31 -0400 Subject: fixes during checking of snapshot --- lib/snapshot.php | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/snapshot.php b/lib/snapshot.php index a014d3435..4b05b502d 100644 --- a/lib/snapshot.php +++ b/lib/snapshot.php @@ -84,9 +84,8 @@ class Snapshot // hits if (rand() % common_config('snapshot', 'frequency') == 0) { $snapshot = new Snapshot(); - if ($snapshot->take()) { - $snapshot->report(); - } + $snapshot->take(); + $snapshot->report(); } break; case 'cron': @@ -94,11 +93,14 @@ class Snapshot if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { break; } + common_log(LOG_INFO, 'Running snapshot from cron job'); // We're running from the command line; assume + $snapshot = new Snapshot(); - if ($snapshot->take()) { - $snapshot->report(); - } + $snapshot->take(); + common_log(LOG_INFO, count($snapshot->stats) . " statistics being uploaded."); + $snapshot->report(); + break; case 'never': break; @@ -155,6 +157,7 @@ class Snapshot $this->stats['memcached'] = common_config('memcached', 'enabled'); $this->stats['language'] = common_config('site', 'language'); $this->stats['timezone'] = common_config('site', 'timezone'); + } /** @@ -186,7 +189,9 @@ class Snapshot $reporturl = common_config('snapshot', 'reporturl'); - $result = file_get_contents($reporturl, false, $context); + $result = @file_get_contents($reporturl, false, $context); + + return $result; } /** @@ -203,14 +208,14 @@ class Snapshot function tableStats($table) { - $inst = DB_DataObject::Factory($table); + $inst = DB_DataObject::factory($table); - $res = $inst->query('SELECT count(*) as cnt, '. - 'min(created) as first, '. - 'max(created) as last '. - 'from ' . $table); + $inst->selectAdd(); + $inst->selectAdd('count(*) as cnt, '. + 'min(created) as first, '. + 'max(created) as last'); - if ($res) { + if ($inst->find(true)) { $this->stats[$table.'count'] = $inst->cnt; $this->stats[$table.'first'] = $inst->first; $this->stats[$table.'last'] = $inst->last; -- cgit v1.2.3-54-g00ecf