From ec648fb71748fa15872eacc66144b2960800d897 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 22 Sep 2010 12:52:34 -0700 Subject: Split Yammer importer files into subdirs before I get too lost adding UI --- plugins/YammerImport/lib/yammerimporter.php | 468 ++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100644 plugins/YammerImport/lib/yammerimporter.php (limited to 'plugins/YammerImport/lib/yammerimporter.php') diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php new file mode 100644 index 000000000..9ce0d1e58 --- /dev/null +++ b/plugins/YammerImport/lib/yammerimporter.php @@ -0,0 +1,468 @@ +. + */ + +/** + * Basic client class for Yammer's OAuth/JSON API. + * + * @package YammerImportPlugin + * @author Brion Vibber + */ +class YammerImporter +{ + protected $client; + protected $users=array(); + protected $groups=array(); + protected $notices=array(); + + function __construct(SN_YammerClient $client) + { + $this->client = $client; + } + + /** + * Load or create an imported profile from Yammer data. + * + * @param object $item loaded JSON data for Yammer importer + * @return Profile + */ + function importUser($item) + { + $data = $this->prepUser($item); + + $profileId = $this->findImportedUser($data['orig_id']); + if ($profileId) { + return Profile::staticGet('id', $profileId); + } else { + $user = User::register($data['options']); + $profile = $user->getProfile(); + if ($data['avatar']) { + try { + $this->saveAvatar($data['avatar'], $profile); + } catch (Exception $e) { + common_log(LOG_ERR, "Error importing Yammer avatar: " . $e->getMessage()); + } + } + $this->recordImportedUser($data['orig_id'], $profile->id); + return $profile; + } + } + + /** + * Load or create an imported group from Yammer data. + * + * @param object $item loaded JSON data for Yammer importer + * @return User_group + */ + function importGroup($item) + { + $data = $this->prepGroup($item); + + $groupId = $this->findImportedGroup($data['orig_id']); + if ($groupId) { + return User_group::staticGet('id', $groupId); + } else { + $group = User_group::register($data['options']); + if ($data['avatar']) { + try { + $this->saveAvatar($data['avatar'], $group); + } catch (Exception $e) { + common_log(LOG_ERR, "Error importing Yammer avatar: " . $e->getMessage()); + } + } + $this->recordImportedGroup($data['orig_id'], $group->id); + return $group; + } + } + + /** + * Load or create an imported notice from Yammer data. + * + * @param object $item loaded JSON data for Yammer importer + * @return Notice + */ + function importNotice($item) + { + $data = $this->prepNotice($item); + + $noticeId = $this->findImportedNotice($data['orig_id']); + if ($noticeId) { + return Notice::staticGet('id', $noticeId); + } else { + $content = $data['content']; + $user = User::staticGet($data['profile']); + + // Fetch file attachments and add the URLs... + $uploads = array(); + foreach ($data['attachments'] as $url) { + try { + $upload = $this->saveAttachment($url, $user); + $content .= ' ' . $upload->shortUrl(); + $uploads[] = $upload; + } catch (Exception $e) { + common_log(LOG_ERR, "Error importing Yammer attachment: " . $e->getMessage()); + } + } + + // Here's the meat! Actually save the dang ol' notice. + $notice = Notice::saveNew($user->id, + $content, + $data['source'], + $data['options']); + + // Save "likes" as favorites... + foreach ($data['faves'] as $nickname) { + $user = User::staticGet('nickname', $nickname); + if ($user) { + Fave::addNew($user->getProfile(), $notice); + } + } + + // And finally attach the upload records... + foreach ($uploads as $upload) { + $upload->attachToNotice($notice); + } + $this->recordImportedNotice($data['orig_id'], $notice->id); + return $notice; + } + } + + /** + * Pull relevant info out of a Yammer data record for a user import. + * + * @param array $item + * @return array + */ + function prepUser($item) + { + if ($item['type'] != 'user') { + throw new Exception('Wrong item type sent to Yammer user import processing.'); + } + + $origId = $item['id']; + $origUrl = $item['url']; + + // @fixme check username rules? + + $options['nickname'] = $item['name']; + $options['fullname'] = trim($item['full_name']); + + // Avatar... this will be the "_small" variant. + // Remove that (pre-extension) suffix to get the orig-size image. + $avatar = $item['mugshot_url']; + + // The following info is only available in full data, not in the reference version. + + // There can be extensive contact info, but for now we'll only pull the primary email. + if (isset($item['contact'])) { + foreach ($item['contact']['email_addresses'] as $addr) { + if ($addr['type'] == 'primary') { + $options['email'] = $addr['address']; + $options['email_confirmed'] = true; + break; + } + } + } + + // There can be multiple external URLs; for now pull the first one as home page. + if (isset($item['external_urls'])) { + foreach ($item['external_urls'] as $url) { + if (common_valid_http_url($url)) { + $options['homepage'] = $url; + break; + } + } + } + + // Combine a few bits into the bio... + $bio = array(); + if (!empty($item['job_title'])) { + $bio[] = $item['job_title']; + } + if (!empty($item['summary'])) { + $bio[] = $item['summary']; + } + if (!empty($item['expertise'])) { + $bio[] = _m('Expertise:') . ' ' . $item['expertise']; + } + $options['bio'] = implode("\n\n", $bio); + + // Pull raw location string, may be lookupable + if (!empty($item['location'])) { + $options['location'] = $item['location']; + } + + // Timezone is in format like 'Pacific Time (US & Canada)' + // We need to convert that to a zone id. :P + // @fixme timezone not yet supported at registration time :) + if (!empty($item['timezone'])) { + $tz = $this->timezone($item['timezone']); + if ($tz) { + $options['timezone'] = $tz; + } + } + + return array('orig_id' => $origId, + 'orig_url' => $origUrl, + 'avatar' => $avatar, + 'options' => $options); + + } + + /** + * Pull relevant info out of a Yammer data record for a group import. + * + * @param array $item + * @return array + */ + function prepGroup($item) + { + if ($item['type'] != 'group') { + throw new Exception('Wrong item type sent to Yammer group import processing.'); + } + + $origId = $item['id']; + $origUrl = $item['url']; + + $privacy = $item['privacy']; // Warning! only public groups in SN so far + + $options['nickname'] = $item['name']; + $options['fullname'] = $item['full_name']; + $options['description'] = $item['description']; + $options['created'] = $this->timestamp($item['created_at']); + + $avatar = $item['mugshot_url']; // as with user profiles... + + + $options['mainpage'] = common_local_url('showgroup', + array('nickname' => $options['nickname'])); + + // @fixme what about admin user for the group? + // bio? homepage etc? aliases? + + $options['local'] = true; + return array('orig_id' => $origId, + 'orig_url' => $origUrl, + 'options' => $options, + 'avatar' => $avatar); + } + + /** + * Pull relevant info out of a Yammer data record for a notice import. + * + * @param array $item + * @return array + */ + function prepNotice($item) + { + if (isset($item['type']) && $item['type'] != 'message') { + throw new Exception('Wrong item type sent to Yammer message import processing.'); + } + + $origId = $item['id']; + $origUrl = $item['url']; + + $profile = $this->findImportedUser($item['sender_id']); + $content = $item['body']['plain']; + $source = 'yammer'; + $options = array(); + + if ($item['replied_to_id']) { + $replyTo = $this->findImportedNotice($item['replied_to_id']); + if ($replyTo) { + $options['reply_to'] = $replyTo; + } + } + $options['created'] = $this->timestamp($item['created_at']); + + if ($item['group_id']) { + $groupId = $this->findImportedGroup($item['group_id']); + if ($groupId) { + $options['groups'] = array($groupId); + + // @fixme if we see a group link inline, don't add this? + $group = User_group::staticGet('id', $groupId); + if ($group) { + $content .= ' !' . $group->nickname; + } + } + } + + $faves = array(); + foreach ($item['liked_by']['names'] as $liker) { + // "permalink" is the username. wtf? + $faves[] = $liker['permalink']; + } + + $attachments = array(); + foreach ($item['attachments'] as $attach) { + if ($attach['type'] == 'image' || $attach['type'] == 'file') { + $attachments[] = $attach[$attach['type']]['url']; + } else { + common_log(LOG_WARNING, "Unrecognized Yammer attachment type: " . $attach['type']); + } + } + + return array('orig_id' => $origId, + 'orig_url' => $origUrl, + 'profile' => $profile, + 'content' => $content, + 'source' => $source, + 'options' => $options, + 'faves' => $faves, + 'attachments' => $attachments); + } + + private function findImportedUser($origId) + { + if (isset($this->users[$origId])) { + return $this->users[$origId]; + } else { + return false; + } + } + + private function findImportedGroup($origId) + { + if (isset($this->groups[$origId])) { + return $this->groups[$origId]; + } else { + return false; + } + } + + private function findImportedNotice($origId) + { + if (isset($this->notices[$origId])) { + return $this->notices[$origId]; + } else { + return false; + } + } + + private function recordImportedUser($origId, $userId) + { + $this->users[$origId] = $userId; + } + + private function recordImportedGroup($origId, $groupId) + { + $this->groups[$origId] = $groupId; + } + + private function recordImportedNotice($origId, $noticeId) + { + $this->notices[$origId] = $noticeId; + } + + /** + * Normalize timestamp format. + * @param string $ts + * @return string + */ + private function timestamp($ts) + { + return common_sql_date(strtotime($ts)); + } + + private function timezone($tz) + { + // Blaaaaaarf! + $known = array('Pacific Time (US & Canada)' => 'America/Los_Angeles', + 'Eastern Time (US & Canada)' => 'America/New_York'); + if (array_key_exists($known, $tz)) { + return $known[$tz]; + } else { + return false; + } + } + + /** + * Download and update given avatar image + * + * @param string $url + * @param mixed $dest either a Profile or User_group object + * @throws Exception in various failure cases + */ + private function saveAvatar($url, $dest) + { + // Yammer API data mostly gives us the small variant. + // Try hitting the source image if we can! + // @fixme no guarantee of this URL scheme I think. + $url = preg_replace('/_small(\..*?)$/', '$1', $url); + + if (!common_valid_http_url($url)) { + throw new ServerException(sprintf(_m("Invalid avatar URL %s."), $url)); + } + + // @fixme this should be better encapsulated + // ripped from oauthstore.php (for old OMB client) + $temp_filename = tempnam(sys_get_temp_dir(), 'listener_avatar'); + if (!copy($url, $temp_filename)) { + throw new ServerException(sprintf(_m("Unable to fetch avatar from %s."), $url)); + } + + $id = $dest->id; + // @fixme should we be using different ids? + $imagefile = new ImageFile($id, $temp_filename); + $filename = Avatar::filename($id, + image_type_to_extension($imagefile->type), + null, + common_timestamp()); + rename($temp_filename, Avatar::path($filename)); + // @fixme hardcoded chmod is lame, but seems to be necessary to + // keep from accidentally saving images from command-line (queues) + // that can't be read from web server, which causes hard-to-notice + // problems later on: + // + // http://status.net/open-source/issues/2663 + chmod(Avatar::path($filename), 0644); + + $dest->setOriginal($filename); + } + + /** + * Fetch an attachment from Yammer and save it into our system. + * Unlike avatars, the attachment URLs are guarded by authentication, + * so we need to run the HTTP hit through our OAuth API client. + * + * @param string $url + * @param User $user + * @return MediaFile + * + * @throws Exception on low-level network or HTTP error + */ + private function saveAttachment($url, User $user) + { + // Fetch the attachment... + // WARNING: file must fit in memory here :( + $body = $this->client->fetchUrl($url); + + // Save to a temporary file and shove it into our file-attachment space... + $temp = tmpfile(); + fwrite($temp, $body); + try { + $upload = MediaFile::fromFileHandle($temp, $user); + fclose($temp); + return $upload; + } catch (Exception $e) { + fclose($temp); + throw $e; + } + } +} -- cgit v1.2.3-54-g00ecf From 472dab4a86c0ba365404b0994c3eb7e9979a80c2 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 22 Sep 2010 17:51:50 -0700 Subject: WORK IN PROGRESS: Starting infrastructure to initiate Yammer import from web UI and process it in the background queues. Totally not complete yet. --- plugins/YammerImport/YammerImportPlugin.php | 61 +++++++- plugins/YammerImport/actions/yammeradminpanel.php | 153 ++++++++++++++++++ plugins/YammerImport/actions/yammerauth.php | 17 ++ plugins/YammerImport/classes/Yammer_common.php | 165 +++++++++++++++++++ plugins/YammerImport/classes/Yammer_group.php | 79 ++++++++++ plugins/YammerImport/classes/Yammer_notice.php | 79 ++++++++++ .../YammerImport/classes/Yammer_notice_stub.php | 174 +++++++++++++++++++++ plugins/YammerImport/classes/Yammer_state.php | 37 +++++ plugins/YammerImport/classes/Yammer_user.php | 79 ++++++++++ plugins/YammerImport/lib/yammerimporter.php | 27 +--- plugins/YammerImport/lib/yammerqueuehandler.php | 47 ++++++ 11 files changed, 892 insertions(+), 26 deletions(-) create mode 100644 plugins/YammerImport/actions/yammeradminpanel.php create mode 100644 plugins/YammerImport/actions/yammerauth.php create mode 100644 plugins/YammerImport/classes/Yammer_common.php create mode 100644 plugins/YammerImport/classes/Yammer_group.php create mode 100644 plugins/YammerImport/classes/Yammer_notice.php create mode 100644 plugins/YammerImport/classes/Yammer_notice_stub.php create mode 100644 plugins/YammerImport/classes/Yammer_state.php create mode 100644 plugins/YammerImport/classes/Yammer_user.php create mode 100644 plugins/YammerImport/lib/yammerqueuehandler.php (limited to 'plugins/YammerImport/lib/yammerimporter.php') diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php index 79b8260b6..f55169a55 100644 --- a/plugins/YammerImport/YammerImportPlugin.php +++ b/plugins/YammerImport/YammerImportPlugin.php @@ -22,9 +22,7 @@ * @maintainer Brion Vibber */ -if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } - -set_include_path(get_include_path() . PATH_SEPARATOR . dirname(__FILE__) . '/extlib/'); +if (!defined('STATUSNET')) { exit(1); } class YammerImportPlugin extends Plugin { @@ -36,8 +34,8 @@ class YammerImportPlugin extends Plugin */ function onRouterInitialized($m) { - $m->connect('admin/import/yammer', - array('action' => 'importyammer')); + $m->connect('admin/yammer', + array('action' => 'yammeradminpanel')); return true; } @@ -53,6 +51,56 @@ class YammerImportPlugin extends Plugin return true; } + /** + * Set up all our tables... + */ + function onCheckSchema() + { + $schema = Schema::get(); + + $tables = array('Yammer_state', + 'Yammer_user', + 'Yammer_group', + 'Yammer_notice', + 'Yammer_notice_stub'); + foreach ($tables as $table) { + $schema->ensureTable($table, $table::schemaDef()); + } + + return true; + } + + /** + * If the plugin's installed, this should be accessible to admins. + */ + function onAdminPanelCheck($name, &$isOK) + { + if ($name == 'yammer') { + $isOK = true; + return false; + } + + return true; + } + + /** + * Add the Yammer admin panel to the list... + */ + function onEndAdminPanelNav($nav) + { + if (AdminPanelAction::canAdmin('yammer')) { + $action_name = $nav->action->trimmed('action'); + + $nav->out->menuItem(common_local_url('yammeradminpanel'), + _m('Yammer'), + _m('Yammer import'), + $action_name == 'yammeradminpanel', + 'nav_yammer_admin_panel'); + } + + return true; + } + /** * Automatically load the actions and libraries used by the plugin * @@ -70,6 +118,9 @@ class YammerImportPlugin extends Plugin case 'yammerimporter': require_once "$base/lib/$lower.php"; return false; + case 'yammeradminpanelaction': + require_once "$base/actions/yammeradminpanel.php"; + return false; default: return true; } diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php new file mode 100644 index 000000000..875debac9 --- /dev/null +++ b/plugins/YammerImport/actions/yammeradminpanel.php @@ -0,0 +1,153 @@ +. + * + * @category Settings + * @package StatusNet + * @author Zach Copley + * @copyright 2010 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +class YammeradminpanelAction extends AdminPanelAction +{ + /** + * Returns the page title + * + * @return string page title + */ + function title() + { + return _m('Yammer Import'); + } + + /** + * Instructions for using this form. + * + * @return string instructions + */ + function getInstructions() + { + return _m('Yammer import tool'); + } + + /** + * Show the Yammer admin panel form + * + * @return void + */ + function showForm() + { + $form = new YammerAdminPanelForm($this); + $form->show(); + return; + } +} + +class YammerAdminPanelForm extends AdminForm +{ + /** + * ID of the form + * + * @return string ID of the form + */ + function id() + { + return 'yammeradminpanel'; + } + + /** + * class of the form + * + * @return string class of the form + */ + function formClass() + { + return 'form_settings'; + } + + /** + * Action of the form + * + * @return string URL of the action + */ + function action() + { + return common_local_url('yammeradminpanel'); + } + + /** + * Data elements of the form + * + * @return void + */ + function formData() + { + $this->out->element('p', array(), 'yammer import IN DA HOUSE'); + + /* + Possible states of the yammer import process: + - null (not doing any sort of import) + - requesting-auth + - authenticated + - import-users + - import-groups + - fetch-messages + - import-messages + - done + */ + $yammerState = Yammer_state::staticGet('id', 1); + $state = $yammerState ? $yammerState->state || null; + + switch($state) + { + case null: + $this->out->element('p', array(), 'Time to start auth:'); + $this->showAuthForm(); + break; + case 'requesting-auth': + $this->out->element('p', array(), 'Need to finish auth!'); + $this->showAuthForm(); + break; + case 'import-users': + case 'import-groups': + case 'import-messages': + case 'save-messages': + $this->showImportState(); + break; + + } + } + + /** + * Action elements + * + * @return void + */ + function formActions() + { + // No submit buttons needed at bottom + } +} diff --git a/plugins/YammerImport/actions/yammerauth.php b/plugins/YammerImport/actions/yammerauth.php new file mode 100644 index 000000000..7e6e7204a --- /dev/null +++ b/plugins/YammerImport/actions/yammerauth.php @@ -0,0 +1,17 @@ +requestToken(); + $url = $yam->authorizeUrl($token); + + // We're going to try doing this in an iframe; if that's not happy + // we can redirect but there doesn't seem to be a way to get Yammer's + // oauth to call us back instead of the manual copy. :( + + //common_redirect($url, 303); + $this->element('iframe', array('id' => 'yammer-oauth', + 'src' => $url)); +} + diff --git a/plugins/YammerImport/classes/Yammer_common.php b/plugins/YammerImport/classes/Yammer_common.php new file mode 100644 index 000000000..81e302ab2 --- /dev/null +++ b/plugins/YammerImport/classes/Yammer_common.php @@ -0,0 +1,165 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2010, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +/** + * Common base class for the Yammer import mappings for users, groups, and notices. + * + * Child classes must override these static methods, since we need to run + * on PHP 5.2.x which has no late static binding: + * - staticGet (as our other classes) + * - schemaDef (call self::doSchemaDef) + * - record (call self::doRecord) + */ + +class Yammer_common extends Memcached_DataObject +{ + public $__table = 'yammer_XXXX'; // table name + public $__field = 'XXXX_id'; // field name to save into + public $id; // int primary_key not_null + public $user_id; // int(4) + public $created; // datetime + + /** + * @fixme add a 'references' thing for the foreign key when we support that + */ + protected static function doSchemaDef($field) + { + return array(new ColumnDef('id', 'bigint', null, + false, 'PRI'), + new ColumnDef($field, 'integer', null, + false, 'UNI'), + new ColumnDef('created', 'datetime', null, + false)); + } + + /** + * return table definition for DB_DataObject + * + * DB_DataObject needs to know something about the table to manipulate + * instances. This method provides all the DB_DataObject needs to know. + * + * @return array array of column definitions + */ + + function table() + { + return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + $this->__field => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + 'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL); + } + + /** + * return key definitions for DB_DataObject + * + * DB_DataObject needs to know about keys that the table has, since it + * won't appear in StatusNet's own keys list. In most cases, this will + * simply reference your keyTypes() function. + * + * @return array list of key field names + */ + + function keys() + { + return array_keys($this->keyTypes()); + } + + /** + * return key definitions for Memcached_DataObject + * + * Our caching system uses the same key definitions, but uses a different + * method to get them. This key information is used to store and clear + * cached data, so be sure to list any key that will be used for static + * lookups. + * + * @return array associative array of key definitions, field name to type: + * 'K' for primary key: for compound keys, add an entry for each component; + * 'U' for unique keys: compound keys are not well supported here. + */ + + function keyTypes() + { + return array('id' => 'K', $this->__field => 'U'); + } + + /** + * Magic formula for non-autoincrementing integer primary keys + * + * If a table has a single integer column as its primary key, DB_DataObject + * assumes that the column is auto-incrementing and makes a sequence table + * to do this incrementation. Since we don't need this for our class, we + * overload this method and return the magic formula that DB_DataObject needs. + * + * @return array magic three-false array that stops auto-incrementing. + */ + + function sequenceKey() + { + return array(false, false, false); + } + + /** + * Save a mapping between a remote Yammer and local imported user. + * + * @param integer $user_id ID of the status in StatusNet + * @param integer $orig_id ID of the notice in Yammer + * + * @return Yammer_common new object for this value + */ + + protected static function doRecord($class, $field, $orig_id, $local_id) + { + $map = self::staticGet('id', $orig_id); + + if (!empty($map)) { + return $map; + } + + $map = self::staticGet($field, $local_id); + + if (!empty($map)) { + return $map; + } + + common_debug("Mapping Yammer $field {$orig_id} to local $field {$local_id}"); + + $map = new $class(); + + $map->id = $orig_id; + $map->$field = $local_id; + $map->created = common_sql_now(); + + $map->insert(); + + return $map; + } +} diff --git a/plugins/YammerImport/classes/Yammer_group.php b/plugins/YammerImport/classes/Yammer_group.php new file mode 100644 index 000000000..4e7a6ebd0 --- /dev/null +++ b/plugins/YammerImport/classes/Yammer_group.php @@ -0,0 +1,79 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2010, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +class Yammer_group extends Yammer_common +{ + public $__table = 'yammer_group'; // table name + public $__field = 'group_id'; // field to map to + public $group_id; // int + + /** + * Get an instance by key + * + * This is a utility method to get a single instance with a given key value. + * + * @param string $k Key to use to lookup + * @param mixed $v Value to lookup + * + * @return Yammer_group object found, or null for no hits + * + */ + + function staticGet($k, $v=null) + { + return Memcached_DataObject::staticGet('Yammer_group', $k, $v); + } + + /** + * Return schema definition to set this table up in onCheckSchema + */ + + static function schemaDef() + { + return self::doSchemaDef('group_id'); + } + + /** + * Save a mapping between a remote Yammer and local imported group. + * + * @param integer $orig_id ID of the notice in Yammer + * @param integer $group_id ID of the status in StatusNet + * + * @return Yammer_group new object for this value + */ + + static function record($orig_id, $group_id) + { + return self::doRecord('Yammer_group', 'group_id', $orig_id, $group_id); + } +} diff --git a/plugins/YammerImport/classes/Yammer_notice.php b/plugins/YammerImport/classes/Yammer_notice.php new file mode 100644 index 000000000..0f63db630 --- /dev/null +++ b/plugins/YammerImport/classes/Yammer_notice.php @@ -0,0 +1,79 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2010, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +class Yammer_notice extends Yammer_common +{ + public $__table = 'yammer_notice'; // table name + public $__field = 'notice_id'; // field to map to + public $notice_id; // int + + /** + * Get an instance by key + * + * This is a utility method to get a single instance with a given key value. + * + * @param string $k Key to use to lookup + * @param mixed $v Value to lookup + * + * @return Yammer_notice object found, or null for no hits + * + */ + + function staticGet($k, $v=null) + { + return Memcached_DataObject::staticGet('Yammer_notice', $k, $v); + } + + /** + * Return schema definition to set this table up in onCheckSchema + */ + + static function schemaDef() + { + return self::doSchemaDef('notice_id'); + } + + /** + * Save a mapping between a remote Yammer and local imported notice. + * + * @param integer $orig_id ID of the notice in Yammer + * @param integer $notice_id ID of the status in StatusNet + * + * @return Yammer_notice new object for this value + */ + + static function record($orig_id, $notice_id) + { + return self::doRecord('Yammer_notice', 'notice_id', $orig_id, $notice_id); + } +} diff --git a/plugins/YammerImport/classes/Yammer_notice_stub.php b/plugins/YammerImport/classes/Yammer_notice_stub.php new file mode 100644 index 000000000..98a5e2cf7 --- /dev/null +++ b/plugins/YammerImport/classes/Yammer_notice_stub.php @@ -0,0 +1,174 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2010, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +/** + * Temporary storage for imported Yammer messages between fetching and saving + * as local notices. + * + * The Yammer API only allows us to page down from the most recent items; in + * order to start saving the oldest notices first, we have to pull them all + * down in reverse chronological order, then go back over them from oldest to + * newest and actually save them into our notice table. + */ + +class Yammer_notice_stub extends Memcached_DataObject +{ + public $__table = 'yammer_notice_stub'; // table name + public $id; // int primary_key not_null + public $json_data; // text + public $created; // datetime + + /** + * Return schema definition to set this table up in onCheckSchema + */ + static function schemaDef($field) + { + return array(new ColumnDef('id', 'bigint', null, + false, 'PRI'), + new ColumnDef('json_data', 'text', null, + false), + new ColumnDef('created', 'datetime', null, + false)); + } + + /** + * return table definition for DB_DataObject + * + * DB_DataObject needs to know something about the table to manipulate + * instances. This method provides all the DB_DataObject needs to know. + * + * @return array array of column definitions + */ + + function table() + { + return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + 'json_data' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + 'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL); + } + + /** + * return key definitions for DB_DataObject + * + * DB_DataObject needs to know about keys that the table has, since it + * won't appear in StatusNet's own keys list. In most cases, this will + * simply reference your keyTypes() function. + * + * @return array list of key field names + */ + + function keys() + { + return array_keys($this->keyTypes()); + } + + /** + * return key definitions for Memcached_DataObject + * + * Our caching system uses the same key definitions, but uses a different + * method to get them. This key information is used to store and clear + * cached data, so be sure to list any key that will be used for static + * lookups. + * + * @return array associative array of key definitions, field name to type: + * 'K' for primary key: for compound keys, add an entry for each component; + * 'U' for unique keys: compound keys are not well supported here. + */ + + function keyTypes() + { + return array('id' => 'K'); + } + + /** + * Magic formula for non-autoincrementing integer primary keys + * + * If a table has a single integer column as its primary key, DB_DataObject + * assumes that the column is auto-incrementing and makes a sequence table + * to do this incrementation. Since we don't need this for our class, we + * overload this method and return the magic formula that DB_DataObject needs. + * + * @return array magic three-false array that stops auto-incrementing. + */ + + function sequenceKey() + { + return array(false, false, false); + } + + /** + * Save the native Yammer API representation of a message for the pending + * import. Since they come in in reverse chronological order, we need to + * record them all as stubs and then go through from the beginning and + * save them as native notices, or we'll lose ordering and threading + * data. + * + * @param integer $orig_id ID of the notice on Yammer + * @param array $data the message record fetched out of Yammer API returnd data + * + * @return Yammer_notice_stub new object for this value + */ + + static function record($orig_id, $data) + { + common_debug("Recording Yammer message stub {$orig_id} for pending import..."); + + $stub = new Yammer_notice_stub(); + + $stub->id = $orig_id; + $stub->json_data = json_encode($data); + $stub->created = common_sql_now(); + + $stub->insert(); + + return $stub; + } + + /** + * Save a mapping between a remote Yammer and local imported user. + * + * @param integer $user_id ID of the status in StatusNet + * + * @return Yammer_notice_stub new object for this value + */ + + static function retrieve($orig_id) + { + $stub = self::staticGet('id', $orig_id); + if ($stub) { + return json_decode($stub->json_data, true); + } else { + return false; + } + } +} diff --git a/plugins/YammerImport/classes/Yammer_state.php b/plugins/YammerImport/classes/Yammer_state.php new file mode 100644 index 000000000..a476fd3be --- /dev/null +++ b/plugins/YammerImport/classes/Yammer_state.php @@ -0,0 +1,37 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://status.net/ + * + * StatusNet - the distributed open-source microblogging tool + * Copyright (C) 2010, StatusNet, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +class Yammer_user extends Yammer_common +{ + public $__table = 'yammer_user'; // table name + public $__field = 'user_id'; // field to map to + public $user_id; // int + + /** + * Get an instance by key + * + * This is a utility method to get a single instance with a given key value. + * + * @param string $k Key to use to lookup + * @param mixed $v Value to lookup + * + * @return Yammer_user object found, or null for no hits + * + */ + + function staticGet($k, $v=null) + { + return Memcached_DataObject::staticGet('Yammer_user', $k, $v); + } + + /** + * Return schema definition to set this table up in onCheckSchema + */ + + static function schemaDef() + { + return self::doSchemaDef('user_id'); + } + + /** + * Save a mapping between a remote Yammer and local imported user. + * + * @param integer $orig_id ID of the notice in Yammer + * @param integer $user_id ID of the status in StatusNet + * + * @return Yammer_user new object for this value + */ + + static function record($orig_id, $user_id) + { + return self::doRecord('Yammer_user', 'user_id', $orig_id, $user_id); + } +} diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php index 9ce0d1e58..b1d2815b9 100644 --- a/plugins/YammerImport/lib/yammerimporter.php +++ b/plugins/YammerImport/lib/yammerimporter.php @@ -26,9 +26,6 @@ class YammerImporter { protected $client; - protected $users=array(); - protected $groups=array(); - protected $notices=array(); function __construct(SN_YammerClient $client) { @@ -330,44 +327,32 @@ class YammerImporter private function findImportedUser($origId) { - if (isset($this->users[$origId])) { - return $this->users[$origId]; - } else { - return false; - } + return Yammer_user::staticGet('id', $origId); } private function findImportedGroup($origId) { - if (isset($this->groups[$origId])) { - return $this->groups[$origId]; - } else { - return false; - } + return Yammer_group::staticGet('id', $origId); } private function findImportedNotice($origId) { - if (isset($this->notices[$origId])) { - return $this->notices[$origId]; - } else { - return false; - } + return Yammer_notice::staticGet('id', $origId); } private function recordImportedUser($origId, $userId) { - $this->users[$origId] = $userId; + Yammer_user::record($origId, $userId); } private function recordImportedGroup($origId, $groupId) { - $this->groups[$origId] = $groupId; + Yammer_group::record($origId, $groupId); } private function recordImportedNotice($origId, $noticeId) { - $this->notices[$origId] = $noticeId; + Yammer_notice::record($origId, $noticeId); } /** diff --git a/plugins/YammerImport/lib/yammerqueuehandler.php b/plugins/YammerImport/lib/yammerqueuehandler.php new file mode 100644 index 000000000..ca81cbb34 --- /dev/null +++ b/plugins/YammerImport/lib/yammerqueuehandler.php @@ -0,0 +1,47 @@ +. + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +/** + * Queue handler for bumping the next chunk of Yammer import activity! + * + * @package YammerImportPlugin + * @author Brion Vibber + */ +class YammerQueueHandler extends QueueHandler +{ + function transport() + { + return 'yammer'; + } + + function handle($notice) + { + $importer = new YammerImporter(); + if ($importer->hasWork()) { + return $importer->iterate(); + } else { + // We're done! + return true; + } + } +} -- cgit v1.2.3-54-g00ecf From 8f438da254defa35d9e066fb29947da208d14319 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 23 Sep 2010 17:55:13 -0700 Subject: Ok, command-line workflow for YammerImportPlugin seems to mostly work, at least on tiny test site :D --- plugins/YammerImport/classes/Yammer_common.php | 4 +- .../YammerImport/classes/Yammer_notice_stub.php | 45 ++++++++----- plugins/YammerImport/classes/Yammer_state.php | 20 +++++- plugins/YammerImport/lib/sn_yammerclient.php | 7 +- plugins/YammerImport/lib/yammerimporter.php | 11 ++-- plugins/YammerImport/lib/yammerrunner.php | 77 ++++++++++++++++------ plugins/YammerImport/scripts/yammer-import.php | 43 ++++++++---- 7 files changed, 144 insertions(+), 63 deletions(-) (limited to 'plugins/YammerImport/lib/yammerimporter.php') diff --git a/plugins/YammerImport/classes/Yammer_common.php b/plugins/YammerImport/classes/Yammer_common.php index 81e302ab2..6ec6fc904 100644 --- a/plugins/YammerImport/classes/Yammer_common.php +++ b/plugins/YammerImport/classes/Yammer_common.php @@ -138,13 +138,13 @@ class Yammer_common extends Memcached_DataObject protected static function doRecord($class, $field, $orig_id, $local_id) { - $map = self::staticGet('id', $orig_id); + $map = parent::staticGet($class, 'id', $orig_id); if (!empty($map)) { return $map; } - $map = self::staticGet($field, $local_id); + $map = parent::staticGet($class, $field, $local_id); if (!empty($map)) { return $map; diff --git a/plugins/YammerImport/classes/Yammer_notice_stub.php b/plugins/YammerImport/classes/Yammer_notice_stub.php index cc52554de..e10300c4c 100644 --- a/plugins/YammerImport/classes/Yammer_notice_stub.php +++ b/plugins/YammerImport/classes/Yammer_notice_stub.php @@ -48,6 +48,23 @@ class Yammer_notice_stub extends Memcached_DataObject public $json_data; // text public $created; // datetime + /** + * Get an instance by key + * + * This is a utility method to get a single instance with a given key value. + * + * @param string $k Key to use to lookup + * @param mixed $v Value to lookup + * + * @return Yammer_notice_stub object found, or null for no hits + * + */ + + function staticGet($k, $v=null) + { + return Memcached_DataObject::staticGet('Yammer_notice_stub', $k, $v); + } + /** * Return schema definition to set this table up in onCheckSchema */ @@ -126,6 +143,16 @@ class Yammer_notice_stub extends Memcached_DataObject return array(false, false, false); } + /** + * Decode the stored data structure. + * + * @return mixed + */ + public function getData() + { + return json_decode($this->json_data, true); + } + /** * Save the native Yammer API representation of a message for the pending * import. Since they come in in reverse chronological order, we need to @@ -153,22 +180,4 @@ class Yammer_notice_stub extends Memcached_DataObject return $stub; } - - /** - * Save a mapping between a remote Yammer and local imported user. - * - * @param integer $user_id ID of the status in StatusNet - * - * @return Yammer_notice_stub new object for this value - */ - - static function retrieve($orig_id) - { - $stub = self::staticGet('id', $orig_id); - if ($stub) { - return json_decode($stub->json_data, true); - } else { - return false; - } - } } diff --git a/plugins/YammerImport/classes/Yammer_state.php b/plugins/YammerImport/classes/Yammer_state.php index 0174ead15..2f1fd7780 100644 --- a/plugins/YammerImport/classes/Yammer_state.php +++ b/plugins/YammerImport/classes/Yammer_state.php @@ -36,7 +36,6 @@ class Yammer_state extends Memcached_DataObject public $__table = 'yammer_state'; // table name public $id; // int primary_key not_null public $state; // import state key - public $request_token; // oauth request token; clear when auth is complete. public $oauth_token; // actual oauth token! clear when import is done? public $oauth_secret; // actual oauth secret! clear when import is done? public $users_page; // last page of users we've fetched @@ -46,6 +45,23 @@ class Yammer_state extends Memcached_DataObject public $created; // datetime public $modified; // datetime + /** + * Get an instance by key + * + * This is a utility method to get a single instance with a given key value. + * + * @param string $k Key to use to lookup + * @param mixed $v Value to lookup + * + * @return Yammer_state object found, or null for no hits + * + */ + + function staticGet($k, $v=null) + { + return Memcached_DataObject::staticGet('Yammer_state', $k, $v); + } + /** * Return schema definition to set this table up in onCheckSchema */ @@ -54,7 +70,6 @@ class Yammer_state extends Memcached_DataObject return array(new ColumnDef('id', 'int', null, false, 'PRI'), new ColumnDef('state', 'text'), - new ColumnDef('request_token', 'text'), new ColumnDef('oauth_token', 'text'), new ColumnDef('oauth_secret', 'text'), new ColumnDef('users_page', 'int'), @@ -78,7 +93,6 @@ class Yammer_state extends Memcached_DataObject { return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, 'state' => DB_DATAOBJECT_STR, - 'request_token' => DB_DATAOBJECT_STR, 'oauth_token' => DB_DATAOBJECT_STR, 'oauth_secret' => DB_DATAOBJECT_STR, 'users_page' => DB_DATAOBJECT_INT, diff --git a/plugins/YammerImport/lib/sn_yammerclient.php b/plugins/YammerImport/lib/sn_yammerclient.php index 830f9dabb..5da1cc5e7 100644 --- a/plugins/YammerImport/lib/sn_yammerclient.php +++ b/plugins/YammerImport/lib/sn_yammerclient.php @@ -104,7 +104,8 @@ class SN_YammerClient { $body = $this->fetchApi("api/v1/$method.json", $params); $data = json_decode($body, true); - if (!$data) { + if ($data === null) { + common_log(LOG_ERR, "Invalid JSON response from Yammer API: " . $body); throw new Exception("Invalid JSON response from Yammer API"); } return $data; @@ -161,7 +162,7 @@ class SN_YammerClient if ($this->token || $this->tokenSecret) { throw new Exception("Requesting a token, but already set up with a token"); } - $data = $this->fetch('oauth/request_token'); + $data = $this->fetchApi('oauth/request_token'); $arr = array(); parse_str($data, $arr); return $arr; @@ -176,7 +177,7 @@ class SN_YammerClient public function accessToken($verifier) { $this->verifier = $verifier; - $data = $this->fetch('oauth/access_token'); + $data = $this->fetchApi('oauth/access_token'); $this->verifier = null; $arr = array(); parse_str($data, $arr); diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php index b1d2815b9..0425b8b04 100644 --- a/plugins/YammerImport/lib/yammerimporter.php +++ b/plugins/YammerImport/lib/yammerimporter.php @@ -327,17 +327,20 @@ class YammerImporter private function findImportedUser($origId) { - return Yammer_user::staticGet('id', $origId); + $map = Yammer_user::staticGet('id', $origId); + return $map ? $map->user_id : null; } private function findImportedGroup($origId) { - return Yammer_group::staticGet('id', $origId); + $map = Yammer_group::staticGet('id', $origId); + return $map ? $map->group_id : null; } private function findImportedNotice($origId) { - return Yammer_notice::staticGet('id', $origId); + $map = Yammer_notice::staticGet('id', $origId); + return $map ? $map->notice_id : null; } private function recordImportedUser($origId, $userId) @@ -370,7 +373,7 @@ class YammerImporter // Blaaaaaarf! $known = array('Pacific Time (US & Canada)' => 'America/Los_Angeles', 'Eastern Time (US & Canada)' => 'America/New_York'); - if (array_key_exists($known, $tz)) { + if (array_key_exists($tz, $known)) { return $known[$tz]; } else { return false; diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php index 95ff78371..c4db48399 100644 --- a/plugins/YammerImport/lib/yammerrunner.php +++ b/plugins/YammerImport/lib/yammerrunner.php @@ -33,18 +33,31 @@ class YammerRunner private $client; private $importer; + /** + * Normalize our singleton state and give us a YammerRunner object to play with! + * + * @return YammerRunner + */ public static function init() { $state = Yammer_state::staticGet('id', 1); if (!$state) { - $state = new Yammer_state(); - $state->id = 1; - $state->state = 'init'; - $state->insert(); + $state = self::initState(); } return new YammerRunner($state); } + private static function initState() + { + $state = new Yammer_state(); + $state->id = 1; + $state->state = 'init'; + $state->created = common_sql_now(); + $state->modified = common_sql_now(); + $state->insert(); + return $state; + } + private function __construct($state) { $this->state = $state; @@ -55,7 +68,7 @@ class YammerRunner $this->state->oauth_token, $this->state->oauth_secret); - $this->importer = new YammerImporter($client); + $this->importer = new YammerImporter($this->client); } /** @@ -81,6 +94,8 @@ class YammerRunner /** * Check if we have work to do in iterate(). + * + * @return boolean */ public function hasWork() { @@ -88,6 +103,15 @@ class YammerRunner return in_array($this->state(), $workStates); } + /** + * Blow away any current state! + */ + public function reset() + { + $this->state->delete(); + $this->state = self::initState(); + } + /** * Start the authentication process! If all goes well, we'll get back a URL. * Have the user visit that URL, log in on Yammer and verify the importer's @@ -102,12 +126,16 @@ class YammerRunner throw ServerError("Cannot request Yammer auth; already there!"); } + $data = $this->client->requestToken(); + $old = clone($this->state); $this->state->state = 'requesting-auth'; - $this->state->request_token = $client->requestToken(); + $this->state->oauth_token = $data['oauth_token']; + $this->state->oauth_secret = $data['oauth_token_secret']; + $this->state->modified = common_sql_now(); $this->state->update($old); - return $this->client->authorizeUrl($this->state->request_token); + return $this->client->authorizeUrl($this->state->oauth_token); } /** @@ -127,12 +155,13 @@ class YammerRunner throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}"); } - $old = clone($this->state); - list($token, $secret) = $this->client->getAuthToken($verifier); - $this->state->verifier = ''; - $this->state->oauth_token = $token; - $this->state->oauth_secret = $secret; + $data = $this->client->accessToken($verifier); + $old = clone($this->state); + $this->state->state = 'import-users'; + $this->state->oauth_token = $data['oauth_token']; + $this->state->oauth_secret = $data['oauth_token_secret']; + $this->state->modified = common_sql_now(); $this->state->update($old); return true; @@ -146,8 +175,7 @@ class YammerRunner */ public function iterate() { - - switch($state->state) + switch($this->state()) { case 'init': case 'requesting-auth': @@ -188,11 +216,12 @@ class YammerRunner $this->state->state = 'import-groups'; } else { foreach ($data as $item) { - $user = $imp->importUser($item); + $user = $this->importer->importUser($item); common_log(LOG_INFO, "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)"); } $this->state->users_page = $page; } + $this->state->modified = common_sql_now(); $this->state->update($old); return true; } @@ -214,14 +243,15 @@ class YammerRunner if (count($data) == 0) { common_log(LOG_INFO, "Finished importing Yammer groups; moving on to messages."); - $this->state->state = 'import-messages'; + $this->state->state = 'fetch-messages'; } else { foreach ($data as $item) { - $group = $imp->importGroup($item); + $group = $this->importer->importGroup($item); common_log(LOG_INFO, "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)"); } $this->state->groups_page = $page; } + $this->state->modified = common_sql_now(); $this->state->update($old); return true; } @@ -248,16 +278,17 @@ class YammerRunner $data = $this->client->messages($params); $messages = $data['messages']; - if (count($data) == 0) { + if (count($messages) == 0) { common_log(LOG_INFO, "Finished fetching Yammer messages; moving on to save messages."); $this->state->state = 'save-messages'; } else { - foreach ($data as $item) { + foreach ($messages as $item) { Yammer_notice_stub::record($item['id'], $item); $oldest = $item['id']; } $this->state->messages_oldest = $oldest; } + $this->state->modified = common_sql_now(); $this->state->update($old); return true; } @@ -267,10 +298,13 @@ class YammerRunner $old = clone($this->state); $newest = intval($this->state->messages_newest); + + $stub = new Yammer_notice_stub(); if ($newest) { - $stub->addWhere('id > ' . $newest); + $stub->whereAdd('id > ' . $newest); } $stub->limit(20); + $stub->orderBy('id'); $stub->find(); if ($stub->N == 0) { @@ -278,13 +312,14 @@ class YammerRunner $this->state->state = 'done'; } else { while ($stub->fetch()) { - $item = json_decode($stub->json_data); + $item = $stub->getData(); $notice = $this->importer->importNotice($item); common_log(LOG_INFO, "Imported Yammer notice " . $item['id'] . " as $notice->id"); $newest = $item['id']; } $this->state->messages_newest = $newest; } + $this->state->modified = common_sql_now(); $this->state->update($old); return true; } diff --git a/plugins/YammerImport/scripts/yammer-import.php b/plugins/YammerImport/scripts/yammer-import.php index 24307d6cd..1491cfd30 100644 --- a/plugins/YammerImport/scripts/yammer-import.php +++ b/plugins/YammerImport/scripts/yammer-import.php @@ -4,41 +4,60 @@ if (php_sapi_name() != 'cli') { die('no'); } + define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__))))); +$longoptions = array('verify=', 'reset'); require INSTALLDIR . "/scripts/commandline.inc"; +echo "Checking current state...\n"; $runner = YammerRunner::init(); +if (have_option('reset')) { + echo "Resetting Yammer import state...\n"; + $runner->reset(); +} + switch ($runner->state()) { case 'init': + echo "Requesting authentication to Yammer API...\n"; $url = $runner->requestAuth(); echo "Log in to Yammer at the following URL and confirm permissions:\n"; echo "\n"; echo " $url\n"; echo "\n"; - echo "Pass the resulting code back by running:\n" - echo "\n" - echo " php yammer-import.php --auth=####\n"; + echo "Pass the resulting code back by running:\n"; + echo "\n"; + echo " php yammer-import.php --verify=####\n"; echo "\n"; break; case 'requesting-auth': - if (empty($options['auth'])) { - echo "Please finish authenticating!\n"; - break; + if (!have_option('verify')) { + echo "Awaiting authentication...\n"; + echo "\n"; + echo "If you need to start over, reset the state:\n"; + echo "\n"; + echo " php yammer-import.php --reset\n"; + echo "\n"; + exit(1); } - $runner->saveAuthToken($options['auth']); + echo "Saving final authentication token for Yammer API...\n"; + $runner->saveAuthToken(get_option_value('verify')); // Fall through... default: - while (true) { - echo "... {$runner->state->state}\n"; + while ($runner->hasWork()) { + echo "... {$runner->state()}\n"; if (!$runner->iterate()) { - echo "... done.\n"; - break; + echo "FAIL??!?!?!\n"; } } + if ($runner->isDone()) { + echo "... done.\n"; + } else { + echo "... no more import work scheduled.\n"; + } break; -} \ No newline at end of file +} -- cgit v1.2.3-54-g00ecf From 782046dc016ac70148447db94c2c45fbe68c056e Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 28 Sep 2010 13:02:34 -0700 Subject: YammerImporter: 'group_id' field not always present in message data; avoid notice warning when checking it if it's missing --- plugins/YammerImport/lib/yammerimporter.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/YammerImport/lib/yammerimporter.php') diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php index 0425b8b04..0ce8ffb48 100644 --- a/plugins/YammerImport/lib/yammerimporter.php +++ b/plugins/YammerImport/lib/yammerimporter.php @@ -287,7 +287,7 @@ class YammerImporter } $options['created'] = $this->timestamp($item['created_at']); - if ($item['group_id']) { + if (!empty($item['group_id'])) { $groupId = $this->findImportedGroup($item['group_id']); if ($groupId) { $options['groups'] = array($groupId); -- cgit v1.2.3-54-g00ecf From 3a1f1a49b22c2be4234a78cd36016a373d3ca5fc Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 28 Sep 2010 13:07:55 -0700 Subject: YammerImport: set some more default fields for user_group registration (still need a group admin tho) --- plugins/YammerImport/lib/yammerimporter.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'plugins/YammerImport/lib/yammerimporter.php') diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php index 0ce8ffb48..ae916dc22 100644 --- a/plugins/YammerImport/lib/yammerimporter.php +++ b/plugins/YammerImport/lib/yammerimporter.php @@ -249,8 +249,11 @@ class YammerImporter $options['mainpage'] = common_local_url('showgroup', array('nickname' => $options['nickname'])); + // Set some default vals or User_group::register will whine + $options['homepage'] = ''; + $options['location'] = ''; + $options['aliases'] = array(); // @fixme what about admin user for the group? - // bio? homepage etc? aliases? $options['local'] = true; return array('orig_id' => $origId, -- cgit v1.2.3-54-g00ecf From c62e4d08008faf7fdfb67f0e5eda3db6d8d1ceef Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 28 Sep 2010 13:24:25 -0700 Subject: YammerImport: fix for importing over existing user/group nicknames; copies the new data if available onto the existing entry --- plugins/YammerImport/lib/yammerimporter.php | 37 ++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) (limited to 'plugins/YammerImport/lib/yammerimporter.php') diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php index ae916dc22..ed1191525 100644 --- a/plugins/YammerImport/lib/yammerimporter.php +++ b/plugins/YammerImport/lib/yammerimporter.php @@ -41,13 +41,22 @@ class YammerImporter function importUser($item) { $data = $this->prepUser($item); + $nickname = $data['options']['nickname']; $profileId = $this->findImportedUser($data['orig_id']); if ($profileId) { return Profile::staticGet('id', $profileId); } else { - $user = User::register($data['options']); - $profile = $user->getProfile(); + $user = User::staticGet('nickname', $nickname); + if ($user) { + common_log(LOG_WARN, "Copying Yammer profile info onto existing user $nickname"); + $profile = $user->getProfile(); + $this->savePropertiesOn($profile, $data['options'], + array('fullname', 'homepage', 'bio', 'location')); + } else { + $user = User::register($data['options']); + $profile = $user->getProfile(); + } if ($data['avatar']) { try { $this->saveAvatar($data['avatar'], $profile); @@ -69,12 +78,21 @@ class YammerImporter function importGroup($item) { $data = $this->prepGroup($item); + $nickname = $data['options']['nickname']; $groupId = $this->findImportedGroup($data['orig_id']); if ($groupId) { return User_group::staticGet('id', $groupId); } else { - $group = User_group::register($data['options']); + $local = Local_group::staticGet('nickname', $nickname); + if ($local) { + common_log(LOG_WARN, "Copying Yammer group info onto existing group $nickname"); + $group = User_group::staticGet('id', $local->group_id); + $this->savePropertiesOn($group, $data['options'], + array('fullname', 'description')); + } else { + $group = User_group::register($data['options']); + } if ($data['avatar']) { try { $this->saveAvatar($data['avatar'], $group); @@ -87,6 +105,19 @@ class YammerImporter } } + private function savePropertiesOn($target, $options, $propList) + { + $changed = 0; + $orig = clone($target); + foreach ($propList as $prop) { + if (!empty($options[$prop]) && $target->$prop != $options[$prop]) { + $target->$prop = $options[$prop]; + $changed++; + } + } + $target->update($orig); + } + /** * Load or create an imported notice from Yammer data. * -- cgit v1.2.3-54-g00ecf From 62d9b66dffb0000122cd611a9d991da008f9a527 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 28 Sep 2010 15:45:00 -0700 Subject: Made YammerImport more robust against errors; can now pause/resume/reset the import state from the admin interface. --- plugins/YammerImport/actions/yammeradminpanel.php | 13 +++++++ plugins/YammerImport/classes/Yammer_state.php | 3 ++ plugins/YammerImport/css/admin.css | 8 ++++- plugins/YammerImport/lib/yammerimporter.php | 1 + plugins/YammerImport/lib/yammerprogressform.php | 39 ++++++++++++++++++--- plugins/YammerImport/lib/yammerqueuehandler.php | 23 +++++++------ plugins/YammerImport/lib/yammerrunner.php | 42 ++++++++++++++++++++++- 7 files changed, 113 insertions(+), 16 deletions(-) (limited to 'plugins/YammerImport/lib/yammerimporter.php') diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php index 13960d905..3faf390ac 100644 --- a/plugins/YammerImport/actions/yammeradminpanel.php +++ b/plugins/YammerImport/actions/yammeradminpanel.php @@ -73,6 +73,7 @@ class YammeradminpanelAction extends AdminPanelAction { // @fixme move this to saveSettings and friends? if ($_SERVER['REQUEST_METHOD'] == 'POST') { + StatusNet::setApi(true); // short error pages :P $this->checkSessionToken(); if ($this->subaction == 'change-apikey') { $form = new YammerApiKeyForm($this); @@ -97,6 +98,18 @@ class YammeradminpanelAction extends AdminPanelAction $this->runner->startBackgroundImport(); $form = new YammerProgressForm($this, $this->runner); + } else if ($this->subaction == 'pause-import') { + $this->runner->recordError(_m('Paused from admin panel.')); + $form = $this->statusForm(); + } else if ($this->subaction == 'continue-import') { + $this->runner->clearError(); + $this->runner->startBackgroundImport(); + $form = $this->statusForm(); + } else if ($this->subaction == 'abort-import') { + $this->runner->reset(); + $form = $this->statusForm(); + } else if ($this->subaction == 'progress') { + $form = $this->statusForm(); } else { throw new ClientException('Invalid POST'); } diff --git a/plugins/YammerImport/classes/Yammer_state.php b/plugins/YammerImport/classes/Yammer_state.php index 2f1fd7780..88bd693bf 100644 --- a/plugins/YammerImport/classes/Yammer_state.php +++ b/plugins/YammerImport/classes/Yammer_state.php @@ -36,6 +36,7 @@ class Yammer_state extends Memcached_DataObject public $__table = 'yammer_state'; // table name public $id; // int primary_key not_null public $state; // import state key + public $last_error; // text of last-encountered error, if any public $oauth_token; // actual oauth token! clear when import is done? public $oauth_secret; // actual oauth secret! clear when import is done? public $users_page; // last page of users we've fetched @@ -70,6 +71,7 @@ class Yammer_state extends Memcached_DataObject return array(new ColumnDef('id', 'int', null, false, 'PRI'), new ColumnDef('state', 'text'), + new ColumnDef('last_error', 'text'), new ColumnDef('oauth_token', 'text'), new ColumnDef('oauth_secret', 'text'), new ColumnDef('users_page', 'int'), @@ -93,6 +95,7 @@ class Yammer_state extends Memcached_DataObject { return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, 'state' => DB_DATAOBJECT_STR, + 'last_error' => DB_DATAOBJECT_STR, 'oauth_token' => DB_DATAOBJECT_STR, 'oauth_secret' => DB_DATAOBJECT_STR, 'users_page' => DB_DATAOBJECT_INT, diff --git a/plugins/YammerImport/css/admin.css b/plugins/YammerImport/css/admin.css index 9c99a0b88..80e0e038a 100644 --- a/plugins/YammerImport/css/admin.css +++ b/plugins/YammerImport/css/admin.css @@ -52,4 +52,10 @@ .magiclink { margin-left: 40px; -} \ No newline at end of file +} + +fieldset.import-error { + margin-top: 12px; + margin-bottom: 0px !important; + background-color: #fee !important; +} diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php index ed1191525..80cbcff8e 100644 --- a/plugins/YammerImport/lib/yammerimporter.php +++ b/plugins/YammerImport/lib/yammerimporter.php @@ -132,6 +132,7 @@ class YammerImporter if ($noticeId) { return Notice::staticGet('id', $noticeId); } else { + $notice = Notice::staticGet('uri', $data['options']['uri']); $content = $data['content']; $user = User::staticGet($data['profile']); diff --git a/plugins/YammerImport/lib/yammerprogressform.php b/plugins/YammerImport/lib/yammerprogressform.php index 776efa100..add8d9ab2 100644 --- a/plugins/YammerImport/lib/yammerprogressform.php +++ b/plugins/YammerImport/lib/yammerprogressform.php @@ -9,7 +9,7 @@ class YammerProgressForm extends Form */ function id() { - return 'yammer-progress'; + return 'yammer-progress-form'; } /** @@ -39,8 +39,11 @@ class YammerProgressForm extends Form */ function formData() { + $this->out->hidden('subaction', 'progress'); + $runner = YammerRunner::init(); + $error = $runner->lastError(); $userCount = $runner->countUsers(); $groupCount = $runner->countGroups(); $fetchedCount = $runner->countFetchedNotices(); @@ -86,7 +89,13 @@ class YammerProgressForm extends Form $steps = array_keys($labels); $currentStep = array_search($runner->state(), $steps); - $this->out->elementStart('fieldset', array('class' => 'yammer-import')); + $classes = array('yammer-import'); + if ($error) { + $classes[] = 'yammer-error'; + } else { + $classes[] = 'yammer-running'; + } + $this->out->elementStart('fieldset', array('class' => implode(' ', $classes))); $this->out->element('legend', array(), _m('Import status')); foreach ($steps as $step => $state) { if ($state == 'init') { @@ -104,7 +113,8 @@ class YammerProgressForm extends Form $this->progressBar($state, 'progress', $labels[$state]['label'], - $labels[$state]['progress']); + $labels[$state]['progress'], + $error); } else { // This step has not yet been done. $this->progressBar($state, @@ -116,13 +126,34 @@ class YammerProgressForm extends Form $this->out->elementEnd('fieldset'); } - private function progressBar($state, $class, $label, $status) + private function progressBar($state, $class, $label, $status, $error=null) { // @fixme prettify ;) $this->out->elementStart('div', array('class' => "import-step import-step-$state $class")); $this->out->element('div', array('class' => 'import-label'), $label); $this->out->element('div', array('class' => 'import-status'), $status); + if ($class == 'progress') { + if ($state == 'done') { + $this->out->submit('abort-import', _m('Reset import state')); + } else { + if ($error) { + $this->errorBox($error); + } else { + $this->out->submit('pause-import', _m('Pause import')); + } + } + } $this->out->elementEnd('div'); } + private function errorBox($msg) + { + $errline = sprintf(_m('Encountered error "%s"'), $msg); + $this->out->elementStart('fieldset', array('class' => 'import-error')); + $this->out->element('legend', array(), _m('Paused')); + $this->out->element('p', array(), $errline); + $this->out->submit('continue-import', _m('Continue')); + $this->out->submit('abort-import', _m('Abort import')); + $this->out->elementEnd('fieldset'); + } } diff --git a/plugins/YammerImport/lib/yammerqueuehandler.php b/plugins/YammerImport/lib/yammerqueuehandler.php index acc807311..0c4e8aec1 100644 --- a/plugins/YammerImport/lib/yammerqueuehandler.php +++ b/plugins/YammerImport/lib/yammerqueuehandler.php @@ -38,21 +38,24 @@ class YammerQueueHandler extends QueueHandler { $runner = YammerRunner::init(); if ($runner->hasWork()) { - if ($runner->iterate()) { - if ($runner->hasWork()) { - // More to do? Shove us back on the queue... - $runner->startBackgroundImport(); + try { + if ($runner->iterate()) { + if ($runner->hasWork()) { + // More to do? Shove us back on the queue... + $runner->startBackgroundImport(); + } + } + } catch (Exception $e) { + try { + $runner->recordError($e->getMessage()); + } catch (Exception $f) { + common_log(LOG_ERR, "Error while recording error in Yammer background import: " . $e->getMessage() . " " . $f->getMessage()); } - return true; - } else { - // Something failed? - // @fixme should we be trying again here, or should we give warning? - return false; } } else { // We're done! common_log(LOG_INFO, "Yammer import has no work to do at this time; discarding."); - return true; } + return true; } } diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php index e0aec0d16..3e53f3361 100644 --- a/plugins/YammerImport/lib/yammerrunner.php +++ b/plugins/YammerImport/lib/yammerrunner.php @@ -298,7 +298,10 @@ class YammerRunner $this->state->state = 'save-messages'; } else { foreach ($messages as $item) { - Yammer_notice_stub::record($item['id'], $item); + $stub = Yammer_notice_stub::staticGet($item['id']); + if (!$stub) { + Yammer_notice_stub::record($item['id'], $item); + } $oldest = $item['id']; } $this->state->messages_oldest = $oldest; @@ -395,4 +398,41 @@ class YammerRunner $qm->enqueue('YammerImport', 'yammer'); } + /** + * Record an error condition from a background run, which we should + * display in progress state for the admin. + * + * @param string $msg + */ + public function recordError($msg) + { + // HACK HACK HACK + try { + $temp = new Yammer_state(); + $temp->query('ROLLBACK'); + } catch (Exception $e) { + common_log(LOG_ERR, 'Exception while confirming rollback while recording error: ' . $e->getMessage()); + } + $old = clone($this->state); + $this->state->last_error = $msg; + $this->state->update($old); + } + + /** + * Clear the error state. + */ + public function clearError() + { + $this->recordError(''); + } + + /** + * Get the last recorded background error message, if any. + * + * @return string + */ + public function lastError() + { + return $this->state->last_error; + } } -- cgit v1.2.3-54-g00ecf