From 2d5c7d50571a797ba178a966061d679cb4bb8c9e Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Mon, 28 Oct 2013 16:43:57 +0800 Subject: [PATCH 01/14] MDL-42666 repository_boxnet: Convert repository to APIv2 --- lib/boxlib.php | 168 ++++++- lib/upgrade.txt | 3 + .../boxnet/lang/en/repository_boxnet.php | 14 +- repository/boxnet/lib.php | 419 ++++++++++++------ 4 files changed, 450 insertions(+), 154 deletions(-) diff --git a/lib/boxlib.php b/lib/boxlib.php index 974124f89ff..ed8f175addb 100644 --- a/lib/boxlib.php +++ b/lib/boxlib.php @@ -15,10 +15,9 @@ // along with Moodle. If not, see . /** - * Box REST Client Library for PHP5 Developers + * Box.net client. * - * - * @package moodlecore + * @package core * @author James Levy * @link http://enabled.box.net * @access public @@ -27,10 +26,171 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +defined('MOODLE_INTERNAL') || die(); +require_once($CFG->libdir . '/oauthlib.php'); + /** - * @package moodlecore + * Box.net client class. + * + * @package core + * @copyright 2013 Frédéric Massart + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class boxnet_client extends oauth2_client { + + /** @const API URL */ + const API = 'https://api.box.com/2.0'; + + /** + * Return authorize URL. + * + * @return string + */ + protected function auth_url() { + return 'https://www.box.com/api/oauth2/authorize'; + } + + /** + * Download the file. + * + * @param int $fileid File ID. + * @param string $path Path to download the file to. + * @return bool Success or not. + */ + public function download_file($fileid, $path) { + $result = $this->download_one($this->make_url("/files/$fileid/content"), array(), array('filepath' => $path)); + return ($result === true && $this->info['http_code'] === 200); + } + + /** + * Get info of a file. + * + * @param int $fileid File ID. + * @return object + */ + public function get_file_info($fileid) { + $result = $this->request($this->make_url("/files/$fileid")); + return json_decode($result); + } + + /** + * Get a folder content. + * + * @param int $folderid Folder ID. + * @return object + */ + public function get_folder_items($folderid = 0) { + $result = $this->request($this->make_url("/folders/$folderid/items", + array('fields' => 'id,name,type,modified_at,size,owned_by'))); + return json_decode($result); + } + + /** + * Log out. + * + * @return void + */ + public function log_out() { + if ($this->accesstoken) { + $params = array( + 'client_id' => $this->clientid, + 'client_secret' => $this->clientsecret, + 'token' => $this->accesstoken->token + ); + $this->post($this->get_revoke_url(), $params); + } + parent::log_out(); + } + + /** + * Build a request URL. + * + * @param string $uri The URI to request. + * @param array $params Query string parameters. + * @return string + */ + protected function make_url($uri, $params = array()) { + $url = new moodle_url(self::API . '/' . ltrim($uri, '/'), $params); + return $url->out(false); + } + + /** + * Return the revoke URL. + * + * @return string + */ + protected function revoke_url() { + return 'https://www.box.com/api/oauth2/revoke'; + } + + /** + * Share a file and return the link to it. + * + * @param string $fileid The file ID. + * @param bool $businesscheck Whether or not to check if the user can share files, has a business account. + * @return object + */ + public function share_file($fileid, $businesscheck = true) { + // Sharing the file, this requires a PUT request with data within it. We cannot use + // the standard PUT request 'CURLOPT_PUT' because it expects a file. + $data = array('shared_link' => array('access' => 'open', 'permissions' => + array('can_download' => true, 'can_preview' => true))); + $options = array( + 'CURLOPT_CUSTOMREQUEST' => 'PUT', + 'CURLOPT_POSTFIELDS' => json_encode($data) + ); + $result = $this->request($this->make_url("/files/$fileid"), $options); + $result = json_decode($result); + + if ($businesscheck) { + // Checks that the user has the right to share the file. If not, throw an exception. + $this->resetopt(); + $this->resetHeader(); + $this->head($result->shared_link->download_url); + $info = $this->get_info(); + if ($info['http_code'] == 403) { + throw new moodle_exception('No permission to share the file'); + } + } + + return $result->shared_link; + } + + /** + * Search. + * + * @return object + */ + public function search($query) { + $result = $this->request($this->make_url('/search', array('query' => $query, 'limit' => 50, 'offset' => 0))); + return json_decode($result); + } + + /** + * Return token URL. + * + * @return string + */ + protected function token_url() { + return 'https://www.box.com/api/oauth2/token'; + } + +} + +/** + * Box REST Client Library for PHP5 Developers. + * + * Deprecation note: As of the 14th of December 2013 Box.net APIv1, used by this class, + * is reaching its end of life. Please use boxnet_client() instead. + * + * @package core + * @author James Levy + * @link http://enabled.box.net + * @access public + * @version 1.0 * @copyright copyright Box.net 2007 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @deprecated since 2.6, 2.5.3, 2.4.7 */ class boxclient { /** @var string */ diff --git a/lib/upgrade.txt b/lib/upgrade.txt index 2db1ecd100b..7807cae0c08 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -56,6 +56,9 @@ information provided here is intended especially for developers. to 6 hours from 24 hours because etags and x-sendfile support should make file serving less expensive. * Date format locale charset for windows server will come from calendar type and for gregorian it will use lang file. +* The library to interact with Box.net (class boxclient) is only compatible with their APIv1 which + reaches its end of life on the 14th of Dec. You should migrate your scripts to make usage of the + new class boxnet_client(). Note that the method names and return values have changed. DEPRECATIONS: Various previously deprecated functions have now been altered to throw DEBUG_DEVELOPER debugging notices diff --git a/repository/boxnet/lang/en/repository_boxnet.php b/repository/boxnet/lang/en/repository_boxnet.php index 27defbeceab..3e920c2b4fd 100644 --- a/repository/boxnet/lang/en/repository_boxnet.php +++ b/repository/boxnet/lang/en/repository_boxnet.php @@ -25,15 +25,12 @@ $string['apikey'] = 'API key'; $string['boxnet:view'] = 'View box.net repository'; +$string['cannotcreatereference'] = 'Cannot create a reference, not enough permissions to share the file on Box.net.'; +$string['clientid'] = 'Client ID'; +$string['clientsecret'] = 'Client secret'; $string['configplugin'] = 'Box.net configuration'; -$string['callbackurl'] = 'Redirect URL'; -$string['callbackurltext'] = '1. Visit www.box.net/developers/services again. -2. Make sure you set the redirect URL of this box.net service to {$a}.'; -$string['callbackwarning'] = '1. Get a Box.net API from www.box.net/developers/services for this Moodle site. -2. Enter the Box.net API key here, then click Save and then return to this page. You will see that Moodle has generated a redirect URL for you. -3. Edit your Box.net details on the box.net website again and set the redirect URL.'; - -$string['information'] = 'Get an API key from the Box.net developer page for your Moodle site.'; +$string['filesourceinfo'] = 'Box.net ({$a->fullname}): {$a->filename}'; +$string['information'] = 'Get a client ID and secret from the Box.net developer page for your Moodle site.'; $string['invalidpassword'] = 'Invalid password'; $string['nullfilelist'] = 'There are no files in this repository'; $string['password'] = 'Password'; @@ -42,3 +39,4 @@ $string['pluginname'] = 'Box.net'; $string['saved'] = 'Box.net data saved'; $string['shareurl'] = 'Share URL'; $string['username'] = 'Username for Box.net'; +$string['warninghttps'] = 'Box.net requires your website to be using HTTPS in order for the repository to work.'; diff --git a/repository/boxnet/lib.php b/repository/boxnet/lib.php index 830b5370752..a597dc21fc5 100644 --- a/repository/boxnet/lib.php +++ b/repository/boxnet/lib.php @@ -30,11 +30,28 @@ require_once($CFG->libdir . '/boxlib.php'); * * @since 2.0 * @package repository_boxnet - * @copyright 2009 Dongsheng Cai {@link http://dongsheng.org} + * @copyright 2010 Dongsheng Cai {@link http://dongsheng.org} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class repository_boxnet extends repository { - private $boxclient; + + /** @const MANAGE_URL Manage URL. */ + const MANAGE_URL = 'https://app.box.com/files'; + + /** @const SESSION_PREFIX Key used to store information in the session. */ + const SESSION_PREFIX = 'repository_boxnet'; + + /** @var string Client ID */ + protected $clientid; + + /** @var string Client secret */ + protected $clientsecret; + + /** @var string Access token */ + protected $accesstoken; + + /** @var object Box.net object */ + protected $boxnetclient; /** * Constructor @@ -45,22 +62,87 @@ class repository_boxnet extends repository { */ public function __construct($repositoryid, $context = SYSCONTEXTID, $options = array()) { parent::__construct($repositoryid, $context, $options); - $this->api_key = $this->get_option('api_key'); - $this->setting_prefix = 'boxnet_'; - $this->auth_token = get_user_preferences($this->setting_prefix.'_auth_token', ''); - $this->logged = false; - if (!empty($this->auth_token)) { - $this->logged = true; - } - // already logged - if(!empty($this->logged)) { - if(empty($this->boxclient)) { - $this->boxclient = new boxclient($this->api_key, $this->auth_token); + $clientid = get_config('boxnet', 'clientid'); + $clientsecret = get_config('boxnet', 'clientsecret'); + $returnurl = new moodle_url('/repository/repository_callback.php'); + $returnurl->param('callback', 'yes'); + $returnurl->param('repo_id', $this->id); + $returnurl->param('sesskey', sesskey()); + + $this->boxnetclient = new boxnet_client($clientid, $clientsecret, $returnurl, ''); + } + + /** + * Construct a breadcrumb from a path. + * + * @param string $fullpath Path containing multiple parts separated by slashes. + * @return array Array expected to be generated in {@link self::get_listing()}. + */ + protected function build_breadcrumb($fullpath) { + $breadcrumb = array(array( + 'name' => get_string('pluginname', 'repository_boxnet'), + 'path' => '' + )); + $breadcrumbpath = ''; + $crumbs = explode('/', $fullpath); + foreach ($crumbs as $crumb) { + if (empty($crumb)) { + // That is probably the root crumb, we've already added it. + continue; } - } else { - $this->boxclient = new boxclient($this->api_key); + list($unused, $tosplit) = explode(':', $crumb, 2); + if (strpos($tosplit, '|') !== false) { + list($id, $crumbname) = explode('|', $tosplit, 2); + } else { + $crumbname = $tosplit; + } + $breadcrumbpath .= '/' . $crumb; + $breadcrumb[] = array( + 'name' => urldecode($crumbname), + 'path' => $breadcrumbpath + ); } + return $breadcrumb; + } + + /** + * Build a part of the path. + * + * This is used to construct the path that the user is currently browsing. + * It must contain a 'type', and a 'value'. Then it can also contain a + * 'name' which is very useful to prevent extra queries to get the name only. + * + * See {@link self::split_part} to extra the information from a part. + * + * @param string $type Type of part, typically 'folder' or 'search'. + * @param string $value The value of the part, eg. a folder ID or search terms. + * @param string $name The name of the part. + * @return string type:value or type:value|name + */ + protected function build_part($type, $value, $name = '') { + $return = $type . ':' . urlencode($value); + if ($name !== '') { + $return .= '|' . urlencode($name); + } + return $return; + } + + /** + * Extract information from a part of path. + * + * @param string $part value generated from {@link self::build_parth()}. + * @return array containing type, value and name. + */ + protected function split_part($part) { + list($type, $tosplit) = explode(':', $part); + $name = ''; + if (strpos($tosplit, '|') !== false) { + list($value, $name) = explode('|', $tosplit, 2); + } else { + $value = $tosplit; + } + return array($type, urldecode($value), urldecode($name)); } /** @@ -69,7 +151,7 @@ class repository_boxnet extends repository { * @return boolean */ public function check_login() { - return $this->logged; + return $this->boxnetclient->is_logged_in(); } /** @@ -78,42 +160,12 @@ class repository_boxnet extends repository { * @return string */ public function logout() { - // reset auth token - set_user_preference($this->setting_prefix . '_auth_token', ''); + if ($this->check_login()) { + $this->boxnetclient->log_out(); + } return $this->print_login(); } - /** - * Save settings - * - * @param array $options - * @return mixed - */ - public function set_option($options = array()) { - if (!empty($options['api_key'])) { - set_config('api_key', trim($options['api_key']), 'boxnet'); - } - unset($options['api_key']); - $ret = parent::set_option($options); - return $ret; - } - - /** - * Get settings - * - * @param string $config - * @return mixed - */ - public function get_option($config = '') { - if($config==='api_key') { - return trim(get_config('boxnet', 'api_key')); - } else { - $options['api_key'] = trim(get_config('boxnet', 'api_key')); - } - $options = parent::get_option($config); - return $options; - } - /** * Search files from box.net * @@ -121,29 +173,33 @@ class repository_boxnet extends repository { * @return mixed */ public function search($search_text, $page = 0) { - global $OUTPUT; - $list = array(); - $ret = array(); - $tree = $this->boxclient->getAccountTree(); - if (!empty($tree)) { - $filenames = $tree['file_name']; - $fileids = $tree['file_id']; - $filesizes = $tree['file_size']; - $filedates = $tree['file_date']; - $fileicon = $tree['thumbnail']; - foreach ($filenames as $n=>$v){ - if(strstr(strtolower($v), strtolower($search_text)) !== false) { - $list[] = array('title'=>$v, - 'size'=>$filesizes[$n], - 'date'=>$filedates[$n], - 'source'=>'https://www.box.com/api/1.0/download/' - .$this->auth_token.'/'.$fileids[$n], - 'thumbnail' => $OUTPUT->pix_url(file_extension_icon($v, 90))->out(false)); - } + return $this->get_listing($this->build_part('search', $search_text)); + } + + /** + * Downloads a repository file and saves to a path. + * + * @param string $ref reference to the file + * @param string $filename to save file as + * @return array + */ + public function get_file($ref, $filename = '') { + $ref = unserialize(self::convert_to_valid_reference($ref)); + $path = $this->prepare_file($filename); + if (!empty($ref->downloadurl)) { + $c = new curl(); + $result = $c->download_one($ref->downloadurl, null, array('filepath' => $filename, + 'timeout' => self::GETFILE_TIMEOUT, 'followlocation' => true)); + $info = $c->get_info(); + if ($result !== true || !isset($info['http_code']) || $info['http_code'] != 200) { + throw new moodle_exception('errorwhiledownload', 'repository', '', $result); + } + } else { + if (!$this->boxnetclient->download_file($ref->fileid, $path)) { + throw new moodle_exception('cannotdownload', 'repository'); } } - $ret['list'] = array_filter($list, array($this, 'filter')); - return $ret; + return array('path' => $path); } /** @@ -153,16 +209,65 @@ class repository_boxnet extends repository { * @param string $page * @return mixed */ - public function get_listing($path = '/', $page = ''){ - $list = array(); - $ret = array(); + public function get_listing($fullpath = '', $page = ''){ + global $OUTPUT; + + $ret = array(); $ret['list'] = array(); - $tree = $this->boxclient->getfiletree($path); - $ret['manage'] = 'http://www.box.com/files'; - $ret['path'] = array(array('name'=>'Root', 'path'=>0)); - if(!empty($tree)) { - $ret['list'] = array_filter($tree, array($this, 'filter')); + $ret['manage'] = self::MANAGE_URL; + $ret['dynload'] = true; + + $crumbs = explode('/', $fullpath); + $path = array_pop($crumbs); + + if (empty($path)) { + $type = 'folder'; + $pathid = 0; + $pathname = get_string('pluginname', 'repository_boxnet'); + } else { + list($type, $pathid, $pathname) = $this->split_part($path); } + + $ret['path'] = $this->build_breadcrumb($fullpath); + $folders = array(); + $files = array(); + + if ($type == 'search') { + $result = $this->boxnetclient->search($pathname); + } else { + $result = $this->boxnetclient->get_folder_items($pathid); + } + foreach ($result->entries as $item) { + if ($item->type == 'folder') { + $folders[$item->name . ':' . $item->id] = array( + 'title' => $item->name, + 'path' => $fullpath . '/' . $this->build_part('folder', $item->id, $item->name), + 'date' => strtotime($item->modified_at), + 'thumbnail' => $OUTPUT->pix_url(file_folder_icon(64))->out(false), + 'thumbnail_height' => 64, + 'thumbnail_width' => 64, + 'children' => array(), + 'size' => $item->size, + ); + } else { + $files[$item->name . ':' . $item->id] = array( + 'title' => $item->name, + 'source' => $this->build_part('file', $item->id, $item->name), + 'size' => $item->size, + 'date' => strtotime($item->modified_at), + 'thumbnail' => $OUTPUT->pix_url(file_extension_icon($item->name, 64))->out(false), + 'thumbnail_height' => 64, + 'thumbnail_width' => 64, + 'author' => $item->owned_by->name, + ); + } + } + + collatorlib::ksort($folders, core_collator::SORT_NATURAL); + collatorlib::ksort($files, core_collator::SORT_NATURAL); + $ret['list'] = array_merge($folders, $files); + $ret['list'] = array_filter($ret['list'], array($this, 'filter')); + return $ret; } @@ -172,24 +277,16 @@ class repository_boxnet extends repository { * @return array */ public function print_login(){ - $t = $this->boxclient->getTicket(); + $url = $this->boxnetclient->get_login_url(); if ($this->options['ajax']) { + $ret = array(); $popup_btn = new stdClass(); $popup_btn->type = 'popup'; - $popup_btn->url = ' https://www.box.com/api/1.0/auth/' . $t['ticket']; - - $ret = array(); + $popup_btn->url = $url->out(false); $ret['login'] = array($popup_btn); return $ret; } else { - echo ''; - echo ''; - echo ''; - echo ''; - echo ''; - echo ''; - echo '
'; - echo ''; + echo html_writer::link($url, get_string('login', 'repository'), array('target' => '_blank')); } } @@ -199,15 +296,14 @@ class repository_boxnet extends repository { * @return array */ public static function get_type_option_names() { - return array('api_key', 'pluginname'); + return array('clientid', 'clientsecret', 'pluginname'); } /** - * Store the auth token returned by box.net + * Catch the request token. */ public function callback() { - $this->auth_token = optional_param('auth_token', '', PARAM_TEXT); - set_user_preference($this->setting_prefix . '_auth_token', $this->auth_token); + $this->boxnetclient->is_logged_in(); } /** @@ -219,32 +315,23 @@ class repository_boxnet extends repository { public static function type_config_form($mform, $classname = 'repository') { global $CFG; parent::type_config_form($mform); - $public_account = get_config('boxnet', 'public_account'); - $api_key = get_config('boxnet', 'api_key'); - if (empty($api_key)) { - $api_key = ''; - } - $strrequired = get_string('required'); - $mform->addElement('text', 'api_key', get_string('apikey', 'repository_boxnet'), array('value'=>$api_key,'size' => '40')); - $mform->addRule('api_key', $strrequired, 'required', null, 'client'); - $mform->setType('api_key', PARAM_RAW_TRIMMED); - $mform->addElement('static', null, '', get_string('information','repository_boxnet')); - //retrieve the flickr instances - $params = array(); - $params['context'] = array(); - //$params['currentcontext'] = $this->context; - $params['onlyvisible'] = false; - $params['type'] = 'boxnet'; - $instances = repository::get_instances($params); - if (empty($instances)) { - $callbackurl = get_string('callbackwarning', 'repository_boxnet'); - $mform->addElement('static', null, '', $callbackurl); - } else { - $instance = array_shift($instances); - $callbackurl = $CFG->wwwroot.'/repository/repository_callback.php?repo_id='.$instance->id; - $mform->addElement('static', 'callbackurl', '', get_string('callbackurltext', 'repository_boxnet', $callbackurl)); - } + $clientid = get_config('boxnet', 'clientid'); + $clientsecret = get_config('boxnet', 'clientsecret'); + $strrequired = get_string('required'); + + $mform->addElement('text', 'clientid', get_string('clientid', 'repository_boxnet'), + array('value' => $clientid, 'size' => '40')); + $mform->addRule('clientid', $strrequired, 'required', null, 'client'); + $mform->setType('clientid', PARAM_RAW_TRIMMED); + + $mform->addElement('text', 'clientsecret', get_string('clientsecret', 'repository_boxnet'), + array('value' => $clientsecret, 'size' => '40')); + $mform->addRule('clientsecret', $strrequired, 'required', null, 'client'); + $mform->setType('clientsecret', PARAM_RAW_TRIMMED); + + $mform->addElement('static', null, '', get_string('information', 'repository_boxnet')); + $mform->addElement('static', null, '', get_string('warninghttps', 'repository_boxnet')); } /** @@ -256,6 +343,25 @@ class repository_boxnet extends repository { return FILE_INTERNAL | FILE_EXTERNAL | FILE_REFERENCE; } + /** + * Convert a reference to the new reference style. + * + * While converting Box.net to APIv2 we introduced a new format for + * file references, see {@link self::get_file_reference()}. This function + * ensures that the format is always the same regardless of the whether + * the reference was from APIv1 or v2. + * + * @param mixed $reference File reference. + * @return stdClass Valid file reference. + */ + public static function convert_to_valid_reference($reference) { + if (strpos($reference, 'http') === 0) { + // It is faster to check if the reference is a URL rather than trying to unserialize it. + $reference = serialize((object) array('downloadurl' => $reference, 'fileid' => '', 'filename' => '', 'userid' => '')); + } + return $reference; + } + /** * Prepare file reference information * @@ -263,19 +369,56 @@ class repository_boxnet extends repository { * @return string file referece */ public function get_file_reference($source) { - // Box.net returns a url. - return $source; + global $USER; + list($type, $fileid, $filename) = $this->split_part($source); + $reference = new stdClass(); + $reference->fileid = $fileid; + $reference->filename = $filename; + $reference->userid = $USER->id; + $reference->downloadurl = ''; + if (optional_param('usefilereference', false, PARAM_BOOL)) { + try { + $shareinfo = $this->boxnetclient->share_file($reference->fileid, false); + } catch (moodle_exception $e) { + throw new repository_exception('cannotcreatereference', 'repository_boxnet'); + } + $reference->downloadurl = $shareinfo->download_url; + } + return serialize($reference); } + /** + * Get a link to the file. + * + * This returns the URL of the web view of the file. To generate this link the + * file must be shared. + * + * @param stdClass $reference Reference. + * @return string URL. + */ + public function get_link($reference) { + $reference = unserialize(self::convert_to_valid_reference($reference)); + $shareinfo = $this->boxnetclient->share_file($reference->fileid, false); + return $shareinfo->url; + } + + /** + * Synchronize the references. + * + * @param stored_file $file Stored file. + * @return boolean + */ public function sync_reference(stored_file $file) { if ($file->get_referencelastsync() + DAYSECS > time()) { // Synchronise not more often than once a day. return false; } - $c = new curl; + $c = new curl(); + $reference = unserialize(self::convert_to_valid_reference($file->get_reference())); + $url = $reference->downloadurl; if (file_extension_in_typegroup($file->get_filename(), 'web_image')) { $path = $this->prepare_file(''); - $result = $c->download_one($file->get_reference(), null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT)); + $result = $c->download_one($url, null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT)); $info = $c->get_info(); if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) { $fs = get_file_storage(); @@ -284,7 +427,7 @@ class repository_boxnet extends repository { return true; } } - $c->get($file->get_reference(), null, array('timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true, 'nobody' => true)); + $c->get($url, null, array('timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true, 'nobody' => true)); $info = $c->get_info(); if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && @@ -306,31 +449,25 @@ class repository_boxnet extends repository { * @return string */ public function get_reference_details($reference, $filestatus = 0) { - // Indicate it's from box.net repository + secure URL - $details = $this->get_name() . ': ' . $reference; + // Indicate it's from box.net repository. + $reference = unserialize(self::convert_to_valid_reference($reference)); if (!$filestatus) { - return $details; + return $this->get_name() . ': ' . $reference->filename; } else { return get_string('lostsource', 'repository', $details); } } /** - * Return the source information + * Return the source information. * - * @param stdClass $url + * @param string $source Not the reference, just the source. * @return string|null */ - public function get_file_source_info($url) { + public function get_file_source_info($source) { global $USER; - $array = explode('/', $url); - $fileid = array_pop($array); - $fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT); - if (!empty($fileinfo)) { - return 'Box ('. fullname($USER). '): '. (string)$fileinfo->file_name. ': '. $url; - } else { - return 'Box: '. $url; - } + list($type, $fileid, $filename) = $this->split_part($source); + return 'Box ('. fullname($USER) . '): ' . $filename; } /** @@ -343,9 +480,7 @@ class repository_boxnet extends repository { * @param array $options additional options affecting the file serving */ public function send_file($storedfile, $lifetime=null , $filter=0, $forcedownload=false, array $options = null) { - $ref = $storedfile->get_reference(); - // Let box.net serve the file. It will return 'no such file' content if file not found - // also if file has the different name than alias, it will be returned with the box.net filename - header('Location: ' . $ref); + $ref = unserialize(self::convert_to_valid_reference($storedfile->get_reference())); + header('Location: ' . $ref->downloadurl); } } From 4cdb94cce765f9b5fdb5e2a38d86bc870376240a Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Thu, 31 Oct 2013 17:04:36 +0800 Subject: [PATCH 02/14] MDL-42666 repository_boxnet: Function to convert APIv1 references to true copies --- repository/boxnet/locallib.php | 211 +++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 repository/boxnet/locallib.php diff --git a/repository/boxnet/locallib.php b/repository/boxnet/locallib.php new file mode 100644 index 00000000000..60aab361116 --- /dev/null +++ b/repository/boxnet/locallib.php @@ -0,0 +1,211 @@ +. + +/** + * Box.net locallib. + * + * @package repository_boxnet + * @copyright 2013 Frédéric Massart + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Migrate the references to local files. + * + * As the APIv1 is reaching its end of life on the 14th of Dec 2013, and we cannot + * convert the existing references to new references, we need to convert them + * to real files. + * + * @todo Deprecate/remove this function after the 14th of December 2013. + * @return void + */ +function repository_boxnet_migrate_references_from_apiv1() { + global $DB; + + // A string that the old references contain. + $apiv1signature = '/api/1.0/download/'; + + // Downloading the files could take a very long time! + @set_time_limit(0); + + // Create directory to download temporary files. + $dir = make_temp_directory('download/repository_boxnet/'); + + // Create a dummy file for the broken files. + $fs = get_file_storage(); + list($dummyhash, $dummysize, $unused) = $fs->add_string_to_pool('Lost reference from Box.net'); + + // Get the Box.net instances. There should be only one. + $sql = "SELECT i.id, i.typeid, r.id, r.type + FROM {repository} r, {repository_instances} i + WHERE i.typeid = r.id + AND r.type = :type"; + $ids = $DB->get_fieldset_sql($sql, array('type' => 'boxnet')); + if (empty($ids)) { + // We did not find any instance of Box.net. Let's just ignore this migration. + mtrace('Could not find any instance of the repository, aborting migration...'); + return; + } + + // The next bit is copied from the function file_storage::instance_sql_fields() + // because it is private and there is nothing in file_storage that suits our needs here. + $filefields = array('contenthash', 'pathnamehash', 'contextid', 'component', 'filearea', + 'itemid', 'filepath', 'filename', 'userid', 'filesize', 'mimetype', 'status', 'source', + 'author', 'license', 'timecreated', 'timemodified', 'sortorder', 'referencefileid'); + $referencefields = array('repositoryid' => 'repositoryid', + 'reference' => 'reference', + 'lastsync' => 'referencelastsync'); + $fields = array(); + $fields[] = 'f.id AS id'; + foreach ($filefields as $field) { + $fields[] = "f.{$field}"; + } + foreach ($referencefields as $field => $alias) { + $fields[] = "r.{$field} AS {$alias}"; + } + $fields = implode(', ', $fields); + + // We are not using repository::convert_references_to_local() or file_storage::get_external_files() + // because they would select too many records and load everything in memory as it is not using a recordset. + // Also, we filter the results not to get the draft area which should not be converted. + list($sqlfragment, $fragmentparams) = $DB->get_in_or_equal($ids, SQL_PARAMS_NAMED); + $sql = "SELECT " . $fields . " + FROM {files_reference} r + LEFT JOIN {files} f + ON f.referencefileid = r.id + WHERE r.repositoryid $sqlfragment + AND f.referencefileid IS NOT NULL + AND NOT (f.component = :component + AND f.filearea = :filearea)"; + + // For each reference we download the file. Then we add it to the file pool and update the references. + // The reason why we are re-inventing the wheel here is because the current API ends up calling + // repository::get_file() which includes a download timeout. As we are trying our best to copy + // the files here, we want to ignre any timeout. + $filerecords = $DB->get_recordset_sql($sql, array_merge($fragmentparams, array('component' => 'user', 'filearea' => 'draft'))); + $referenceids = array(); + foreach ($filerecords as $filerecord) { + $file = $fs->get_file_instance($filerecord); + $reference = unserialize(repository_boxnet::convert_to_valid_reference($file->get_reference())); + + if (empty($reference->downloadurl)) { + // Something is wrong... + mtrace('Skipping malformed reference (id: ' . $file->get_referencefileid() . ')'); + continue; + } else if (strpos($reference->downloadurl, $apiv1signature) === false) { + // This is not an old reference, we are not supposed to work on thos. + mtrace('Skipping non APIv1 reference (id: ' . $file->get_referencefileid() . ')'); + continue; + } else if (isset($referenceids[$file->get_referencefileid()])) { + // We have already worked on that reference, we skip any other file related to it. + // We cannot work on them here because they have been updated in the database but our + // recordset does not have those new values. They will be taken care of after this foreach. + continue; + } + + // Manually import the file to the file pool to prevent timeout limitations of the repository method get_file(). + // We ignore the fact that the content of the file could exist locally because we want to synchronize the file + // now to prevent the repository to try to download the file as well. + $saveas = $dir . uniqid('', true) . '_' . time() . '.tmp'; + $c = new curl(); + $result = $c->download_one($reference->downloadurl, null, array('filepath' => $saveas, 'followlocation' => true)); + $info = $c->get_info(); + if ($result !== true || !isset($info['http_code']) || $info['http_code'] != 200) { + // There was a problem while trying to download the reference... + if ($fs->content_exists($file->get_contenthash()) && $file->get_contenthash() != sha1('')) { + // Fortunately we already had a local version of this reference, so we keep it. We have to + // set it synchronized or there is a risk that repository::sync_reference() will try to download + // the file again. We cannot use $file->get_contenthash() and $file->get_filesize() because they + // cause repository::sync_reference() to be called. + $file->set_synchronized($filerecord->contenthash, $filerecord->filesize); + mtrace('Could not download reference, using last synced file. (id: ' . $file->get_referencefileid() . ')'); + } else { + // We don't know what the file was, but what can we do? In order to prevent a re-attempt to fetch the + // file in the next bit of this script (import_external_file()), we set a dummy content to the reference. + $file->set_synchronized($dummyhash, $dummysize); + mtrace('Could not download reference, dummy file used. (id: ' . $file->get_referencefileid() . ')'); + } + } else { + try { + // The file has been downloaded, we add it to the file pool and synchronize + // all the files using this reference. + list($contenthash, $filesize, $unused) = $fs->add_file_to_pool($saveas); + $file->set_synchronized($contenthash, $filesize); + } catch (moodle_exception $e) { + // Something wrong happened... + mtrace('Something went wrong during sync (id: ' . $file->get_referencefileid() . ')'); + } + } + + // OK, now that we have a content for that reference we will synchronize the file we are working on. + // If we delay this to the next bit of the script, then we would have to redownload everything if + // something fails and the script needs to be re-run. + try { + // Updating source to remove trace of APIv1 URL. + $file->set_source('Box APIv1 reference'); + } catch (moodle_exception $e) { + // Do not fail for this lame reason... + } + try { + $fs->import_external_file($file); + mtrace('File using reference converted to local file (id: ' . $file->get_id() . ')'); + } catch (moodle_exception $e) { + // Oh well... we tried what we could! + $file->delete_reference(); + mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $file->get_id() . ')'); + } + + // Log the reference IDs. + $referenceids[$file->get_referencefileid()] = $file->get_referencefileid(); + } + + // Nothing to do, we can return. And get_in_or_equal() will complain if we do. + if (empty($referenceids)) { + mtrace('The job is done, we didn\'t find any reference needing import.'); + return; + } + + // Now that we have downloaded the files, we can loop over every single file that was a reference + // to convert the remaining ones to local copies. + list($refsqlfragment, $refparams) = $DB->get_in_or_equal($referenceids, SQL_PARAMS_NAMED, 'ref'); + $sql = "SELECT " . $fields . " + FROM {files} f + LEFT JOIN {files_reference} r + ON f.referencefileid = r.id + WHERE f.referencefileid $refsqlfragment + AND NOT (f.component = :component + AND f.filearea = :filearea)"; + $filerecords = $DB->get_recordset_sql($sql, array_merge($refparams, array('component' => 'user', 'filearea' => 'draft'))); + foreach ($filerecords as $filerecord) { + $file = $fs->get_file_instance($filerecord); + try { + // Updating source to remove trace of APIv1 URL. + $file->set_source('Box APIv1 reference'); + } catch (moodle_exception $e) { + // Do not fail for this lame reason... + } + try { + $fs->import_external_file($file); + mtrace('File using reference converted to local file (id: ' . $file->get_id() . ')'); + } catch (moodle_exception $e) { + // Oh well... we tried what we could! + $file->delete_reference(); + mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $file->get_id() . ')'); + } + } + + mtrace('Migration finished.'); +} From cc4de9ea58bb2f8cade37eb4580c81b11617d106 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Fri, 1 Nov 2013 10:26:11 +0800 Subject: [PATCH 03/14] MDL-42666 repository_boxnet: Tool to launch the migration --- repository/boxnet/cli/migrationv1.php | 61 ++++++++++++++++ .../boxnet/lang/en/repository_boxnet.php | 6 ++ repository/boxnet/migrationv1.php | 69 +++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 repository/boxnet/cli/migrationv1.php create mode 100644 repository/boxnet/migrationv1.php diff --git a/repository/boxnet/cli/migrationv1.php b/repository/boxnet/cli/migrationv1.php new file mode 100644 index 00000000000..0187f416124 --- /dev/null +++ b/repository/boxnet/cli/migrationv1.php @@ -0,0 +1,61 @@ +. + +/** + * Box.net migration CLI script. + * + * @package repository_boxnet + * @copyright 2013 Frédéric Massart + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +define('CLI_SCRIPT', true); +require(__DIR__ . '/../../../config.php'); +require_once($CFG->libdir . '/clilib.php'); +require_once($CFG->dirroot . '/repository/boxnet/locallib.php'); + +// Now get cli options. +list($options, $unrecognized) = cli_get_params(array( + 'help' => false, + 'confirm' => '', +)); + +if ($unrecognized) { + $unrecognized = implode("\n ", $unrecognized); + cli_error(get_string('cliunknowoption', 'admin', $unrecognized)); +} + +$help = +"Box.net APIv1 migration tool. + +Options: +-h, --help Print out this help +--confirm Proceed with the migration + +Example: +\$ sudo -u www-data /usr/bin/php admin/tool/boxnetv1migrationtool/cli/migrate.php --confirm=1 +"; + +if ($options['help'] || empty($options['confirm'])) { + echo $help; + die(); +} + +if ($options['confirm']) { + mtrace("Box.net migration running..."); + repository_boxnet_migrate_references_from_apiv1(); +} + diff --git a/repository/boxnet/lang/en/repository_boxnet.php b/repository/boxnet/lang/en/repository_boxnet.php index 3e920c2b4fd..02093491a57 100644 --- a/repository/boxnet/lang/en/repository_boxnet.php +++ b/repository/boxnet/lang/en/repository_boxnet.php @@ -32,10 +32,16 @@ $string['configplugin'] = 'Box.net configuration'; $string['filesourceinfo'] = 'Box.net ({$a->fullname}): {$a->filename}'; $string['information'] = 'Get a client ID and secret from the Box.net developer page for your Moodle site.'; $string['invalidpassword'] = 'Invalid password'; +$string['migrationinfo'] = '

As part of the migration to the new API provided by Box.net, your file references have to be migrated. Unfortunately the reference system is not compatible with the API v2, so we are going to download them and convert them to real files.

+

Please also be aware that the migration can take a very long time, depending on how many references are used, and how large their files are.

+

You can run the migration tool by clicking the button below, or alternatively by executing the CLI script: repository/boxnet/cli/migrationv1.php.

+

Find out more here.

'; +$string['migrationtool'] = 'Box.net APIv1 migration tool'; $string['nullfilelist'] = 'There are no files in this repository'; $string['password'] = 'Password'; $string['pluginname_help'] = 'Repository on Box.net'; $string['pluginname'] = 'Box.net'; +$string['runthemigrationnow'] = 'Run the migration tool now'; $string['saved'] = 'Box.net data saved'; $string['shareurl'] = 'Share URL'; $string['username'] = 'Username for Box.net'; diff --git a/repository/boxnet/migrationv1.php b/repository/boxnet/migrationv1.php new file mode 100644 index 00000000000..35c782a0e08 --- /dev/null +++ b/repository/boxnet/migrationv1.php @@ -0,0 +1,69 @@ +. + +/** + * Box.net APIv1 migration tool. + * + * This tool is intended to migrate the references of the APIv1 of Box.net + * as this API is going end of life in December 14th 2013. As there is no + * way to support the references in the APIv2, we will convert those old + * references to local files. + * + * This operation can take a long time depending on the number of references + * used and their size. + * + * @package repository_boxnet + * @copyright 2013 Frédéric Massart + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @todo Deprecate/remove this tool after the 14th of December 2013. + */ + +require_once(__DIR__ . '/../../config.php'); +require_once($CFG->libdir . '/adminlib.php'); +require_once($CFG->dirroot . '/repository/boxnet/locallib.php'); + +require_login(); +require_capability('moodle/site:config', context_system::instance()); + +$title = get_string('migrationtool', 'repository_boxnet'); +$PAGE->set_context(context_system::instance()); +$PAGE->set_pagelayout('admin'); +$PAGE->set_title($title); +$PAGE->set_heading($title); +$PAGE->set_url(new moodle_url('/repository/boxnet/migrationv1.php')); +$PAGE->navbar->add($title); + +$confirm = optional_param('confirm', false, PARAM_BOOL); + +echo $OUTPUT->header(); +echo $OUTPUT->heading('Reference migration tool'); + +if (strtotime('15 December 2013') < time()) { + echo html_writer::tag('p', 'Sorry, the API v1 has already reached its end of life, the migration is no more possible.'); +} else if ($confirm && confirm_sesskey()) { + echo html_writer::start_tag('pre', array()); + repository_boxnet_migrate_references_from_apiv1(); + echo html_writer::end_tag('pre', array()); +} else { + $a = new stdClass(); + $a->docsurl = get_docs_url('Box.net_APIv1_migration'); + echo html_writer::tag('p', get_string('migrationinfo', 'repository_boxnet', $a)); + $execurl = new moodle_url('/repository/boxnet/migrationv1.php', array('confirm' => 1, 'sesskey' => sesskey())); + $button = new single_button($execurl, get_string('runthemigrationnow', 'repository_boxnet')); + echo $OUTPUT->render($button); +} + +echo $OUTPUT->footer(); From 05eaf6439729d088d4db21e22835eec658b89d8f Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Fri, 1 Nov 2013 12:43:30 +0800 Subject: [PATCH 04/14] MDL-42666 repository_boxnet: Advise the user to run the migration tool --- repository/boxnet/lang/en/repository_boxnet.php | 1 + repository/boxnet/lib.php | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/repository/boxnet/lang/en/repository_boxnet.php b/repository/boxnet/lang/en/repository_boxnet.php index 02093491a57..a2a447644ad 100644 --- a/repository/boxnet/lang/en/repository_boxnet.php +++ b/repository/boxnet/lang/en/repository_boxnet.php @@ -32,6 +32,7 @@ $string['configplugin'] = 'Box.net configuration'; $string['filesourceinfo'] = 'Box.net ({$a->fullname}): {$a->filename}'; $string['information'] = 'Get a client ID and secret from the Box.net developer page for your Moodle site.'; $string['invalidpassword'] = 'Invalid password'; +$string['migrationadvised'] = 'It appears that you were using Box.net with the API version 1, have you run the migration tool to convert the old references?'; $string['migrationinfo'] = '

As part of the migration to the new API provided by Box.net, your file references have to be migrated. Unfortunately the reference system is not compatible with the API v2, so we are going to download them and convert them to real files.

Please also be aware that the migration can take a very long time, depending on how many references are used, and how large their files are.

You can run the migration tool by clicking the button below, or alternatively by executing the CLI script: repository/boxnet/cli/migrationv1.php.

diff --git a/repository/boxnet/lib.php b/repository/boxnet/lib.php index a597dc21fc5..c767a1e81d0 100644 --- a/repository/boxnet/lib.php +++ b/repository/boxnet/lib.php @@ -332,6 +332,12 @@ class repository_boxnet extends repository { $mform->addElement('static', null, '', get_string('information', 'repository_boxnet')); $mform->addElement('static', null, '', get_string('warninghttps', 'repository_boxnet')); + + if (get_config('boxnet', 'api_key') && strtotime('15 December 2013') > time()) { + $url = new moodle_url('/repository/boxnet/migrationv1.php'); + $url = $url->out(); + $mform->addElement('static', null, '', get_string('migrationadvised', 'repository_boxnet', $url)); + } } /** From 091ac5969e13e4885e5b43e160c236903b69cc18 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Fri, 1 Nov 2013 16:23:34 +0800 Subject: [PATCH 05/14] MDL-42666 repository_boxnet: Disabling file references --- repository/boxnet/lib.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/repository/boxnet/lib.php b/repository/boxnet/lib.php index c767a1e81d0..8a233ed26cb 100644 --- a/repository/boxnet/lib.php +++ b/repository/boxnet/lib.php @@ -341,12 +341,15 @@ class repository_boxnet extends repository { } /** - * Box.net supports file linking and copying + * Box.net supports copied and links. + * + * Theoretically this API is ready for references, though it only works for + * Box.net Business accounts, but it is not enabled because we are not supporting it. * * @return int */ public function supported_returntypes() { - return FILE_INTERNAL | FILE_EXTERNAL | FILE_REFERENCE; + return FILE_INTERNAL | FILE_EXTERNAL; } /** @@ -384,7 +387,7 @@ class repository_boxnet extends repository { $reference->downloadurl = ''; if (optional_param('usefilereference', false, PARAM_BOOL)) { try { - $shareinfo = $this->boxnetclient->share_file($reference->fileid, false); + $shareinfo = $this->boxnetclient->share_file($reference->fileid); } catch (moodle_exception $e) { throw new repository_exception('cannotcreatereference', 'repository_boxnet'); } From b2f9028559487fcbb390635260fef3c12a9f29e9 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Mon, 4 Nov 2013 10:03:44 +0800 Subject: [PATCH 06/14] MDL-42666 repository_boxnet: Remove hardcoded date for APIv1 end of life --- repository/boxnet/lib.php | 2 +- repository/boxnet/migrationv1.php | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/repository/boxnet/lib.php b/repository/boxnet/lib.php index 8a233ed26cb..dd8185eb3f4 100644 --- a/repository/boxnet/lib.php +++ b/repository/boxnet/lib.php @@ -333,7 +333,7 @@ class repository_boxnet extends repository { $mform->addElement('static', null, '', get_string('information', 'repository_boxnet')); $mform->addElement('static', null, '', get_string('warninghttps', 'repository_boxnet')); - if (get_config('boxnet', 'api_key') && strtotime('15 December 2013') > time()) { + if (get_config('boxnet', 'api_key')) { $url = new moodle_url('/repository/boxnet/migrationv1.php'); $url = $url->out(); $mform->addElement('static', null, '', get_string('migrationadvised', 'repository_boxnet', $url)); diff --git a/repository/boxnet/migrationv1.php b/repository/boxnet/migrationv1.php index 35c782a0e08..c20245f832f 100644 --- a/repository/boxnet/migrationv1.php +++ b/repository/boxnet/migrationv1.php @@ -51,9 +51,7 @@ $confirm = optional_param('confirm', false, PARAM_BOOL); echo $OUTPUT->header(); echo $OUTPUT->heading('Reference migration tool'); -if (strtotime('15 December 2013') < time()) { - echo html_writer::tag('p', 'Sorry, the API v1 has already reached its end of life, the migration is no more possible.'); -} else if ($confirm && confirm_sesskey()) { +if ($confirm && confirm_sesskey()) { echo html_writer::start_tag('pre', array()); repository_boxnet_migrate_references_from_apiv1(); echo html_writer::end_tag('pre', array()); From 79d89136f08962ea4866aab87d9284250c944a94 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Mon, 4 Nov 2013 15:55:33 +0800 Subject: [PATCH 07/14] MDL-42666 oauthlib: Getter methods for accesstoken, client id and secret --- lib/oauthlib.php | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/lib/oauthlib.php b/lib/oauthlib.php index afc92643379..423480c2329 100644 --- a/lib/oauthlib.php +++ b/lib/oauthlib.php @@ -617,6 +617,39 @@ abstract class oauth2_client extends curl { return null; } + /** + * Get access token. + * + * This is just a getter to read the private property. + * + * @return string + */ + public function get_accesstoken() { + return $this->accesstoken; + } + + /** + * Get the client ID. + * + * This is just a getter to read the private property. + * + * @return string + */ + public function get_clientid() { + return $this->clientid; + } + + /** + * Get the client secret. + * + * This is just a getter to read the private property. + * + * @return string + */ + public function get_clientsecret() { + return $this->clientsecret; + } + /** * Should HTTP GET be used instead of POST? * Some APIs do not support POST and want oauth to use From e27ed28785edfe87b211b7eb99f632c02dd3f07f Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Mon, 4 Nov 2013 16:08:44 +0800 Subject: [PATCH 08/14] MDL-42666 repository_boxnet: Revoke access token when logging out --- lib/boxlib.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/boxlib.php b/lib/boxlib.php index ed8f175addb..d11475b7968 100644 --- a/lib/boxlib.php +++ b/lib/boxlib.php @@ -91,13 +91,13 @@ class boxnet_client extends oauth2_client { * @return void */ public function log_out() { - if ($this->accesstoken) { + if ($accesstoken = $this->get_accesstoken()) { $params = array( - 'client_id' => $this->clientid, - 'client_secret' => $this->clientsecret, - 'token' => $this->accesstoken->token + 'client_id' => $this->get_clientid(), + 'client_secret' => $this->get_clientsecret(), + 'token' => $accesstoken->token ); - $this->post($this->get_revoke_url(), $params); + $this->post($this->revoke_url(), $params); } parent::log_out(); } From 090976b9664e18ffe353ca282e49adbd40ba6e69 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Tue, 5 Nov 2013 17:32:44 +0800 Subject: [PATCH 09/14] MDL-42666 repository_boxnet: Alert the admins of the migration process --- repository/boxnet/db/upgrade.php | 69 +++++++++++++++++++ repository/boxnet/db/upgradelib.php | 55 +++++++++++++++ .../boxnet/lang/en/repository_boxnet.php | 3 + repository/boxnet/version.php | 2 +- 4 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 repository/boxnet/db/upgrade.php create mode 100644 repository/boxnet/db/upgradelib.php diff --git a/repository/boxnet/db/upgrade.php b/repository/boxnet/db/upgrade.php new file mode 100644 index 00000000000..39fdb17608f --- /dev/null +++ b/repository/boxnet/db/upgrade.php @@ -0,0 +1,69 @@ +. + +/** + * Upgrade. + * + * @package repository_boxnet + * @copyright 2013 Frédéric Massart + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Upgrade function. + * + * @param int $oldversion the version we are upgrading from. + * @return bool result + */ +function xmldb_repository_boxnet_upgrade($oldversion) { + global $CFG, $DB; + + $dbman = $DB->get_manager(); + + if ($oldversion < 2013110503) { + // Delete old user preferences containing auth tokens. + $DB->delete_records('user_preferences', array('name' => 'boxnet__auth_token')); + upgrade_plugin_savepoint(true, 2013110503, 'repository', 'boxnet'); + } + + if ($oldversion < 2013110504) { + require_once($CFG->dirroot . '/repository/lib.php'); + require_once($CFG->dirroot . '/repository/boxnet/db/upgradelib.php'); + + $params = array(); + $params['context'] = array(); + $params['onlyvisible'] = false; + $params['type'] = 'boxnet'; + $instances = repository::get_instances($params); + + // Notify the admin about the migration process if they are using the repo. + if (!empty($instances)) { + repository_boxnet_admin_upgrade_notification(); + } + + // Hide the repository. + $repositorytype = repository::get_type_by_typename('boxnet'); + if (!empty($repositorytype)) { + $repositorytype->update_visibility(false); + } + + upgrade_plugin_savepoint(true, 2013110504, 'repository', 'boxnet'); + } + + return true; +} diff --git a/repository/boxnet/db/upgradelib.php b/repository/boxnet/db/upgradelib.php new file mode 100644 index 00000000000..c0cf9de80d6 --- /dev/null +++ b/repository/boxnet/db/upgradelib.php @@ -0,0 +1,55 @@ +. + +/** + * Locallib. + * + * @package repository_boxnet + * @copyright 2013 Frédéric Massart + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Send a message to the admin in regard with the APIv1 migration. + * + * @return void + */ +function repository_boxnet_admin_upgrade_notification() { + $admins = get_admins(); + + if (empty($admins)) { + return; + } + $a = new stdClass(); + $a->docsurl = get_docs_url('Box.net_APIv1_migration'); + + foreach ($admins as $admin) { + $message = new stdClass(); + $message->component = 'moodle'; + $message->name = 'notices'; + $message->userfrom = get_admin(); + $message->userto = $admin; + $message->smallmessage = get_string('apiv1migration_message_small', 'repository_boxnet'); + $message->subject = get_string('apiv1migration_message_subject', 'repository_boxnet'); + $message->fullmessage = get_string('apiv1migration_message_content', 'repository_boxnet', $a); + $message->fullmessagehtml = get_string('apiv1migration_message_content', 'repository_boxnet', $a); + $message->fullmessageformat = FORMAT_PLAIN; + $message->notification = 1; + message_send($message); + } +} diff --git a/repository/boxnet/lang/en/repository_boxnet.php b/repository/boxnet/lang/en/repository_boxnet.php index a2a447644ad..84c65929ec1 100644 --- a/repository/boxnet/lang/en/repository_boxnet.php +++ b/repository/boxnet/lang/en/repository_boxnet.php @@ -24,6 +24,9 @@ */ $string['apikey'] = 'API key'; +$string['apiv1migration_message_content'] = 'As part of the recent Moodle upgrade (2.6, 2.5.3, 2.4.7), the Box.net repository plugin has been disabled. To re-enable it, you need to re-configure it as described in the documentation {$a->docsurl}.'; +$string['apiv1migration_message_small'] = 'This plugin has been disabled, as it requires configuration as described in the documentation Box.net APIv1 migration.'; +$string['apiv1migration_message_subject'] = 'Important information regarding Box.net repository plugin'; $string['boxnet:view'] = 'View box.net repository'; $string['cannotcreatereference'] = 'Cannot create a reference, not enough permissions to share the file on Box.net.'; $string['clientid'] = 'Client ID'; diff --git a/repository/boxnet/version.php b/repository/boxnet/version.php index ec971839672..a32ea60af2b 100644 --- a/repository/boxnet/version.php +++ b/repository/boxnet/version.php @@ -26,6 +26,6 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2013110500; // The current plugin version (Date: YYYYMMDDXX) +$plugin->version = 2013110504; // The current plugin version (Date: YYYYMMDDXX) $plugin->requires = 2013110500; // Requires this Moodle version $plugin->component = 'repository_boxnet'; // Full name of the plugin (used for diagnostics) From ab227cc6a9f25d1d8bb0d49435dac7141d048c77 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Tue, 5 Nov 2013 17:33:13 +0800 Subject: [PATCH 10/14] MDL-42666 repository_boxnet: Warn on HTTPS requirement when non HTTPS --- repository/boxnet/lib.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/repository/boxnet/lib.php b/repository/boxnet/lib.php index dd8185eb3f4..5e7c330f868 100644 --- a/repository/boxnet/lib.php +++ b/repository/boxnet/lib.php @@ -331,7 +331,10 @@ class repository_boxnet extends repository { $mform->setType('clientsecret', PARAM_RAW_TRIMMED); $mform->addElement('static', null, '', get_string('information', 'repository_boxnet')); - $mform->addElement('static', null, '', get_string('warninghttps', 'repository_boxnet')); + + if (strpos($CFG->wwwroot, 'https') !== 0) { + $mform->addElement('static', null, '', get_string('warninghttps', 'repository_boxnet')); + } if (get_config('boxnet', 'api_key')) { $url = new moodle_url('/repository/boxnet/migrationv1.php'); From d7274bfead44a7f83c85514f4935a0458875061f Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Wed, 6 Nov 2013 09:55:44 +0800 Subject: [PATCH 11/14] MDL-42666 repository_boxnet: Convert all the references asap This has been done in case the script fails during the processing and needs to be re-run. Now all the references are converted as soon as we can, where as before they could have ended up being re-downloaded. --- repository/boxnet/locallib.php | 80 ++++++++++++---------------------- 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/repository/boxnet/locallib.php b/repository/boxnet/locallib.php index 60aab361116..838ee7f1f05 100644 --- a/repository/boxnet/locallib.php +++ b/repository/boxnet/locallib.php @@ -150,60 +150,38 @@ function repository_boxnet_migrate_references_from_apiv1() { } } - // OK, now that we have a content for that reference we will synchronize the file we are working on. - // If we delay this to the next bit of the script, then we would have to redownload everything if - // something fails and the script needs to be re-run. - try { - // Updating source to remove trace of APIv1 URL. - $file->set_source('Box APIv1 reference'); - } catch (moodle_exception $e) { - // Do not fail for this lame reason... - } - try { - $fs->import_external_file($file); - mtrace('File using reference converted to local file (id: ' . $file->get_id() . ')'); - } catch (moodle_exception $e) { - // Oh well... we tried what we could! - $file->delete_reference(); - mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $file->get_id() . ')'); - } - // Log the reference IDs. $referenceids[$file->get_referencefileid()] = $file->get_referencefileid(); - } - // Nothing to do, we can return. And get_in_or_equal() will complain if we do. - if (empty($referenceids)) { - mtrace('The job is done, we didn\'t find any reference needing import.'); - return; - } - - // Now that we have downloaded the files, we can loop over every single file that was a reference - // to convert the remaining ones to local copies. - list($refsqlfragment, $refparams) = $DB->get_in_or_equal($referenceids, SQL_PARAMS_NAMED, 'ref'); - $sql = "SELECT " . $fields . " - FROM {files} f - LEFT JOIN {files_reference} r - ON f.referencefileid = r.id - WHERE f.referencefileid $refsqlfragment - AND NOT (f.component = :component - AND f.filearea = :filearea)"; - $filerecords = $DB->get_recordset_sql($sql, array_merge($refparams, array('component' => 'user', 'filearea' => 'draft'))); - foreach ($filerecords as $filerecord) { - $file = $fs->get_file_instance($filerecord); - try { - // Updating source to remove trace of APIv1 URL. - $file->set_source('Box APIv1 reference'); - } catch (moodle_exception $e) { - // Do not fail for this lame reason... - } - try { - $fs->import_external_file($file); - mtrace('File using reference converted to local file (id: ' . $file->get_id() . ')'); - } catch (moodle_exception $e) { - // Oh well... we tried what we could! - $file->delete_reference(); - mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $file->get_id() . ')'); + // Now that the file is downloaded, we can loop over all the files using this reference + // to convert them to local copies. We have chosen to do that in this loop so that if the + // execution fails in the middle, we would not have to redownload the files again and again. + // By the way, we cannot use the records fetched in $filerecords because they will not be updated. + $sql = "SELECT " . $fields . " + FROM {files} f + LEFT JOIN {files_reference} r + ON f.referencefileid = r.id + WHERE f.referencefileid = :refid + AND NOT (f.component = :component + AND f.filearea = :filearea)"; + $reffilerecords = $DB->get_recordset_sql($sql, array('component' => 'user', 'filearea' => 'draft', + 'refid' => $file->get_referencefileid())); + foreach ($reffilerecords as $reffilerecord) { + $reffile = $fs->get_file_instance($reffilerecord); + try { + // Updating source to remove trace of APIv1 URL. + $reffile->set_source('Box APIv1 reference'); + } catch (moodle_exception $e) { + // Do not fail for this lame reason... + } + try { + $fs->import_external_file($reffile); + mtrace('File using reference converted to local file (id: ' . $reffile->get_id() . ')'); + } catch (moodle_exception $e) { + // Oh well... we tried what we could! + $reffile->delete_reference(); + mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $reffile->get_id() . ')'); + } } } From 102a241998adca6c086bb2dda1854ec056e1c9e9 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Wed, 6 Nov 2013 12:40:42 +0800 Subject: [PATCH 12/14] MDL-42666 repository_boxnet: Fix undefined variable usage --- repository/boxnet/lib.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/boxnet/lib.php b/repository/boxnet/lib.php index 5e7c330f868..cde732723f3 100644 --- a/repository/boxnet/lib.php +++ b/repository/boxnet/lib.php @@ -466,7 +466,7 @@ class repository_boxnet extends repository { if (!$filestatus) { return $this->get_name() . ': ' . $reference->filename; } else { - return get_string('lostsource', 'repository', $details); + return get_string('lostsource', 'repository', $reference->filename); } } From 51ada519a6881c97a24584725616f1f342fa8edb Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Wed, 6 Nov 2013 12:42:22 +0800 Subject: [PATCH 13/14] MDL-42666 repository_boxnet: Downloading a file follows locations --- lib/boxlib.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/boxlib.php b/lib/boxlib.php index d11475b7968..60db1760173 100644 --- a/lib/boxlib.php +++ b/lib/boxlib.php @@ -58,7 +58,8 @@ class boxnet_client extends oauth2_client { * @return bool Success or not. */ public function download_file($fileid, $path) { - $result = $this->download_one($this->make_url("/files/$fileid/content"), array(), array('filepath' => $path)); + $result = $this->download_one($this->make_url("/files/$fileid/content"), array(), + array('filepath' => $path, 'CURLOPT_FOLLOWLOCATION' => true)); return ($result === true && $this->info['http_code'] === 200); } From f591c247cadb090b294bfbba4489ea3f740a4e6c Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Thu, 7 Nov 2013 10:18:20 +0800 Subject: [PATCH 14/14] MDL-42666 repository_boxnet: Improved output during migration --- repository/boxnet/locallib.php | 2 ++ repository/boxnet/migrationv1.php | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/repository/boxnet/locallib.php b/repository/boxnet/locallib.php index 838ee7f1f05..29ba3071161 100644 --- a/repository/boxnet/locallib.php +++ b/repository/boxnet/locallib.php @@ -116,6 +116,8 @@ function repository_boxnet_migrate_references_from_apiv1() { continue; } + mtrace('Starting migration of file reference ' . $file->get_referencefileid()); + // Manually import the file to the file pool to prevent timeout limitations of the repository method get_file(). // We ignore the fact that the content of the file could exist locally because we want to synchronize the file // now to prevent the repository to try to download the file as well. diff --git a/repository/boxnet/migrationv1.php b/repository/boxnet/migrationv1.php index c20245f832f..30d80d1d031 100644 --- a/repository/boxnet/migrationv1.php +++ b/repository/boxnet/migrationv1.php @@ -31,6 +31,8 @@ * @todo Deprecate/remove this tool after the 14th of December 2013. */ +define('NO_OUTPUT_BUFFERING', true); + require_once(__DIR__ . '/../../config.php'); require_once($CFG->libdir . '/adminlib.php'); require_once($CFG->dirroot . '/repository/boxnet/locallib.php'); @@ -40,7 +42,7 @@ require_capability('moodle/site:config', context_system::instance()); $title = get_string('migrationtool', 'repository_boxnet'); $PAGE->set_context(context_system::instance()); -$PAGE->set_pagelayout('admin'); +$PAGE->set_pagelayout('maintenance'); $PAGE->set_title($title); $PAGE->set_heading($title); $PAGE->set_url(new moodle_url('/repository/boxnet/migrationv1.php'));