From 3f944b3350911bc0e58cd543012b7d6b30ed0704 Mon Sep 17 00:00:00 2001 From: Frederic Massart Date: Thu, 31 Oct 2013 17:04:36 +0800 Subject: [PATCH] MDL-42666 repository_boxnet: Function to convert APIv1 references to true copies --- repository/boxnet/locallib.php | 211 +++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 repository/boxnet/locallib.php diff --git a/repository/boxnet/locallib.php b/repository/boxnet/locallib.php new file mode 100644 index 00000000000..60aab361116 --- /dev/null +++ b/repository/boxnet/locallib.php @@ -0,0 +1,211 @@ +. + +/** + * Box.net locallib. + * + * @package repository_boxnet + * @copyright 2013 Frédéric Massart + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Migrate the references to local files. + * + * As the APIv1 is reaching its end of life on the 14th of Dec 2013, and we cannot + * convert the existing references to new references, we need to convert them + * to real files. + * + * @todo Deprecate/remove this function after the 14th of December 2013. + * @return void + */ +function repository_boxnet_migrate_references_from_apiv1() { + global $DB; + + // A string that the old references contain. + $apiv1signature = '/api/1.0/download/'; + + // Downloading the files could take a very long time! + @set_time_limit(0); + + // Create directory to download temporary files. + $dir = make_temp_directory('download/repository_boxnet/'); + + // Create a dummy file for the broken files. + $fs = get_file_storage(); + list($dummyhash, $dummysize, $unused) = $fs->add_string_to_pool('Lost reference from Box.net'); + + // Get the Box.net instances. There should be only one. + $sql = "SELECT i.id, i.typeid, r.id, r.type + FROM {repository} r, {repository_instances} i + WHERE i.typeid = r.id + AND r.type = :type"; + $ids = $DB->get_fieldset_sql($sql, array('type' => 'boxnet')); + if (empty($ids)) { + // We did not find any instance of Box.net. Let's just ignore this migration. + mtrace('Could not find any instance of the repository, aborting migration...'); + return; + } + + // The next bit is copied from the function file_storage::instance_sql_fields() + // because it is private and there is nothing in file_storage that suits our needs here. + $filefields = array('contenthash', 'pathnamehash', 'contextid', 'component', 'filearea', + 'itemid', 'filepath', 'filename', 'userid', 'filesize', 'mimetype', 'status', 'source', + 'author', 'license', 'timecreated', 'timemodified', 'sortorder', 'referencefileid'); + $referencefields = array('repositoryid' => 'repositoryid', + 'reference' => 'reference', + 'lastsync' => 'referencelastsync'); + $fields = array(); + $fields[] = 'f.id AS id'; + foreach ($filefields as $field) { + $fields[] = "f.{$field}"; + } + foreach ($referencefields as $field => $alias) { + $fields[] = "r.{$field} AS {$alias}"; + } + $fields = implode(', ', $fields); + + // We are not using repository::convert_references_to_local() or file_storage::get_external_files() + // because they would select too many records and load everything in memory as it is not using a recordset. + // Also, we filter the results not to get the draft area which should not be converted. + list($sqlfragment, $fragmentparams) = $DB->get_in_or_equal($ids, SQL_PARAMS_NAMED); + $sql = "SELECT " . $fields . " + FROM {files_reference} r + LEFT JOIN {files} f + ON f.referencefileid = r.id + WHERE r.repositoryid $sqlfragment + AND f.referencefileid IS NOT NULL + AND NOT (f.component = :component + AND f.filearea = :filearea)"; + + // For each reference we download the file. Then we add it to the file pool and update the references. + // The reason why we are re-inventing the wheel here is because the current API ends up calling + // repository::get_file() which includes a download timeout. As we are trying our best to copy + // the files here, we want to ignre any timeout. + $filerecords = $DB->get_recordset_sql($sql, array_merge($fragmentparams, array('component' => 'user', 'filearea' => 'draft'))); + $referenceids = array(); + foreach ($filerecords as $filerecord) { + $file = $fs->get_file_instance($filerecord); + $reference = unserialize(repository_boxnet::convert_to_valid_reference($file->get_reference())); + + if (empty($reference->downloadurl)) { + // Something is wrong... + mtrace('Skipping malformed reference (id: ' . $file->get_referencefileid() . ')'); + continue; + } else if (strpos($reference->downloadurl, $apiv1signature) === false) { + // This is not an old reference, we are not supposed to work on thos. + mtrace('Skipping non APIv1 reference (id: ' . $file->get_referencefileid() . ')'); + continue; + } else if (isset($referenceids[$file->get_referencefileid()])) { + // We have already worked on that reference, we skip any other file related to it. + // We cannot work on them here because they have been updated in the database but our + // recordset does not have those new values. They will be taken care of after this foreach. + continue; + } + + // Manually import the file to the file pool to prevent timeout limitations of the repository method get_file(). + // We ignore the fact that the content of the file could exist locally because we want to synchronize the file + // now to prevent the repository to try to download the file as well. + $saveas = $dir . uniqid('', true) . '_' . time() . '.tmp'; + $c = new curl(); + $result = $c->download_one($reference->downloadurl, null, array('filepath' => $saveas, 'followlocation' => true)); + $info = $c->get_info(); + if ($result !== true || !isset($info['http_code']) || $info['http_code'] != 200) { + // There was a problem while trying to download the reference... + if ($fs->content_exists($file->get_contenthash()) && $file->get_contenthash() != sha1('')) { + // Fortunately we already had a local version of this reference, so we keep it. We have to + // set it synchronized or there is a risk that repository::sync_reference() will try to download + // the file again. We cannot use $file->get_contenthash() and $file->get_filesize() because they + // cause repository::sync_reference() to be called. + $file->set_synchronized($filerecord->contenthash, $filerecord->filesize); + mtrace('Could not download reference, using last synced file. (id: ' . $file->get_referencefileid() . ')'); + } else { + // We don't know what the file was, but what can we do? In order to prevent a re-attempt to fetch the + // file in the next bit of this script (import_external_file()), we set a dummy content to the reference. + $file->set_synchronized($dummyhash, $dummysize); + mtrace('Could not download reference, dummy file used. (id: ' . $file->get_referencefileid() . ')'); + } + } else { + try { + // The file has been downloaded, we add it to the file pool and synchronize + // all the files using this reference. + list($contenthash, $filesize, $unused) = $fs->add_file_to_pool($saveas); + $file->set_synchronized($contenthash, $filesize); + } catch (moodle_exception $e) { + // Something wrong happened... + mtrace('Something went wrong during sync (id: ' . $file->get_referencefileid() . ')'); + } + } + + // OK, now that we have a content for that reference we will synchronize the file we are working on. + // If we delay this to the next bit of the script, then we would have to redownload everything if + // something fails and the script needs to be re-run. + try { + // Updating source to remove trace of APIv1 URL. + $file->set_source('Box APIv1 reference'); + } catch (moodle_exception $e) { + // Do not fail for this lame reason... + } + try { + $fs->import_external_file($file); + mtrace('File using reference converted to local file (id: ' . $file->get_id() . ')'); + } catch (moodle_exception $e) { + // Oh well... we tried what we could! + $file->delete_reference(); + mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $file->get_id() . ')'); + } + + // Log the reference IDs. + $referenceids[$file->get_referencefileid()] = $file->get_referencefileid(); + } + + // Nothing to do, we can return. And get_in_or_equal() will complain if we do. + if (empty($referenceids)) { + mtrace('The job is done, we didn\'t find any reference needing import.'); + return; + } + + // Now that we have downloaded the files, we can loop over every single file that was a reference + // to convert the remaining ones to local copies. + list($refsqlfragment, $refparams) = $DB->get_in_or_equal($referenceids, SQL_PARAMS_NAMED, 'ref'); + $sql = "SELECT " . $fields . " + FROM {files} f + LEFT JOIN {files_reference} r + ON f.referencefileid = r.id + WHERE f.referencefileid $refsqlfragment + AND NOT (f.component = :component + AND f.filearea = :filearea)"; + $filerecords = $DB->get_recordset_sql($sql, array_merge($refparams, array('component' => 'user', 'filearea' => 'draft'))); + foreach ($filerecords as $filerecord) { + $file = $fs->get_file_instance($filerecord); + try { + // Updating source to remove trace of APIv1 URL. + $file->set_source('Box APIv1 reference'); + } catch (moodle_exception $e) { + // Do not fail for this lame reason... + } + try { + $fs->import_external_file($file); + mtrace('File using reference converted to local file (id: ' . $file->get_id() . ')'); + } catch (moodle_exception $e) { + // Oh well... we tried what we could! + $file->delete_reference(); + mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $file->get_id() . ')'); + } + } + + mtrace('Migration finished.'); +}