From 6feae1d256448b0a9867a010065deccfc3f54527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Wed, 20 Jun 2012 11:48:42 +0200 Subject: [PATCH 01/10] MDL-33430 Make file_storage::unpack_reference() more picky on what it gets --- lib/filestorage/file_storage.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index fd9f63c3c17..933c5c39dbb 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -1659,7 +1659,14 @@ class file_storage { * @return array */ public static function unpack_reference($str, $cleanparams = false) { - $params = unserialize(base64_decode($str)); + $decoded = base64_decode($str, true); + if ($decoded === false) { + throw new file_reference_exception(null, $str, null, null, 'Invalid base64 format'); + } + $params = @unserialize($decoded); // hide E_NOTICE + if ($params === false) { + throw new file_reference_exception(null, $decoded, null, null, 'Not an unserializeable value'); + } if (is_array($params) && $cleanparams) { $params = array( 'component' => is_null($params['component']) ? '' : clean_param($params['component'], PARAM_COMPONENT), From 79b810fdc0fd9fb0b16e941008b5ec38d93a015d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Tue, 19 Jun 2012 13:56:30 +0200 Subject: [PATCH 02/10] MDL-33430 Include repository type in files.xml in MBZ backup file This is a valuable information needed especially when restoring the MBZ at another site. The repositoryid itself is not enough there to identify the type of the repository. --- backup/moodle2/backup_stepslib.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backup/moodle2/backup_stepslib.php b/backup/moodle2/backup_stepslib.php index a2fa9488605..fa515210c0e 100644 --- a/backup/moodle2/backup_stepslib.php +++ b/backup/moodle2/backup_stepslib.php @@ -1404,7 +1404,8 @@ class backup_final_files_structure_step extends backup_structure_step { 'contenthash', 'contextid', 'component', 'filearea', 'itemid', 'filepath', 'filename', 'userid', 'filesize', 'mimetype', 'status', 'timecreated', 'timemodified', - 'source', 'author', 'license', 'sortorder', 'reference', 'repositoryid')); + 'source', 'author', 'license', 'sortorder', + 'repositorytype', 'repositoryid', 'reference')); // Build the tree @@ -1412,12 +1413,12 @@ class backup_final_files_structure_step extends backup_structure_step { // Define sources - $file->set_source_sql("SELECT f.*, r.repositoryid, r.reference + $file->set_source_sql("SELECT f.*, r.type AS repositorytype, fr.repositoryid, fr.reference FROM {files} f - LEFT JOIN {files_reference} r - ON r.id = f.referencefileid - JOIN {backup_ids_temp} bi - ON f.id = bi.itemid + LEFT JOIN {files_reference} fr ON fr.id = f.referencefileid + LEFT JOIN {repository_instances} ri ON ri.id = fr.repositoryid + LEFT JOIN {repository} r ON r.id = ri.typeid + JOIN {backup_ids_temp} bi ON f.id = bi.itemid WHERE bi.backupid = ? AND bi.itemname = 'filefinal'", array(backup::VAR_BACKUPID)); From 4b6b087fafe55d8d28b8a1b024db2e977dfa032e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Tue, 19 Jun 2012 16:56:33 +0200 Subject: [PATCH 03/10] MDL-33430 Do not restore aliases yet, stash them for later processing --- backup/moodle2/restore_stepslib.php | 13 +--- backup/util/dbops/restore_dbops.class.php | 93 ++++++++++++++++------- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/backup/moodle2/restore_stepslib.php b/backup/moodle2/restore_stepslib.php index 60a4086d6c8..9bc8d436cf3 100644 --- a/backup/moodle2/restore_stepslib.php +++ b/backup/moodle2/restore_stepslib.php @@ -591,22 +591,14 @@ class restore_load_included_files extends restore_structure_step { } /** - * Processing functions go here + * Process one element from files.xml * - * @param array $data one file record including repositoryid and reference + * @param array $data the element data */ public function process_file($data) { $data = (object)$data; // handy - $isreference = !empty($data->repositoryid); - $issamesite = $this->task->is_samesite(); - - // If it's not samesite, we skip file refernces - if (!$issamesite && $isreference) { - return; - } - // load it if needed: // - it it is one of the annotated inforef files (course/section/activity/block) // - it is one "user", "group", "grouping", "grade", "question" or "qtype_xxxx" component file (that aren't sent to inforef ever) @@ -617,7 +609,6 @@ class restore_load_included_files extends restore_structure_step { $data->component == 'grouping' || $data->component == 'grade' || $data->component == 'question' || substr($data->component, 0, 5) == 'qtype'); if ($isfileref || $iscomponent) { - // Process files restore_dbops::set_backup_files_record($this->get_restoreid(), $data); } } diff --git a/backup/util/dbops/restore_dbops.class.php b/backup/util/dbops/restore_dbops.class.php index 4254e698e90..de3ff5fd06b 100644 --- a/backup/util/dbops/restore_dbops.class.php +++ b/backup/util/dbops/restore_dbops.class.php @@ -808,6 +808,17 @@ abstract class restore_dbops { * Given one component/filearea/context and * optionally one source itemname to match itemids * put the corresponding files in the pool + * + * @param string $basepath the full path to the root of unzipped backup file + * @param string $restoreid the restore job's identification + * @param string $component + * @param string $filearea + * @param int $oldcontextid + * @param int $dfltuserid default $file->user if the old one can't be mapped + * @param string|null $itemname + * @param int|null $olditemid + * @param int|null $forcenewcontextid explicit value for the new contextid (skip mapping) + * @param bool $skipparentitemidctxmatch */ public static function send_files_to_pool($basepath, $restoreid, $component, $filearea, $oldcontextid, $dfltuserid, $itemname = null, $olditemid = null, $forcenewcontextid = null, $skipparentitemidctxmatch = false) { global $DB; @@ -872,47 +883,73 @@ abstract class restore_dbops { foreach ($rs as $rec) { $file = (object)unserialize(base64_decode($rec->info)); - $isreference = !empty($file->repositoryid); - // ignore root dirs (they are created automatically) if ($file->filepath == '/' && $file->filename == '.') { continue; } + // set the best possible user $mappeduser = self::get_backup_ids_record($restoreid, 'user', $file->userid); - $file->userid = !empty($mappeduser) ? $mappeduser->newitemid : $dfltuserid; - // dir found (and not root one), let's create if + $mappeduserid = !empty($mappeduser) ? $mappeduser->newitemid : $dfltuserid; + + // dir found (and not root one), let's create it if ($file->filename == '.') { - $fs->create_directory($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->userid); + $fs->create_directory($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $mappeduserid); continue; } - // arrived here, file found - // Find file in backup pool - $backuppath = $basepath . backup_file_manager::get_backup_content_file_location($file->contenthash); + if (empty($file->repositoryid)) { + // this is a regular file, it must be present in the backup pool + $backuppath = $basepath . backup_file_manager::get_backup_content_file_location($file->contenthash); - if (!file_exists($backuppath) && !$isreference) { - throw new restore_dbops_exception('file_not_found_in_pool', $file); - } - if (!$fs->file_exists($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->filename)) { - $file_record = array( - 'contextid' => $newcontextid, - 'component' => $component, - 'filearea' => $filearea, - 'itemid' => $rec->newitemid, - 'filepath' => $file->filepath, - 'filename' => $file->filename, - 'timecreated' => $file->timecreated, - 'timemodified'=> $file->timemodified, - 'userid' => $file->userid, - 'author' => $file->author, - 'license' => $file->license, - 'sortorder' => $file->sortorder); - if ($isreference) { - $fs->create_file_from_reference($file_record, $file->repositoryid, $file->reference); - } else { + if (!file_exists($backuppath)) { + throw new restore_dbops_exception('file_not_found_in_pool', $file); + } + + // create the file in the filepool if it does not exist yet + if (!$fs->file_exists($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->filename)) { + $file_record = array( + 'contextid' => $newcontextid, + 'component' => $component, + 'filearea' => $filearea, + 'itemid' => $rec->newitemid, + 'filepath' => $file->filepath, + 'filename' => $file->filename, + 'timecreated' => $file->timecreated, + 'timemodified'=> $file->timemodified, + 'userid' => $mappeduserid, + 'author' => $file->author, + 'license' => $file->license, + 'sortorder' => $file->sortorder + ); $fs->create_file_from_pathname($file_record, $backuppath); } + + } else { + // this is an alias - we can't create it yet so we stash it in a temp + // table and will let the final task to deal with it + if (!$fs->file_exists($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->filename)) { + $info = new stdClass(); + // oldfile holds the raw information stored in MBZ (including reference-related info) + $info->oldfile = $file; + // newfile holds the info for the new file_record with the context, user and itemid mapped + $info->newfile = (object)array( + 'contextid' => $newcontextid, + 'component' => $component, + 'filearea' => $filearea, + 'itemid' => $rec->newitemid, + 'filepath' => $file->filepath, + 'filename' => $file->filename, + 'timecreated' => $file->timecreated, + 'timemodified'=> $file->timemodified, + 'userid' => $mappeduserid, + 'author' => $file->author, + 'license' => $file->license, + 'sortorder' => $file->sortorder + ); + + restore_dbops::set_backup_ids_record($restoreid, 'file_aliases_queue', $file->id, 0, null, $info); + } } } $rs->close(); From 38ca2f1a839d8ec4ca315b71c0f76786b4988984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Wed, 20 Jun 2012 11:15:47 +0200 Subject: [PATCH 04/10] MDL-33430 Add newcontextid and newitemid to backup_files_template These new fields will be used to hold the new location of the file once it is sent to the filepool during the restore. --- lib/db/install.xml | 8 +++++--- lib/db/upgrade.php | 24 ++++++++++++++++++++++++ version.php | 2 +- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/db/install.xml b/lib/db/install.xml index 2cbd0e94b35..9dbff3d99c7 100644 --- a/lib/db/install.xml +++ b/lib/db/install.xml @@ -1,5 +1,5 @@ - @@ -2753,7 +2753,9 @@ - + + + @@ -2874,4 +2876,4 @@ - + \ No newline at end of file diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php index f0c8ba1bfef..b85b3785fe6 100644 --- a/lib/db/upgrade.php +++ b/lib/db/upgrade.php @@ -830,5 +830,29 @@ function xmldb_main_upgrade($oldversion) { upgrade_main_savepoint(true, 2012060600.04); } + if ($oldversion < 2012061800.01) { + // Add field newcontextid to backup_files_template + $table = new xmldb_table('backup_files_template'); + $field = new xmldb_field('newcontextid', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'info'); + + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + + upgrade_main_savepoint(true, 2012061800.01); + } + + if ($oldversion < 2012061800.02) { + // Add field newitemid to backup_files_template + $table = new xmldb_table('backup_files_template'); + $field = new xmldb_field('newitemid', XMLDB_TYPE_INTEGER, '10', null, null, null, null, 'newcontextid'); + + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + + upgrade_main_savepoint(true, 2012061800.02); + } + return true; } diff --git a/version.php b/version.php index 2d8e83a16d2..c4fe5270c3c 100644 --- a/version.php +++ b/version.php @@ -30,7 +30,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2012061800.00; // YYYYMMDD = weekly release date of this DEV branch +$version = 2012061800.02; // YYYYMMDD = weekly release date of this DEV branch // RR = release increments - 00 in DEV branches // .XX = incremental changes From 83b2cc562dd9bd5d8b7aaae5482908978ea3acd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Wed, 20 Jun 2012 11:36:41 +0200 Subject: [PATCH 05/10] MDL-33430 Set newcontextid and newitemid when sending file to the pool --- backup/util/dbops/restore_dbops.class.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/backup/util/dbops/restore_dbops.class.php b/backup/util/dbops/restore_dbops.class.php index de3ff5fd06b..2e73e8fcee1 100644 --- a/backup/util/dbops/restore_dbops.class.php +++ b/backup/util/dbops/restore_dbops.class.php @@ -850,17 +850,17 @@ abstract class restore_dbops { // itemname = null, we are going to match only by context, no need to use itemid (all them are 0) if ($itemname == null) { - $sql = 'SELECT contextid, component, filearea, itemid, itemid AS newitemid, info + $sql = "SELECT id AS bftid, contextid, component, filearea, itemid, itemid AS newitemid, info FROM {backup_files_temp} WHERE backupid = ? AND contextid = ? AND component = ? - AND filearea = ?'; + AND filearea = ?"; $params = array($restoreid, $oldcontextid, $component, $filearea); // itemname not null, going to join with backup_ids to perform the old-new mapping of itemids } else { - $sql = "SELECT f.contextid, f.component, f.filearea, f.itemid, i.newitemid, f.info + $sql = "SELECT f.id AS bftid, f.contextid, f.component, f.filearea, f.itemid, i.newitemid, f.info FROM {backup_files_temp} f JOIN {backup_ids_temp} i ON i.backupid = f.backupid $parentitemctxmatchsql @@ -925,6 +925,13 @@ abstract class restore_dbops { $fs->create_file_from_pathname($file_record, $backuppath); } + // store the the new contextid and the new itemid in case we need to remap + // references to this file later + $DB->update_record('backup_files_temp', array( + 'id' => $rec->bftid, + 'newcontextid' => $newcontextid, + 'newitemid' => $rec->newitemid), true); + } else { // this is an alias - we can't create it yet so we stash it in a temp // table and will let the final task to deal with it From 2c1426743a63145fc39cd739b576a337b4648ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Thu, 21 Jun 2012 01:05:06 +0200 Subject: [PATCH 06/10] MDL-33430 Add process_file_aliases_queue restore step to the final task This execution step tries to actually restore all aliases that were stashed in the file aliases queue. --- backup/moodle2/restore_final_task.class.php | 8 +- backup/moodle2/restore_stepslib.php | 275 ++++++++++++++++++++ 2 files changed, 282 insertions(+), 1 deletion(-) diff --git a/backup/moodle2/restore_final_task.class.php b/backup/moodle2/restore_final_task.class.php index 3f627bf16af..d336631eb5e 100644 --- a/backup/moodle2/restore_final_task.class.php +++ b/backup/moodle2/restore_final_task.class.php @@ -83,9 +83,15 @@ class restore_final_task extends restore_task { // executing it to perform some final adjustments of information // not available when the task was executed. // This step is always the last one performing modifications on restored information - // Don't add any new step after it. Only cache rebuild and clean are allowed. + // Don't add any new step after it. Only aliases queue, cache rebuild and clean are allowed. $this->add_step(new restore_execute_after_restore('executing_after_restore')); + // All files were sent to the filepool by now. We need to process + // the aliases yet as they were not actually created but stashed for us instead. + // We execute this step after executing_after_restore so that there can't be no + // more files sent to the filepool after this. + $this->add_step(new restore_process_file_aliases_queue('process_file_aliases_queue')); + // Rebuild course cache to see results, whoah! $this->add_step(new restore_rebuild_course_cache('rebuild_course_cache')); diff --git a/backup/moodle2/restore_stepslib.php b/backup/moodle2/restore_stepslib.php index 9bc8d436cf3..6178c29399e 100644 --- a/backup/moodle2/restore_stepslib.php +++ b/backup/moodle2/restore_stepslib.php @@ -3093,6 +3093,281 @@ class restore_create_question_files extends restore_execution_step { } } + +/** + * Try to restore aliases and references to external files. + * + * The queue of these files was prepared for us in {@link restore_dbops::send_files_to_pool()}. + * We expect that all regular (non-alias) files have already been restored. Make sure + * there is no restore step executed after this one that would call send_files_to_pool() again. + * + * You may notice we have hardcoded support for Server files, Legacy course files + * and user Private files here at the moment. This could be eventually replaced with a set of + * callbacks in the future if needed. + * + * @copyright 2012 David Mudrak + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class restore_process_file_aliases_queue extends restore_execution_step { + + /** + * What to do when this step is executed. + */ + protected function define_execution() { + global $DB; + + $this->log('processing file aliases queue', backup::LOG_INFO); + + $fs = get_file_storage(); + + // Load the queue. + $rs = $DB->get_recordset('backup_ids_temp', + array('backupid' => $this->get_restoreid(), 'itemname' => 'file_aliases_queue'), + '', 'info'); + + // Iterate over aliases in the queue. + foreach ($rs as $record) { + $info = unserialize(base64_decode($record->info)); + + // Try to pick a repository instance that should serve the alias. + $repository = $this->choose_repository($info); + + if (is_null($repository)) { + $this->notify_failure($info, 'unable to find a matching repository instance'); + continue; + } + + if ($info->oldfile->repositorytype === 'local' or $info->oldfile->repositorytype === 'coursefiles') { + // Aliases to Server files and Legacy course files may refer to a file + // contained in the backup file or to some existing file (if we are on the + // same site). + try { + $reference = file_storage::unpack_reference($info->oldfile->reference); + } catch (Exception $e) { + $this->notify_failure($info, 'invalid reference field format'); + continue; + } + + // Let's see if the referred source file was also included in the backup. + $candidates = $DB->get_recordset('backup_files_temp', array( + 'backupid' => $this->get_restoreid(), + 'contextid' => $reference['contextid'], + 'component' => $reference['component'], + 'filearea' => $reference['filearea'], + 'itemid' => $reference['itemid'], + ), '', 'info, newcontextid, newitemid'); + + $source = null; + + foreach ($candidates as $candidate) { + $candidateinfo = unserialize(base64_decode($candidate->info)); + if ($candidateinfo->filename === $reference['filename'] + and $candidateinfo->filepath === $reference['filepath'] + and !is_null($candidate->newcontextid) + and !is_null($candidate->newitemid) ) { + $source = $candidateinfo; + $source->contextid = $candidate->newcontextid; + $source->itemid = $candidate->newitemid; + break; + } + } + $candidates->close(); + + if ($source) { + // We have an alias that refers to another file also included in + // the backup. Let us change the reference field so that it refers + // to the restored copy of the original file. + $reference = file_storage::pack_reference($source); + + // Send the new alias to the filepool. + $fs->create_file_from_reference($info->newfile, $repository->id, $reference); + $this->notify_success($info); + continue; + + } else { + // This is a reference to some moodle file that was not contained in the backup + // file. If we are restoring to the same site, keep the reference untouched + // and restore the alias as is if the referenced file exists. + if ($this->task->is_samesite()) { + if ($fs->file_exists($reference['contextid'], $reference['component'], $reference['filearea'], + $reference['itemid'], $reference['filepath'], $reference['filename'])) { + $reference = file_storage::pack_reference($reference); + $fs->create_file_from_reference($info->newfile, $repository->id, $reference); + $this->notify_success($info); + continue; + } else { + $this->notify_failure($info, 'referenced file not found'); + continue; + } + + // If we are at other site, we can't restore this alias. + } else { + $this->notify_failure($info, 'referenced file not included'); + continue; + } + } + + } else if ($info->oldfile->repositorytype === 'user') { + if ($this->task->is_samesite()) { + // For aliases to user Private files at the same site, we have a chance to check + // if the referenced file still exists. + try { + $reference = file_storage::unpack_reference($info->oldfile->reference); + } catch (Exception $e) { + $this->notify_failure($info, 'invalid reference field format'); + continue; + } + if ($fs->file_exists($reference['contextid'], $reference['component'], $reference['filearea'], + $reference['itemid'], $reference['filepath'], $reference['filename'])) { + $reference = file_storage::pack_reference($reference); + $fs->create_file_from_reference($info->newfile, $repository->id, $reference); + $this->notify_success($info); + continue; + } else { + $this->notify_failure($info, 'referenced file not found'); + continue; + } + + // If we are at other site, we can't restore this alias. + } else { + $this->notify_failure($info, 'restoring at another site'); + continue; + } + + } else { + // This is a reference to some external file such as in boxnet or dropbox. + // If we are restoring to the same site, keep the reference untouched and + // restore the alias as is. + if ($this->task->is_samesite()) { + $fs->create_file_from_reference($info->newfile, $repository->id, $info->oldfile->reference); + $this->notify_success($info); + continue; + + // If we are at other site, we can't restore this alias. + } else { + $this->notify_failure($info, 'restoring at another site'); + continue; + } + } + } + $rs->close(); + } + + /** + * Choose the repository instance that should handle the alias. + * + * At the same site, we can rely on repository instance id and we just + * check it still exists. On other site, try to find matching Server files or + * Legacy course files repository instance. Return null if no matching + * repository instance can be found. + * + * @param stdClass $info + * @return repository|null + */ + private function choose_repository(stdClass $info) { + global $DB, $CFG; + require_once($CFG->dirroot.'/repository/lib.php'); + + if ($this->task->is_samesite()) { + // We can rely on repository instance id. + try { + return repository::get_repository_by_id($info->oldfile->repositoryid, SYSCONTEXTID); + } catch (Exception $e) { + return null; + } + + } else { + // We can rely on repository type only. + if (empty($info->oldfile->repositorytype)) { + return null; + } + + // Both Server files and Legacy course files repositories have a single + // instance at the system context to use. Let us try to find it. + if ($info->oldfile->repositorytype === 'local' or $info->oldfile->repositorytype === 'coursefiles') { + $sql = "SELECT ri.id + FROM {repository} r + JOIN {repository_instances} ri ON ri.typeid = r.id + WHERE r.type = ? AND ri.contextid = ?"; + $ris = $DB->get_records_sql($sql, array($info->oldfile->repositorytype, SYSCONTEXTID)); + if (empty($ris)) { + return null; + } + $repoid = reset(array_keys($ris)); + try { + return repository::get_repository_by_id($repoid, SYSCONTEXTID); + } catch (Exception $e) { + return null; + } + } + + return null; + } + } + + /** + * Let the user know that the given alias was successfully restored + * + * @param stdClass $info + */ + private function notify_success(stdClass $info) { + $filedesc = $this->describe_alias($info); + $this->log('successfully restored alias', backup::LOG_DEBUG, $filedesc, 1); + } + + /** + * Let the user know that the given alias can't be restored + * + * @param stdClass $info + * @param string $reason detailed reason to be logged + */ + private function notify_failure(stdClass $info, $reason = '') { + $filedesc = $this->describe_alias($info); + if ($reason) { + $reason = ' ('.$reason.')'; + } + $this->log('unable to restore alias'.$reason, backup::LOG_WARNING, $filedesc, 1); + } + + /** + * Return a human readable description of the alias file + * + * @param stdClass $info + * @return string + */ + private function describe_alias(stdClass $info) { + + $filedesc = $this->expected_alias_location($info->newfile); + + if (!is_null($info->oldfile->source)) { + $filedesc .= ' ('.$info->oldfile->source.')'; + } + + return $filedesc; + } + + /** + * Return the expected location of a file + * + * Please note this may and may not work as a part of URL to pluginfile.php + * (depends on how the given component/filearea deals with the itemid). + * + * @param stdClass $filerecord + * @return string + */ + private function expected_alias_location($filerecord) { + + $filedesc = '/'.$filerecord->contextid.'/'.$filerecord->component.'/'.$filerecord->filearea; + if (!is_null($filerecord->itemid)) { + $filedesc .= '/'.$filerecord->itemid; + } + $filedesc .= $filerecord->filepath.$filerecord->filename; + + return $filedesc; + } +} + + /** * Abstract structure step, to be used by all the activities using core questions stuff * (like the quiz module), to support qtype plugins, states and sessions From d129c3608f158c8157a8c33ff21e7058b7c5e185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Thu, 21 Jun 2012 12:15:48 +0200 Subject: [PATCH 07/10] MDL-33430 Add caching into choose_repository() method Simply do not repeat expensive search once the restore process decides how to deal with the given repositoryid or repositorytype. --- backup/moodle2/restore_stepslib.php | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/backup/moodle2/restore_stepslib.php b/backup/moodle2/restore_stepslib.php index 6178c29399e..503521c9d00 100644 --- a/backup/moodle2/restore_stepslib.php +++ b/backup/moodle2/restore_stepslib.php @@ -3110,6 +3110,12 @@ class restore_create_question_files extends restore_execution_step { */ class restore_process_file_aliases_queue extends restore_execution_step { + /** @var array internal cache for {@link choose_repository() */ + private $cachereposbyid = array(); + + /** @var array internal cache for {@link choose_repository() */ + private $cachereposbytype = array(); + /** * What to do when this step is executed. */ @@ -3270,18 +3276,34 @@ class restore_process_file_aliases_queue extends restore_execution_step { if ($this->task->is_samesite()) { // We can rely on repository instance id. + + if (array_key_exists($info->oldfile->repositoryid, $this->cachereposbyid)) { + return $this->cachereposbyid[$info->oldfile->repositoryid]; + } + + $this->log('looking for repository instance by id', backup::LOG_DEBUG, $info->oldfile->repositoryid, 1); + try { - return repository::get_repository_by_id($info->oldfile->repositoryid, SYSCONTEXTID); + $this->cachereposbyid[$info->oldfile->repositoryid] = repository::get_repository_by_id($info->oldfile->repositoryid, SYSCONTEXTID); + return $this->cachereposbyid[$info->oldfile->repositoryid]; } catch (Exception $e) { + $this->cachereposbyid[$info->oldfile->repositoryid] = null; return null; } } else { // We can rely on repository type only. + if (empty($info->oldfile->repositorytype)) { return null; } + if (array_key_exists($info->oldfile->repositorytype, $this->cachereposbytype)) { + return $this->cachereposbytype[$info->oldfile->repositorytype]; + } + + $this->log('looking for repository instance by type', backup::LOG_DEBUG, $info->oldfile->repositorytype, 1); + // Both Server files and Legacy course files repositories have a single // instance at the system context to use. Let us try to find it. if ($info->oldfile->repositorytype === 'local' or $info->oldfile->repositorytype === 'coursefiles') { @@ -3295,12 +3317,15 @@ class restore_process_file_aliases_queue extends restore_execution_step { } $repoid = reset(array_keys($ris)); try { - return repository::get_repository_by_id($repoid, SYSCONTEXTID); + $this->cachereposbytype[$info->oldfile->repositorytype] = repository::get_repository_by_id($repoid, SYSCONTEXTID); + return $this->cachereposbytype[$info->oldfile->repositorytype]; } catch (Exception $e) { + $this->cachereposbytype[$info->oldfile->repositorytype] = null; return null; } } + $this->cachereposbytype[$info->oldfile->repositorytype] = null; return null; } } From fb3a57e2d44de02f837105319f1267c1723b7578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Thu, 21 Jun 2012 12:27:06 +0200 Subject: [PATCH 08/10] MDL-33430 Fix strict standards warning Only variables should be passed by reference. --- backup/moodle2/restore_stepslib.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backup/moodle2/restore_stepslib.php b/backup/moodle2/restore_stepslib.php index 503521c9d00..91048d80286 100644 --- a/backup/moodle2/restore_stepslib.php +++ b/backup/moodle2/restore_stepslib.php @@ -3315,7 +3315,8 @@ class restore_process_file_aliases_queue extends restore_execution_step { if (empty($ris)) { return null; } - $repoid = reset(array_keys($ris)); + $repoids = array_keys($ris); + $repoid = reset($repoids); try { $this->cachereposbytype[$info->oldfile->repositorytype] = repository::get_repository_by_id($repoid, SYSCONTEXTID); return $this->cachereposbytype[$info->oldfile->repositorytype]; From ff8734f21d9c313f3100288a4a16a143a6ae6206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Fri, 15 Jun 2012 16:46:08 +0200 Subject: [PATCH 09/10] MDL-33430 Give tasks an access to their current plan's results This in turn provides access to the plan's results for both structure and execution steps so they can register something useful there. --- backup/util/plan/base_plan.class.php | 17 ++++++++++++++++ backup/util/plan/base_task.class.php | 30 +++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/backup/util/plan/base_plan.class.php b/backup/util/plan/base_plan.class.php index eab1869d6d3..d8c4b4d7c98 100644 --- a/backup/util/plan/base_plan.class.php +++ b/backup/util/plan/base_plan.class.php @@ -75,10 +75,27 @@ abstract class base_plan implements checksumable, executable { return $this->tasks; } + /** + * Add the passed info to the plan results + * + * At the moment we expect an associative array structure to be merged into + * the current results. In the future, some sort of base_result class may + * be introduced. + * + * @param array $result associative array describing a result of a task/step + */ public function add_result($result) { + if (!is_array($result)) { + throw new coding_exception('Associative array is expected as a parameter of add_result()'); + } $this->results = array_merge($this->results, $result); } + /** + * Return the results collected via {@link self::add_result()} method + * + * @return array + */ public function get_results() { return $this->results; } diff --git a/backup/util/plan/base_task.class.php b/backup/util/plan/base_task.class.php index 1527634713f..440521b54de 100644 --- a/backup/util/plan/base_task.class.php +++ b/backup/util/plan/base_task.class.php @@ -154,7 +154,7 @@ abstract class base_task implements checksumable, executable, loggable { // If step returns array, it will be forwarded to plan // (TODO: shouldn't be array but proper result object) if (is_array($result) and !empty($result)) { - $this->plan->add_result($result); + $this->add_result($result); } } // Mark as executed if any step has been executed @@ -191,6 +191,34 @@ abstract class base_task implements checksumable, executable, loggable { backup_general_helper::array_checksum_recursive($this->steps)); } + /** + * Add the given info to the current plan's results. + * + * @see base_plan::add_result() + * @param array $result associative array describing a result of a task/step + */ + public function add_result($result) { + if (!is_null($this->plan)) { + $this->plan->add_result($result); + } else { + debugging('Attempting to add a result of a task not binded with a plan', DEBUG_DEVELOPER); + } + } + + /** + * Return the current plan's results + * + * @return array|null + */ + public function get_results() { + if (!is_null($this->plan)) { + return $this->plan->get_results(); + } else { + debugging('Attempting to get results of a task not binded with a plan', DEBUG_DEVELOPER); + return null; + } + } + // Protected API starts here /** From 7b5f67f77fbed8f4876edab00df21d39d9bad7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mudr=C3=A1k?= Date: Thu, 21 Jun 2012 15:22:38 +0200 Subject: [PATCH 10/10] MDL-33430 Notify user about file aliases failures at the final stage of the restore --- backup/moodle2/restore_stepslib.php | 24 +++++++++++++++++++++++ backup/util/ui/restore_ui_stage.class.php | 14 +++++++++++++ lang/en/backup.php | 6 ++++++ theme/base/style/core.css | 2 ++ 4 files changed, 46 insertions(+) diff --git a/backup/moodle2/restore_stepslib.php b/backup/moodle2/restore_stepslib.php index 91048d80286..adb3cd6b968 100644 --- a/backup/moodle2/restore_stepslib.php +++ b/backup/moodle2/restore_stepslib.php @@ -3353,6 +3353,7 @@ class restore_process_file_aliases_queue extends restore_execution_step { $reason = ' ('.$reason.')'; } $this->log('unable to restore alias'.$reason, backup::LOG_WARNING, $filedesc, 1); + $this->add_result_item('file_aliases_restore_failures', $filedesc); } /** @@ -3391,6 +3392,29 @@ class restore_process_file_aliases_queue extends restore_execution_step { return $filedesc; } + + /** + * Append a value to the given resultset + * + * @param string $name name of the result containing a list of values + * @param mixed $value value to add as another item in that result + */ + private function add_result_item($name, $value) { + + $results = $this->task->get_results(); + + if (isset($results[$name])) { + if (!is_array($results[$name])) { + throw new coding_exception('Unable to append a result item into a non-array structure.'); + } + $current = $results[$name]; + $current[] = $value; + $this->task->add_result(array($name => $current)); + + } else { + $this->task->add_result(array($name => array($value))); + } + } } diff --git a/backup/util/ui/restore_ui_stage.class.php b/backup/util/ui/restore_ui_stage.class.php index c8ecc9eab50..7e920695502 100644 --- a/backup/util/ui/restore_ui_stage.class.php +++ b/backup/util/ui/restore_ui_stage.class.php @@ -757,6 +757,20 @@ class restore_ui_stage_complete extends restore_ui_stage_process { public function display(core_backup_renderer $renderer) { $html = ''; + if (!empty($this->results['file_aliases_restore_failures'])) { + $html .= $renderer->box_start('generalbox filealiasesfailures'); + $html .= $renderer->heading_with_help(get_string('filealiasesrestorefailures', 'core_backup'), + 'filealiasesrestorefailures', 'core_backup'); + $html .= $renderer->container(get_string('filealiasesrestorefailuresinfo', 'core_backup')); + $html .= $renderer->container_start('aliaseslist'); + $html .= html_writer::start_tag('ul'); + foreach ($this->results['file_aliases_restore_failures'] as $alias) { + $html .= html_writer::tag('li', s($alias)); + } + $html .= html_writer::end_tag('ul'); + $html .= $renderer->container_end(); + $html .= $renderer->box_end(); + } $html .= $renderer->box_start(); $html .= $renderer->notification(get_string('restoreexecutionsuccess', 'backup'), 'notifysuccess'); $html .= $renderer->continue_button(new moodle_url('/course/view.php', array( diff --git a/lang/en/backup.php b/lang/en/backup.php index fa809254ddb..5fde19e8722 100644 --- a/lang/en/backup.php +++ b/lang/en/backup.php @@ -117,6 +117,12 @@ $string['errorinvalidformat'] = 'Unknown backup format'; $string['errorinvalidformatinfo'] = 'The selected file is not a valid Moodle backup file and can\'t be restored.'; $string['executionsuccess'] = 'The backup file was successfully created.'; $string['filename'] = 'Filename'; +$string['filealiasesrestorefailures'] = 'Aliases restore failures'; +$string['filealiasesrestorefailuresinfo'] = 'Some aliases included in the backup file could not be restored. The following list contains their expected location and the source file they were referring to at the original site.'; +$string['filealiasesrestorefailures_help'] = 'Aliases are symbolic links to other files, including those stored in external repositories. In some cases, Moodle cannot restore them - for example when restoring the backup at another site or when the referenced file does not exist. + +More details and the actual reason of the failure can be found in the restore log file.'; +$string['filealiasesrestorefailures_link'] = 'restore/filealiases'; $string['filereferencesincluded'] = 'File references to external contents included in backup package, they won\'t work on other sites.'; $string['filereferencessamesite'] = 'Backup is from the same site, file references can be restored'; $string['filereferencesnotsamesite'] = 'Backup is from other site, file references cannot be restored'; diff --git a/theme/base/style/core.css b/theme/base/style/core.css index 28153ca9439..efa27f86456 100644 --- a/theme/base/style/core.css +++ b/theme/base/style/core.css @@ -461,6 +461,8 @@ body.tag .managelink {padding: 5px;} .path-backup .backup_progress .backup_stage.backup_stage_current {font-weight:bold;color:inherit;} .path-backup .backup_progress .backup_stage.backup_stage_next {} .path-backup .backup_progress span.backup_stage.backup_stage_complete {color:inherit;} +#page-backup-restore .filealiasesfailures {background-color:#ffd3d9} +#page-backup-restore .filealiasesfailures .aliaseslist {width:90%;margin:0.8em auto;background-color:white;border:1px dotted #666;} /** * Web Service