From 2df0f295b4fbb258bbb38a64e861cf58d92f9fce Mon Sep 17 00:00:00 2001 From: Eloy Lafuente Date: Sun, 18 Jul 2010 01:35:18 +0000 Subject: [PATCH] MDL-21432 backup - file handling and other changes. Saving before changes tomorrow due to MySQL temp tables limits --- backup/util/dbops/restore_dbops.class.php | 98 +++++++++++++++++-- .../util/helper/backup_file_manager.class.php | 19 ---- .../helper/backup_general_helper.class.php | 1 + backup/util/includes/restore_includes.php | 1 + .../plan/restore_structure_step.class.php | 86 +++++++++++++++- backup/util/plan/restore_task.class.php | 4 + 6 files changed, 182 insertions(+), 27 deletions(-) diff --git a/backup/util/dbops/restore_dbops.class.php b/backup/util/dbops/restore_dbops.class.php index 382bc0fd90e..e68f3745fd0 100644 --- a/backup/util/dbops/restore_dbops.class.php +++ b/backup/util/dbops/restore_dbops.class.php @@ -100,7 +100,7 @@ abstract class restore_dbops { public static function load_users_to_tempids($restoreid, $usersfile) { if (!file_exists($usersfile)) { // Shouldn't happen ever, but... - throw new backup_helper_exception('missing_users_xml_file', $inforeffile); + throw new backup_helper_exception('missing_users_xml_file', $usersfile); } // Let's parse, custom processor will do its work, sending info to DB $xmlparser = new progressive_parser(); @@ -110,6 +110,88 @@ abstract class restore_dbops { $xmlparser->process(); } + /** + * Given one component/filearea/context and + * optionally one source itemname to match itemids + * put the corresponding files in the pool + */ + public static function send_files_to_pool($basepath, $restoreid, $component, $filearea, $oldcontextid, $itemname = null) { + global $DB; + + // Get new context, must exist or this will fail + if (!$newcontextid = self::get_backup_ids_record($restoreid, 'context', $oldcontextid)->newitemid) { + throw new restore_dbops_exception('unknown_context_mapping', $oldcontextid); + } + + // Important: remember how files have been loaded to backup_ids_temp + // - itemname: contains "file*component*fileara" + // - itemid: contains the original id of the file + // - newitemid: contains the itemid of the file + // - parentitemid: contains the context of the file + // - info: contains the whole original object (times, names...) + // (all them being original ids as loaded from xml) + + // itemname = null, we are going to match only by context, no need to use itemid (all them are 0) + if ($itemname == null) { + $sql = 'SELECT id, itemname, itemid, 0 AS newitemid + FROM {backup_ids_temp} + WHERE backupid = ? + AND itemname = ? + AND parentitemid = ?'; + $params = array($restoreid, 'file*' . $component . '*' . $filearea, $oldcontextid); + + // itemname not null, going to join by context and itemid, we'll need itemid (non zero) + } else { + $sql = 'SELECT f.id, f.itemname, f.itemid, i.newitemid + FROM {backup_ids_temp} f + JOIN {backup_ids_temp} i ON i.backupid = f.backupid + AND i.parentitemid = f.parentitemid + AND i.itemid = f.newitemid + WHERE f.backupid = ? + AND f.itemname = ? + AND f.parentitemid = ? + AND i.itemname = ?'; + $params = array($restoreid, 'file*' . $component . '*' . $filearea, $oldcontextid, $itemname); + } + + $rs = $DB->get_recordset_sql($sql, $params); + $fs = get_file_storage(); // Get moodle file storage + $basepath = $basepath . '/files/';// Get backup file pool base + foreach ($rs as $rec) { + $file = (object)self::get_backup_ids_record($restoreid, $rec->itemname, $rec->itemid)->info; + // ignore root dirs (they are created automatically) + if ($file->filepath == '/' && $file->filename == '.') { + continue; + } + // dir found (and not root one), let's create if + if ($file->filename == '.') { + $fs->create_directory($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath); + continue; + } + // arrived here, file found + // Find file in backup pool + $backuppath = $basepath . backup_file_manager::get_content_file_location($file->contenthash); + if (!file_exists($backuppath)) { + throw new restore_dbops_exception('file_not_found_in_pool', $file); + } + if (!$fs->file_exists($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->filename)) { + $file_record = array( + 'contextid' => $newcontextid, + 'component' => $component, + 'filearea' => $filearea, + 'itemid' => $rec->newitemid, + 'filepath' => $file->filepath, + 'filename' => $file->filename, + 'timecreated' => $file->timecreated, + 'timemodified'=> $file->timemodified, + 'author' => $file->author, + 'license' => $file->license); + $fs->create_file_from_pathname($file_record, $backuppath); + } + } + $rs->close(); + } + /** * Given one restoreid, create in DB all the users present * in backup_ids having newitemid = 0, as far as @@ -117,7 +199,7 @@ abstract class restore_dbops { * ready to be created. Also, annotate their newids * once created for later reference */ - protected static function create_included_users($restoreid) { + public static function create_included_users($basepath, $restoreid, $userfiles) { global $CFG, $DB; $authcache = array(); // Cache to get some bits from authentication plugins @@ -244,7 +326,7 @@ abstract class restore_dbops { // Process preferences if (isset($user->preferences)) { // if present in backup - foreach($user->preferences as $preference) { + foreach($user->preferences['preference'] as $preference) { $preference = (object)$preference; // Prepare the record and insert it $preference->userid = $newuserid; @@ -252,6 +334,13 @@ abstract class restore_dbops { } } + // Create user files in pool (profile, icon, private) by context + restore_dbops::send_files_to_pool($basepath, $restoreid, 'user', 'icon', $recuser->parentitemid); + restore_dbops::send_files_to_pool($basepath, $restoreid, 'user', 'profile', $recuser->parentitemid); + if ($userfiles) { // private files only if enabled in settings + restore_dbops::send_files_to_pool($basepath, $restoreid, 'user', 'private', $recuser->parentitemid); + } + } $rs->close(); } @@ -558,9 +647,6 @@ abstract class restore_dbops { if (!empty($problems)) { throw new restore_dbops_exception('restore_problems_processing_users', null, implode(', ', $problems)); } - - // Now, create all the users that we haven't been able to map - self::create_included_users($restoreid); } diff --git a/backup/util/helper/backup_file_manager.class.php b/backup/util/helper/backup_file_manager.class.php index f8ff0365029..5e3f21b7185 100644 --- a/backup/util/helper/backup_file_manager.class.php +++ b/backup/util/helper/backup_file_manager.class.php @@ -114,23 +114,4 @@ class backup_file_manager { } } } - - /** - * Copy one file from backup storage to moodle storage - */ - public static function copy_file_backup2moodle($backupid, $filerecorid) { - global $DB; - - // Normalise param - if (!is_object($filerecorid)) { - $filerecorid = $DB->get_record('files', array('id' => $filerecorid)); - } - - // Directory, nothing to do - if ($filerecorid->filename === '.') { - return; - } - - // TODO: Finish this on restore - } } diff --git a/backup/util/helper/backup_general_helper.class.php b/backup/util/helper/backup_general_helper.class.php index c07b4957a68..969fde04b19 100644 --- a/backup/util/helper/backup_general_helper.class.php +++ b/backup/util/helper/backup_general_helper.class.php @@ -139,6 +139,7 @@ abstract class backup_general_helper extends backup_helper { $info->original_wwwroot = $infoarr['original_wwwroot']; $info->original_site_identifier_hash = $infoarr['original_site_identifier_hash']; $info->original_course_id = $infoarr['original_course_id']; + $info->original_course_contextid= $infoarr['original_course_contextid']; $info->type = $infoarr['details']['detail'][0]['type']; $info->format = $infoarr['details']['detail'][0]['format']; $info->mode = $infoarr['details']['detail'][0]['mode']; diff --git a/backup/util/includes/restore_includes.php b/backup/util/includes/restore_includes.php index ac07c7d087e..8a8161eca7d 100644 --- a/backup/util/includes/restore_includes.php +++ b/backup/util/includes/restore_includes.php @@ -32,6 +32,7 @@ require_once($CFG->dirroot . '/backup/util/interfaces/checksumable.class.php'); require_once($CFG->dirroot . '/backup/util/interfaces/loggable.class.php'); require_once($CFG->dirroot . '/backup/util/interfaces/executable.class.php'); require_once($CFG->dirroot . '/backup/util/structure/restore_path_element.class.php'); +require_once($CFG->dirroot . '/backup/util/helper/backup_file_manager.class.php'); require_once($CFG->dirroot . '/backup/util/helper/restore_moodlexml_parser_processor.class.php'); require_once($CFG->dirroot . '/backup/util/helper/restore_inforef_parser_processor.class.php'); require_once($CFG->dirroot . '/backup/util/helper/restore_users_parser_processor.class.php'); diff --git a/backup/util/plan/restore_structure_step.class.php b/backup/util/plan/restore_structure_step.class.php index dc0bd9a8628..7e326b6860d 100644 --- a/backup/util/plan/restore_structure_step.class.php +++ b/backup/util/plan/restore_structure_step.class.php @@ -31,6 +31,8 @@ abstract class restore_structure_step extends restore_step { protected $filename; // Name of the file to be parsed protected $pathelements; // Array of pathelements to process + protected $elementsoldid; // Array to store last oldid used on each element + protected $elementsnewid; // Array to store last newid used on each element /** * Constructor - instantiates one object of this class @@ -41,6 +43,8 @@ abstract class restore_structure_step extends restore_step { } $this->filename = $filename; $this->pathelements = array(); + $this->elementsoldid = array(); + $this->elementsnewid = array(); parent::__construct($name, $task); } @@ -68,6 +72,12 @@ abstract class restore_structure_step extends restore_step { // Get restore_path elements array adapting and preparing it for processing $this->pathelements = $this->prepare_pathelements($this->define_structure()); + // Populate $elementsoldid and $elementsoldid based on available pathelements + foreach ($this->pathelements as $pathelement) { + $this->elementsoldid[$pathelement->get_name()] = null; + $this->elementsnewid[$pathelement->get_name()] = null; + } + // Create parser and processor $xmlparser = new progressive_parser(); $xmlparser->set_file($fullpath); @@ -81,9 +91,10 @@ abstract class restore_structure_step extends restore_step { // And process it, dispatch to target methods in step will start automatically $xmlparser->process(); - } -// Protected API starts here + // Have finished, call to the after_execute method + $this->after_execute(); + } /** * Receive one chunk of information form the xml parser processor and @@ -107,6 +118,19 @@ abstract class restore_structure_step extends restore_step { } } +// Protected API starts here + + /** + * This method will be executed after the whole structure step have been processed + * + * After execution method for code needed to be executed after the whole structure + * has been processed. Useful for cleaning tasks, files process and others. Simply + * overwrite in in your steps if needed + */ + protected function after_execute() { + // do nothing by default + } + /** * Prepare the pathelements for processing, looking for duplicates, applying * processing objects and other adjustments @@ -167,6 +191,64 @@ abstract class restore_structure_step extends restore_step { return true; } + /** + * To send ids pairs to backup_ids_table and to store them into paths + * + * This method will send the given itemname and old/new ids to the + * backup_ids_temp table, and, at the same time, will save the new id + * into the corresponding restore_path_element for easier access + * by children. Also will inject the known old context id for the task + * in case it's going to be used for restoring files later + */ + protected function set_mapping($itemname, $oldid, $newid, $restorefiles = false) { + $parentitemid = $restorefiles ? $this->task->get_old_contextid() : null; + // Let's call the low level one + restore_dbops::set_backup_ids_record($this->get_restoreid(), $itemname, $oldid, $newid, $parentitemid); + // Now, if the itemname matches any pathelement->name, store the latest $newid + if (array_key_exists($itemname, $this->elementsoldid)) { // If present in $this->elementsoldid, is valid, put both ids + $this->elementsoldid[$itemname] = $oldid; + $this->elementsnewid[$itemname] = $newid; + } + } + + /** + * Returns the latest (parent) old id mapped by one pathelement + */ + protected function get_old_parentid($itemname) { + return array_key_exists($itemname, $this->elementsoldid) ? $this->elementsoldid[$itemname] : null; + } + + /** + * Returns the latest (parent) new id mapped by one pathelement + */ + protected function get_new_parentid($itemname) { + return array_key_exists($itemname, $this->elementsnewid) ? $this->elementsnewid[$itemname] : null; + } + + /** + * Return the new id of a mapping for the given itemname + * + */ + protected function get_mappingid($itemname, $oldid) { + $mapping = $this->get_mapping($itemname, $oldid); + return $mapping ? $mapping->newitemid : false; + } + + /** + * Return the complete mapping from the given itemname, itemid + */ + protected function get_mapping($itemname, $oldid) { + return restore_dbops::get_backup_ids_record($this->get_restoreid(), $itemname, $oldid); + } + + /** + * Add all the existing file, given their component and filearea and one backup_ids itemname to match with + */ + protected function add_related_files($componentname, $filearea, $mappingitemname) { + restore_dbops::send_files_to_pool($this->get_basepath(), $this->get_restoreid(), $componentname, + $filearea, $this->task->get_old_contextid(), $mappingitemname); + } + /** * Function that will return the structure to be processed by this restore_step. * Must return one array of @restore_path_element elements diff --git a/backup/util/plan/restore_task.class.php b/backup/util/plan/restore_task.class.php index 194b0d6a91b..a63ea10e986 100644 --- a/backup/util/plan/restore_task.class.php +++ b/backup/util/plan/restore_task.class.php @@ -66,6 +66,10 @@ abstract class restore_task extends base_task { public function get_preloaded_information() { return $this->plan->get_preloaded_information(); } + + public function get_old_contextid() { + return $this->plan->get_info()->original_course_contextid; + } } /*