diff --git a/admin/settings/courses.php b/admin/settings/courses.php index ace90d3f057..c07f26fe5ee 100644 --- a/admin/settings/courses.php +++ b/admin/settings/courses.php @@ -206,6 +206,11 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) { $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_role_assignments', new lang_string('generalroleassignments','backup'), new lang_string('configgeneralroleassignments','backup'), array('value'=>1, 'locked'=>0))); $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_activities', new lang_string('generalactivities','backup'), new lang_string('configgeneralactivities','backup'), array('value'=>1, 'locked'=>0))); $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_blocks', new lang_string('generalblocks','backup'), new lang_string('configgeneralblocks','backup'), array('value'=>1, 'locked'=>0))); + $temp->add(new admin_setting_configcheckbox_with_lock( + 'backup/backup_general_files', + new lang_string('generalfiles', 'backup'), + new lang_string('configgeneralfiles', 'backup'), + array('value' => '1', 'locked' => 0))); $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_filters', new lang_string('generalfilters','backup'), new lang_string('configgeneralfilters','backup'), array('value'=>1, 'locked'=>0))); $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_comments', new lang_string('generalcomments','backup'), new lang_string('configgeneralcomments','backup'), array('value'=>1, 'locked'=>0))); $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_badges', new lang_string('generalbadges','backup'), new lang_string('configgeneralbadges','backup'), array('value'=>1,'locked'=>0))); @@ -342,6 +347,10 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) { $temp->add(new admin_setting_configcheckbox('backup/backup_auto_activities', new lang_string('generalactivities','backup'), new lang_string('configgeneralactivities','backup'), 1)); $temp->add(new admin_setting_configcheckbox('backup/backup_auto_blocks', new lang_string('generalblocks','backup'), new lang_string('configgeneralblocks','backup'), 1)); $temp->add(new admin_setting_configcheckbox('backup/backup_auto_filters', new lang_string('generalfilters','backup'), new lang_string('configgeneralfilters','backup'), 1)); + $temp->add(new admin_setting_configcheckbox( + 'backup/backup_auto_files', + new lang_string('generalfiles', 'backup'), + new lang_string('configgeneralfiles', 'backup'), '1')); $temp->add(new admin_setting_configcheckbox('backup/backup_auto_comments', new lang_string('generalcomments','backup'), new lang_string('configgeneralcomments','backup'), 1)); $temp->add(new admin_setting_configcheckbox('backup/backup_auto_badges', new lang_string('generalbadges','backup'), new lang_string('configgeneralbadges','backup'), 1)); $temp->add(new admin_setting_configcheckbox('backup/backup_auto_calendarevents', new lang_string('generalcalendarevents','backup'), new lang_string('configgeneralcalendarevents','backup'), 1)); diff --git a/admin/settings/server.php b/admin/settings/server.php index a869ce6294d..cfb2f21acb6 100644 --- a/admin/settings/server.php +++ b/admin/settings/server.php @@ -172,7 +172,10 @@ $temp->add(new admin_setting_configselect('tempdatafoldercleanup', new lang_stri $ADMIN->add('server', $temp); - + $temp->add(new admin_setting_configduration('filescleanupperiod', + new lang_string('filescleanupperiod', 'admin'), + new lang_string('filescleanupperiod_help', 'admin'), + 86400)); $ADMIN->add('server', new admin_externalpage('environment', new lang_string('environment','admin'), "$CFG->wwwroot/$CFG->admin/environment.php")); $ADMIN->add('server', new admin_externalpage('phpinfo', new lang_string('phpinfo'), "$CFG->wwwroot/$CFG->admin/phpinfo.php")); diff --git a/backup/backup.php b/backup/backup.php index e0606d1b384..1be7dcde060 100644 --- a/backup/backup.php +++ b/backup/backup.php @@ -150,6 +150,13 @@ if (!async_helper::is_async_pending($id, 'course', 'backup')) { $loghtml = ''; if ($backup->get_stage() == backup_ui::STAGE_FINAL) { + // Before we perform the backup check settings to see if user + // or setting defaults are set to exclude files from the backup. + if ($backup->get_setting_value('files') == 0) { + $bc->set_mode(backup::MODE_SAMESITE); + $renderer->set_samesite_notification(); + } + if ($backupmode != backup::MODE_ASYNC) { // Synchronous backup handling. @@ -180,6 +187,7 @@ if (!async_helper::is_async_pending($id, 'course', 'backup')) { // Hide the progress display and first backup step bar (the 'finished' step will show next). echo html_writer::end_div(); echo html_writer::script('document.getElementById("executionprogress").style.display = "none";'); + } else { // Async backup handling. $backup->get_controller()->finish_ui(); @@ -203,6 +211,8 @@ if (!async_helper::is_async_pending($id, 'course', 'backup')) { 'restoreurl' => $restoreurl->out(), 'headingident' => 'backup' ); + + echo $renderer->set_samesite_notification(); echo $renderer->render_from_template('core/async_backup_status', $progresssetup); } diff --git a/backup/controller/backup_controller.class.php b/backup/controller/backup_controller.class.php index 9ac5f5b831f..245fca4fe9b 100644 --- a/backup/controller/backup_controller.class.php +++ b/backup/controller/backup_controller.class.php @@ -185,6 +185,20 @@ class backup_controller extends base_controller { backup_check::check_security($this, false); } + /** + * Sets the mode (purpose) of the backup. + * + * @param int $mode The mode to set. + */ + public function set_mode($mode) { + $this->mode = $mode; + $this->set_include_files(); // Need to check if files are included as mode may have changed. + $this->save_controller(); + $tbc = self::load_controller($this->backupid); + $this->logger = $tbc->logger; // Wakeup loggers. + $tbc->plan->destroy(); // Clean plan controller structures, keeping logger alive. + } + public function set_status($status) { // Note: never save_controller() with the object info after STATUS_EXECUTING or the whole controller, // containing all the steps will be sent to DB. 100% (monster) useless. @@ -410,6 +424,13 @@ class backup_controller extends base_controller { $includefiles = false; } + // If backup is automated and we have set auto backup config to exclude + // files then set them to be excluded here. + $backupautofiles = (bool)get_config('backup', 'backup_auto_files'); + if ($this->get_mode() === backup::MODE_AUTOMATED && !$backupautofiles) { + $includefiles = false; + } + $this->includefiles = (int) $includefiles; $this->log("setting file inclusion to {$this->includefiles}", backup::LOG_DEBUG); return $this->includefiles; diff --git a/backup/moodle2/backup_root_task.class.php b/backup/moodle2/backup_root_task.class.php index 03e989da279..ee434c1e9d3 100644 --- a/backup/moodle2/backup_root_task.class.php +++ b/backup/moodle2/backup_root_task.class.php @@ -110,6 +110,12 @@ class backup_root_task extends backup_task { $this->add_setting($blocks); $this->converter_deps($blocks, $converters); + // Define files. + $files = new backup_generic_setting('files', base_setting::IS_BOOLEAN, true); + $files->set_ui(new backup_setting_ui_checkbox($files, get_string('rootsettingfiles', 'backup'))); + $this->add_setting($files); + $this->converter_deps($files, $converters); + // Define filters $filters = new backup_generic_setting('filters', base_setting::IS_BOOLEAN, true); $filters->set_ui(new backup_setting_ui_checkbox($filters, get_string('rootsettingfilters', 'backup'))); diff --git a/backup/util/dbops/backup_plan_dbops.class.php b/backup/util/dbops/backup_plan_dbops.class.php index 615f802fc6c..d3a650bf91e 100644 --- a/backup/util/dbops/backup_plan_dbops.class.php +++ b/backup/util/dbops/backup_plan_dbops.class.php @@ -201,7 +201,8 @@ abstract class backup_plan_dbops extends backup_dbops { * @param bool $useidonly only use the ID in the file name * @return string The filename to use */ - public static function get_default_backup_filename($format, $type, $id, $users, $anonymised, $useidonly = false) { + public static function get_default_backup_filename($format, $type, $id, $users, $anonymised, + $useidonly = false, $files = true) { global $DB; // Calculate backup word @@ -251,6 +252,11 @@ abstract class backup_plan_dbops extends backup_dbops { $info = '-an'; } + // Indicate if backup doesn't contain files. + if (!$files) { + $info .= '-nf'; + } + return $backupword . '-' . $format . '-' . $type . '-' . $name . '-' . $date . $info . '.mbz'; } diff --git a/backup/util/dbops/restore_dbops.class.php b/backup/util/dbops/restore_dbops.class.php index 7ef9117e230..165864ec269 100644 --- a/backup/util/dbops/restore_dbops.class.php +++ b/backup/util/dbops/restore_dbops.class.php @@ -1054,17 +1054,25 @@ abstract class restore_dbops { // Create the file in the filepool if it does not exist yet. if (!$fs->file_exists($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->filename)) { - // Even if a file has been deleted since the backup was made, the file metadata will remain in the - // files table, and the file will not be moved to the trashdir. - // Files are not cleared from the files table by cron until several days after deletion. + // Even if a file has been deleted since the backup was made, the file metadata may remain in the + // files table, and the file will not yet have been moved to the trashdir. e.g. a draft file version. + // Try to recover from file table first. if ($foundfiles = $DB->get_records('files', array('contenthash' => $file->contenthash), '', '*', 0, 1)) { // Only grab one of the foundfiles - the file content should be the same for all entries. $foundfile = reset($foundfiles); $fs->create_file_from_storedfile($file_record, $foundfile->id); } else { - // A matching existing file record was not found in the database. - $results[] = self::get_missing_file_result($file); - continue; + // Finally try to restore the file from trash. + $filesytem = $fs->get_file_system(); + $restorefile = $file; + $restorefile->contextid = $newcontextid; + $storedfile = new stored_file($fs, $restorefile); + $trashrecovery = $filesytem->recover_file($storedfile, true); + if (!$trashrecovery) { + // A matching file was not found. + $results[] = self::get_missing_file_result($file); + continue; + } } } } diff --git a/backup/util/helper/backup_cron_helper.class.php b/backup/util/helper/backup_cron_helper.class.php index 980baf4457c..68ff8222b6b 100644 --- a/backup/util/helper/backup_cron_helper.class.php +++ b/backup/util/helper/backup_cron_helper.class.php @@ -415,8 +415,9 @@ abstract class backup_cron_automated_helper { $id = $bc->get_id(); $users = $bc->get_plan()->get_setting('users')->get_value(); $anonymised = $bc->get_plan()->get_setting('anonymize')->get_value(); + $incfiles = (bool)$config->backup_auto_files; $bc->get_plan()->get_setting('filename')->set_value(backup_plan_dbops::get_default_backup_filename($format, $type, - $id, $users, $anonymised)); + $id, $users, $anonymised, false, $incfiles)); $bc->set_status(backup::STATUS_AWAITING); diff --git a/backup/util/helper/backup_helper.class.php b/backup/util/helper/backup_helper.class.php index fa3bc79cf7c..167eae7bbae 100644 --- a/backup/util/helper/backup_helper.class.php +++ b/backup/util/helper/backup_helper.class.php @@ -287,7 +287,15 @@ abstract class backup_helper { $config = get_config('backup'); $dir = $config->backup_auto_destination; if ($config->backup_auto_storage == 1 and $dir and is_dir($dir) and is_writable($dir)) { - $filedest = $dir.'/'.backup_plan_dbops::get_default_backup_filename($format, $backuptype, $courseid, $hasusers, $isannon, !$config->backup_shortname); + $filedest = $dir.'/' + .backup_plan_dbops::get_default_backup_filename( + $format, + $backuptype, + $courseid, + $hasusers, + $isannon, + !$config->backup_shortname, + (bool)$config->backup_auto_files); // first try to move the file, if it is not possible copy and delete instead if (@rename($filepath, $filedest)) { return null; diff --git a/backup/util/ui/backup_ui_stage.class.php b/backup/util/ui/backup_ui_stage.class.php index 1e1caf8833f..aff941b9542 100644 --- a/backup/util/ui/backup_ui_stage.class.php +++ b/backup/util/ui/backup_ui_stage.class.php @@ -155,7 +155,9 @@ class backup_ui_stage_initial extends backup_ui_stage { $this->ui->get_type(), $this->ui->get_controller_id(), $this->ui->get_setting_value('users'), - $this->ui->get_setting_value('anonymize') + $this->ui->get_setting_value('anonymize'), + false, + (bool)$this->ui->get_setting_value('files') ); $setting->set_value($filename); } @@ -457,7 +459,16 @@ class backup_ui_stage_confirmation extends backup_ui_stage { $id = $this->ui->get_controller_id(); $users = $this->ui->get_setting_value('users'); $anonymised = $this->ui->get_setting_value('anonymize'); - $setting->set_value(backup_plan_dbops::get_default_backup_filename($format, $type, $id, $users, $anonymised)); + $files = (bool)$this->ui->get_setting_value('files'); + $filename = backup_plan_dbops::get_default_backup_filename( + $format, + $type, + $id, + $users, + $anonymised, + false, + $files); + $setting->set_value($filename); } $form->add_setting($setting, $task); break; @@ -628,6 +639,7 @@ class backup_ui_stage_complete extends backup_ui_stage_final { if (!empty($this->results['missing_files_in_pool'])) { $output .= $renderer->notification(get_string('missingfilesinpool', 'backup'), 'notifyproblem'); } + $output .= $renderer->get_samesite_notification(); $output .= $renderer->notification(get_string('executionsuccess', 'backup'), 'notifysuccess'); $output .= $renderer->continue_button($restorerul); $output .= $renderer->box_end(); diff --git a/backup/util/ui/renderer.php b/backup/util/ui/renderer.php index 4ee83e966f5..a46499f5400 100644 --- a/backup/util/ui/renderer.php +++ b/backup/util/ui/renderer.php @@ -43,6 +43,13 @@ require_once($CFG->dirroot . '/backup/moodle2/backup_plan_builder.class.php'); */ class core_backup_renderer extends plugin_renderer_base { + /** + * Same site notification display. + * + * @var string + */ + private $samesitenotification = ''; + /** * Renderers a progress bar for the backup or restore given the items that make it up. * @@ -80,6 +87,22 @@ class core_backup_renderer extends plugin_renderer_base { return $out; } + /** + * Set the same site backup notification. + * + */ + public function set_samesite_notification() { + $this->samesitenotification = $this->output->notification(get_string('samesitenotification', 'backup'), 'info'); + } + + /** + * Get the same site backup notification. + * + */ + public function get_samesite_notification() { + return $this->samesitenotification; + } + /** * Prints a dependency notification * diff --git a/lang/en/admin.php b/lang/en/admin.php index 1a0a8882904..0f8ba1703ba 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -564,6 +564,8 @@ $string['experimentalsettings'] = 'Experimental settings'; $string['extendedusernamechars'] = 'Allow extended characters in usernames'; $string['extramemorylimit'] = 'Extra PHP memory limit'; $string['fatalsessionautostart'] = '

Serious configuration error detected, please notify server administrator.

To operate properly, Moodle requires that administrator changes PHP settings.

session.auto_start must be set to off.

This setting is controlled by editing php.ini, Apache/IIS
configuration or .htaccess file on the server.

'; +$string['filescleanupperiod'] = 'Clean trash pool files'; +$string['filescleanupperiod_help'] = 'How often trash files are removed. These are files that are associated with a context that no longer exists'; $string['fileconversioncleanuptask'] = 'Cleanup of temporary records for file conversions.'; $string['filecreated'] = 'New file created'; $string['filestoredin'] = 'Save file into folder :'; diff --git a/lang/en/backup.php b/lang/en/backup.php index de681c62f9a..9cfe05e1b17 100644 --- a/lang/en/backup.php +++ b/lang/en/backup.php @@ -127,6 +127,7 @@ $string['configgeneralblocks'] = 'Sets the default for including blocks in a bac $string['configgeneralcalendarevents'] = 'Sets the default for including calendar events in a backup.'; $string['configgeneralcomments'] = 'Sets the default for including comments in a backup.'; $string['configgeneralcompetencies'] = 'Sets the default for including competencies in a backup.'; +$string['configgeneralfiles'] = 'Sets the default for including files in a backup.'; $string['configgeneralfilters'] = 'Sets the default for including filters in a backup.'; $string['configgeneralhistories'] = 'Sets the default for including user history within a backup.'; $string['configgenerallogs'] = 'If enabled logs will be included in backups by default.'; @@ -202,6 +203,7 @@ $string['generalcalendarevents'] = 'Include calendar events'; $string['generalcomments'] = 'Include comments'; $string['generalcompetencies'] = 'Include competencies'; $string['generalenrolments'] = 'Include enrolment methods'; +$string['generalfiles'] = 'Include files'; $string['generalfilters'] = 'Include filters'; $string['generalhistories'] = 'Include histories'; $string['generalgradehistories'] = 'Include histories'; @@ -328,6 +330,7 @@ $string['rootsettingbadges'] = 'Include badges'; $string['rootsettingblocks'] = 'Include blocks'; $string['rootsettingcompetencies'] = 'Include competencies'; $string['rootsettingfilters'] = 'Include filters'; +$string['rootsettingfiles'] = 'Include files'; $string['rootsettingcomments'] = 'Include comments'; $string['rootsettingcalendarevents'] = 'Include calendar events'; $string['rootsettinguserscompletion'] = 'Include user completion details'; @@ -337,6 +340,7 @@ $string['rootsettinggradehistories'] = 'Include grade history'; $string['rootsettinggroups'] = 'Include groups and groupings'; $string['rootsettingimscc1'] = 'Convert to IMS Common Cartridge 1.0'; $string['rootsettingimscc11'] = 'Convert to IMS Common Cartridge 1.1'; +$string['samesitenotification'] = 'This backup was created with only references to files, not the files themselves. Restoring will only work on this site.'; $string['sitecourseformatwarning'] = 'This is a front page backup, note that they can only be restored on the front page'; $string['storagecourseonly'] = 'Course backup filearea'; $string['storagecourseandexternal'] = 'Course backup filearea and the specified directory'; diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index b297159ab64..743beb98b39 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -2241,7 +2241,8 @@ class file_storage { // remove trash pool files once a day // if you want to disable purging of trash put $CFG->fileslastcleanup=time(); into config.php - if (empty($CFG->fileslastcleanup) or $CFG->fileslastcleanup < time() - 60*60*24) { + $filescleanupperiod = empty($CFG->filescleanupperiod) ? 86400 : $CFG->filescleanupperiod; + if (empty($CFG->fileslastcleanup) || ($CFG->fileslastcleanup < time() - $filescleanupperiod)) { require_once($CFG->libdir.'/filelib.php'); // Delete files that are associated with a context that no longer exists. mtrace('Cleaning up files from deleted contexts... ', ''); diff --git a/lib/filestorage/file_system.php b/lib/filestorage/file_system.php index ba6b8be8f6b..8dcc755489d 100644 --- a/lib/filestorage/file_system.php +++ b/lib/filestorage/file_system.php @@ -236,6 +236,15 @@ abstract class file_system { */ abstract public function remove_file($contenthash); + /** + * Tries to recover missing content of file from trash. + * + * @param stored_file $file stored_file instance + * @param bool $createrecord Create file record for stored file. + * @return bool success + */ + abstract public function recover_file(stored_file $file, $createrecord=false); + /** * Check whether a file is removable. * diff --git a/lib/filestorage/file_system_filedir.php b/lib/filestorage/file_system_filedir.php index f3840ca9ca0..4229f7dd8aa 100644 --- a/lib/filestorage/file_system_filedir.php +++ b/lib/filestorage/file_system_filedir.php @@ -234,13 +234,46 @@ class file_system_filedir extends file_system { return copy($source, $target); } + /** + * Create a file record from a stored file object. + * + * This is required in cases where we are recoverying a file + * from the trash and we need to also recrete the file record + * in the database. + * + * @param stored_file $file + * @return stdClass + */ + protected function create_recovery_record(stored_file $file) { + $filerecord = new stdClass(); + + $filerecord->contextid = $file->get_contextid(); + $filerecord->component = $file->get_component(); + $filerecord->filearea = $file->get_filearea(); + $filerecord->itemid = $file->get_itemid(); + $filerecord->filepath = $file->get_filepath(); + $filerecord->filename = $file->get_filename(); + $filerecord->timecreated = $file->get_timecreated(); + $filerecord->timemodified = $file->get_timemodified(); + $filerecord->userid = empty($file->get_userid()) ? null : $file->get_userid(); + $filerecord->source = empty($file->get_source()) ? null : $file->get_source(); + $filerecord->author = empty($file->get_author()) ? null : $file->get_author(); + $filerecord->license = empty($file->get_license()) ? null : $file->get_license(); + $filerecord->status = empty($file->get_status()) ? 0 : $file->get_status(); + $filerecord->sortorder = $file->get_sortorder(); + $filerecord->contenthash = $file->get_contenthash(); + + return $filerecord; + } + /** * Tries to recover missing content of file from trash. * * @param stored_file $file stored_file instance + * @param bool $createrecord Create file record for stored file. * @return bool success */ - protected function recover_file(stored_file $file) { + public function recover_file(stored_file $file, $createrecord=false) { $contentfile = $this->get_local_path_from_storedfile($file, false); if (file_exists($contentfile)) { @@ -274,9 +307,25 @@ class file_system_filedir extends file_system { } } - // Perform a rename - these are generally atomic which gives us big - // performance wins, especially for large files. - return rename($trashfile, $contentfile); + // Restore file from trash and create file record in database if needed. + if ($createrecord) { + $recoveryrecord = $this->create_recovery_record($file); + + $fs = new file_storage(); + $fs->create_file_from_pathname($recoveryrecord, $trashfile); + + // Remove copy of file still in trash. + // There are no references to this file anywhere so we just unlink it. + unlink($trashfile); + + } else { + // If record exists in database then perform a rename. + // These are generally atomic which gives us big + // performance wins, especially for large files. + return rename($trashfile, $contentfile); + } + + return true; } /** diff --git a/version.php b/version.php index d3b5604a427..6cc8698ea05 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2019072500.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2019072600.00; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes.