From d0734df40426c40eae9bde0cc41c08b43560962b Mon Sep 17 00:00:00 2001 From: John Yao Date: Tue, 6 Aug 2019 12:49:35 +1000 Subject: [PATCH] MDL-65729 Backup: Split individual backups into ad-hoc tasks * refactor run automated backup * new backup status of Queued * adhoc task for backing up a course * course locking for course backup adhoc task * use Moodle locking instead of custom locking for run automated backup task * add unit tests to extracted methods --- backup/tests/automated_backup_test.php | 300 +++++++++++ .../util/helper/backup_cron_helper.class.php | 501 +++++++++--------- lang/en/moodle.php | 2 + lib/classes/task/automated_backup_task.php | 1 - lib/classes/task/course_backup_task.php | 97 ++++ report/backups/index.php | 4 + theme/boost/scss/moodle/admin.scss | 3 +- theme/boost/style/moodle.css | 9 +- theme/classic/style/moodle.css | 9 +- 9 files changed, 675 insertions(+), 251 deletions(-) create mode 100644 backup/tests/automated_backup_test.php create mode 100644 lib/classes/task/course_backup_task.php diff --git a/backup/tests/automated_backup_test.php b/backup/tests/automated_backup_test.php new file mode 100644 index 00000000000..9ac514f97dc --- /dev/null +++ b/backup/tests/automated_backup_test.php @@ -0,0 +1,300 @@ +. + +/** + * Automated backup tests. + * + * @package core_backup + * @copyright 2019 John Yao + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/backup/util/helper/backup_cron_helper.class.php'); +require_once($CFG->libdir.'/cronlib.php'); +require_once($CFG->libdir . '/completionlib.php'); + +/** + * Automated backup tests. + * + * @copyright 2019 John Yao + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_backup_automated_backup_testcase extends advanced_testcase { + /** + * @var \backup_cron_automated_helper + */ + protected $backupcronautomatedhelper; + + /** + * @var stdClass $course + */ + protected $course; + + protected function setUp() { + global $DB, $CFG; + + $this->resetAfterTest(true); + $this->setAdminUser(); + $CFG->enableavailability = true; + $CFG->enablecompletion = true; + + // Getting a testable backup_cron_automated_helper class. + $this->backupcronautomatedhelper = new test_backup_cron_automated_helper(); + + $generator = $this->getDataGenerator(); + $this->course = $generator->create_course( + array('format' => 'topics', 'numsections' => 3, + 'enablecompletion' => COMPLETION_ENABLED), + array('createsections' => true)); + $forum = $generator->create_module('forum', array( + 'course' => $this->course->id)); + $forum2 = $generator->create_module('forum', array( + 'course' => $this->course->id, 'completion' => COMPLETION_TRACKING_MANUAL)); + + // We need a grade, easiest is to add an assignment. + $assignrow = $generator->create_module('assign', array( + 'course' => $this->course->id)); + $assign = new assign(context_module::instance($assignrow->cmid), false, false); + $item = $assign->get_grade_item(); + + // Make a test grouping as well. + $grouping = $generator->create_grouping(array('courseid' => $this->course->id, + 'name' => 'Grouping!')); + + $availability = '{"op":"|","show":false,"c":[' . + '{"type":"completion","cm":' . $forum2->cmid .',"e":1},' . + '{"type":"grade","id":' . $item->id . ',"min":4,"max":94},' . + '{"type":"grouping","id":' . $grouping->id . '}' . + ']}'; + $DB->set_field('course_modules', 'availability', $availability, array( + 'id' => $forum->cmid)); + $DB->set_field('course_sections', 'availability', $availability, array( + 'course' => $this->course->id, 'section' => 1)); + } + + /** + * Tests the automated backup run when the there is course backup should be skipped. + */ + public function test_automated_backup_skipped_run() { + global $DB; + + // Enable automated back up. + set_config('backup_auto_active', true, 'backup'); + set_config('backup_auto_weekdays', '1111111', 'backup'); + + // Start backup process. + $admin = get_admin(); + + // Backup entry should not exist. + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $this->course->id)); + $this->assertFalse($backupcourse); + $this->assertInstanceOf( + backup_cron_automated_helper::class, + $this->backupcronautomatedhelper->return_this() + ); + + $classobject = $this->backupcronautomatedhelper->return_this(); + + $method = new ReflectionMethod('\backup_cron_automated_helper', 'get_courses'); + $method->setAccessible(true); // Allow accessing of private method. + $courses = $method->invoke($classobject); + + $method = new ReflectionMethod('\backup_cron_automated_helper', 'check_and_push_automated_backups'); + $method->setAccessible(true); // Allow accessing of private method. + $emailpending = $method->invokeArgs($classobject, [$courses, $admin]); + + $coursename = $this->course->fullname; + $this->expectOutputRegex("/Skipping $coursename \(Not scheduled for backup until/"); + $this->assertFalse($emailpending); + + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $this->course->id)); + $this->assertNotNull($backupcourse->laststatus); + } + + /** + * Tests the automated backup run when the there is course backup can be pushed to adhoc task. + */ + public function test_automated_backup_push_run() { + global $DB; + + // Enable automated back up. + set_config('backup_auto_active', true, 'backup'); + set_config('backup_auto_weekdays', '1111111', 'backup'); + + $admin = get_admin(); + + $classobject = $this->backupcronautomatedhelper->return_this(); + + $method = new ReflectionMethod('\backup_cron_automated_helper', 'get_courses'); + $method->setAccessible(true); // Allow accessing of private method. + $courses = $method->invoke($classobject); + + // Create this backup course. + $backupcourse = new stdClass; + $backupcourse->courseid = $this->course->id; + $backupcourse->laststatus = backup_cron_automated_helper::BACKUP_STATUS_NOTYETRUN; + $DB->insert_record('backup_courses', $backupcourse); + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $this->course->id)); + + // We now manually trigger a backup pushed to adhoc task. + // Make sure is in the past, which means should run now. + $backupcourse->nextstarttime = time() - 10; + $DB->update_record('backup_courses', $backupcourse); + + $method = new ReflectionMethod('\backup_cron_automated_helper', 'check_and_push_automated_backups'); + $method->setAccessible(true); // Allow accessing of private method. + $emailpending = $method->invokeArgs($classobject, [$courses, $admin]); + $this->assertTrue($emailpending); + + $coursename = $this->course->fullname; + $this->expectOutputRegex("/Putting backup of $coursename in adhoc task queue/"); + + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $this->course->id)); + // Now this backup course status should be queued. + $this->assertEquals(backup_cron_automated_helper::BACKUP_STATUS_QUEUED, $backupcourse->laststatus); + } + + /** + * Tests the automated backup inactive run. + */ + public function test_inactive_run() { + backup_cron_automated_helper::run_automated_backup(); + $this->expectOutputString("Checking automated backup status...INACTIVE\n"); + } + + /** + * Tests the invisible course being skipped. + */ + public function test_should_skip_invisible_course() { + global $DB; + + set_config('backup_auto_active', true, 'backup'); + set_config('backup_auto_skip_hidden', true, 'backup'); + set_config('backup_auto_weekdays', '1111111', 'backup'); + // Create this backup course. + $backupcourse = new stdClass; + $backupcourse->courseid = $this->course->id; + // This is the status we believe last run was OK. + $backupcourse->laststatus = backup_cron_automated_helper::BACKUP_STATUS_SKIPPED; + $DB->insert_record('backup_courses', $backupcourse); + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $this->course->id)); + + $this->assertTrue(course_change_visibility($this->course->id, false)); + $course = $DB->get_record('course', array('id' => $this->course->id)); + $this->assertEquals('0', $course->visible); + $classobject = $this->backupcronautomatedhelper->return_this(); + $nextstarttime = backup_cron_automated_helper::calculate_next_automated_backup(null, time()); + + $method = new ReflectionMethod('\backup_cron_automated_helper', 'should_skip_course_backup'); + $method->setAccessible(true); // Allow accessing of private method. + $skipped = $method->invokeArgs($classobject, [$backupcourse, $course, $nextstarttime]); + + $this->assertTrue($skipped); + $this->expectOutputRegex("/Skipping $course->fullname \(Not visible\)/"); + } + + /** + * Tests the not modified course being skipped. + */ + public function test_should_skip_not_modified_course_in_days() { + global $DB; + + set_config('backup_auto_active', true, 'backup'); + // Skip if not modified in two days. + set_config('backup_auto_skip_modif_days', 2, 'backup'); + set_config('backup_auto_weekdays', '1111111', 'backup'); + + // Create this backup course. + $backupcourse = new stdClass; + $backupcourse->courseid = $this->course->id; + // This is the status we believe last run was OK. + $backupcourse->laststatus = backup_cron_automated_helper::BACKUP_STATUS_SKIPPED; + $backupcourse->laststarttime = time() - 2 * DAYSECS; + $backupcourse->lastendtime = time() - 1 * DAYSECS; + $DB->insert_record('backup_courses', $backupcourse); + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $this->course->id)); + $course = $DB->get_record('course', array('id' => $this->course->id)); + + $course->timemodified = time() - 2 * DAYSECS - 1; + + $classobject = $this->backupcronautomatedhelper->return_this(); + $nextstarttime = backup_cron_automated_helper::calculate_next_automated_backup(null, time()); + + $method = new ReflectionMethod('\backup_cron_automated_helper', 'should_skip_course_backup'); + $method->setAccessible(true); // Allow accessing of private method. + $skipped = $method->invokeArgs($classobject, [$backupcourse, $course, $nextstarttime]); + + $this->assertTrue($skipped); + $this->expectOutputRegex("/Skipping $course->fullname \(Not modified in the past 2 days\)/"); + } + + /** + * Tests the backup not modified course being skipped. + */ + public function test_should_skip_not_modified_course_since_prev() { + global $DB; + + set_config('backup_auto_active', true, 'backup'); + // Skip if not modified in two days. + set_config('backup_auto_skip_modif_prev', 2, 'backup'); + set_config('backup_auto_weekdays', '1111111', 'backup'); + + // Create this backup course. + $backupcourse = new stdClass; + $backupcourse->courseid = $this->course->id; + // This is the status we believe last run was OK. + $backupcourse->laststatus = backup_cron_automated_helper::BACKUP_STATUS_SKIPPED; + $backupcourse->laststarttime = time() - 2 * DAYSECS; + $backupcourse->lastendtime = time() - 1 * DAYSECS; + $DB->insert_record('backup_courses', $backupcourse); + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $this->course->id)); + $course = $DB->get_record('course', array('id' => $this->course->id)); + + $course->timemodified = time() - 2 * DAYSECS - 1; + + $classobject = $this->backupcronautomatedhelper->return_this(); + $nextstarttime = backup_cron_automated_helper::calculate_next_automated_backup(null, time()); + + $method = new ReflectionMethod('\backup_cron_automated_helper', 'should_skip_course_backup'); + $method->setAccessible(true); // Allow accessing of private method. + $skipped = $method->invokeArgs($classobject, [$backupcourse, $course, $nextstarttime]); + + $this->assertTrue($skipped); + $this->expectOutputRegex("/Skipping $course->fullname \(Not modified since previous backup\)/"); + } +} + +/** + * New backup_cron_automated_helper class for testing. + * + * This class extends the helper backup_cron_automated_helper class + * in order to utilise abstract class for testing. + * + * @package core + * @copyright 2019 John Yao + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class test_backup_cron_automated_helper extends backup_cron_automated_helper { + /** + * Returning this for testing. + */ + public function return_this() { + return $this; + } +} diff --git a/backup/util/helper/backup_cron_helper.class.php b/backup/util/helper/backup_cron_helper.class.php index 68ff8222b6b..70a2d8a547e 100644 --- a/backup/util/helper/backup_cron_helper.class.php +++ b/backup/util/helper/backup_cron_helper.class.php @@ -1,5 +1,4 @@ 0) { - $showtime = date('r', $nextstarttime); + mtrace("Getting admin info"); + $admin = get_admin(); + if (!$admin) { + mtrace("Error: No admin account was found"); + return; } $rs = self::get_courses($now); // Get courses to backup. - foreach ($rs as $course) { - $backupcourse = $DB->get_record('backup_courses', array('courseid' => $course->id)); - if (!$backupcourse) { - $backupcourse = new stdClass; - $backupcourse->courseid = $course->id; - $backupcourse->laststatus = self::BACKUP_STATUS_NOTYETRUN; - $DB->insert_record('backup_courses', $backupcourse); - $backupcourse = $DB->get_record('backup_courses', array('courseid' => $course->id)); - } - - // The last backup is considered as successful when OK or SKIPPED. - $lastbackupwassuccessful = ($backupcourse->laststatus == self::BACKUP_STATUS_SKIPPED || - $backupcourse->laststatus == self::BACKUP_STATUS_OK) && ( - $backupcourse->laststarttime > 0 && $backupcourse->lastendtime > 0); - - // Assume that we are not skipping anything. - $skipped = false; - $skippedmessage = ''; - - // Check if we are going to be running the backup now. - $shouldrunnow = (($backupcourse->nextstarttime > 0 && $backupcourse->nextstarttime < $now) - || $rundirective == self::RUN_IMMEDIATELY); - - // If config backup_auto_skip_hidden is set to true, skip courses that are not visible. - if ($shouldrunnow && $config->backup_auto_skip_hidden) { - $skipped = ($config->backup_auto_skip_hidden && !$course->visible); - $skippedmessage = 'Not visible'; - } - - // If config backup_auto_skip_modif_days is set to true, skip courses - // that have not been modified since the number of days defined. - if ($shouldrunnow && !$skipped && $lastbackupwassuccessful && $config->backup_auto_skip_modif_days) { - $timenotmodifsincedays = $now - ($config->backup_auto_skip_modif_days * DAYSECS); - // Check log if there were any modifications to the course content. - $logexists = self::is_course_modified($course->id, $timenotmodifsincedays); - $skipped = ($course->timemodified <= $timenotmodifsincedays && !$logexists); - $skippedmessage = 'Not modified in the past '.$config->backup_auto_skip_modif_days.' days'; - } - - // If config backup_auto_skip_modif_prev is set to true, skip courses - // that have not been modified since previous backup. - if ($shouldrunnow && !$skipped && $lastbackupwassuccessful && $config->backup_auto_skip_modif_prev) { - // Check log if there were any modifications to the course content. - $logexists = self::is_course_modified($course->id, $backupcourse->laststarttime); - $skipped = ($course->timemodified <= $backupcourse->laststarttime && !$logexists); - $skippedmessage = 'Not modified since previous backup'; - } - - // Check if the course is not scheduled to run right now. - if (!$shouldrunnow) { - $backupcourse->nextstarttime = $nextstarttime; - $DB->update_record('backup_courses', $backupcourse); - mtrace('Skipping ' . $course->fullname . ' (Not scheduled for backup until ' . $showtime . ')'); - } else { - if ($skipped) { // Must have been skipped for a reason. - $backupcourse->laststatus = self::BACKUP_STATUS_SKIPPED; - $backupcourse->nextstarttime = $nextstarttime; - $DB->update_record('backup_courses', $backupcourse); - mtrace('Skipping ' . $course->fullname . ' (' . $skippedmessage . ')'); - mtrace('Backup of \'' . $course->fullname . '\' is scheduled on ' . $showtime); - } else { - // Backup every non-skipped courses. - mtrace('Backing up '.$course->fullname.'...'); - - // We have to send an email because we have included at least one backup. - $emailpending = true; - - // Only make the backup if laststatus isn't 2-UNFINISHED (uncontrolled error). - if ($backupcourse->laststatus != self::BACKUP_STATUS_UNFINISHED) { - // Set laststarttime. - $starttime = time(); - - $backupcourse->laststarttime = time(); - $backupcourse->laststatus = self::BACKUP_STATUS_UNFINISHED; - $DB->update_record('backup_courses', $backupcourse); - - $backupcourse->laststatus = self::launch_automated_backup($course, $backupcourse->laststarttime, - $admin->id); - $backupcourse->lastendtime = time(); - $backupcourse->nextstarttime = $nextstarttime; - - $DB->update_record('backup_courses', $backupcourse); - - mtrace("complete - next execution: $showtime"); - } - } - - // Remove excess backups. - $removedcount = self::remove_excess_backups($course, $now); - } - } + $emailpending = self::check_and_push_automated_backups($rs, $admin); $rs->close(); - } - //Send email to admin if necessary - if ($emailpending) { - mtrace("Sending email to admin"); - $message = ""; - - $count = backup_cron_automated_helper::get_backup_status_array(); - $haserrors = ($count[self::BACKUP_STATUS_ERROR] != 0 || $count[self::BACKUP_STATUS_UNFINISHED] != 0); - - // Build the message text. - // Summary. - $message .= get_string('summary') . "\n"; - $message .= "==================================================\n"; - $message .= ' ' . get_string('courses') . ': ' . array_sum($count) . "\n"; - $message .= ' ' . get_string('ok') . ': ' . $count[self::BACKUP_STATUS_OK] . "\n"; - $message .= ' ' . get_string('skipped') . ': ' . $count[self::BACKUP_STATUS_SKIPPED] . "\n"; - $message .= ' ' . get_string('error') . ': ' . $count[self::BACKUP_STATUS_ERROR] . "\n"; - $message .= ' ' . get_string('unfinished') . ': ' . $count[self::BACKUP_STATUS_UNFINISHED] . "\n"; - $message .= ' ' . get_string('warning') . ': ' . $count[self::BACKUP_STATUS_WARNING] . "\n"; - $message .= ' ' . get_string('backupnotyetrun') . ': ' . $count[self::BACKUP_STATUS_NOTYETRUN]."\n\n"; - - //Reference - if ($haserrors) { - $message .= " ".get_string('backupfailed')."\n\n"; - $dest_url = "$CFG->wwwroot/report/backups/index.php"; - $message .= " ".get_string('backuptakealook','',$dest_url)."\n\n"; - //Set message priority - $admin->priority = 1; - //Reset unfinished to error - $DB->set_field('backup_courses','laststatus','0', array('laststatus'=>'2')); - } else { - $message .= " ".get_string('backupfinished')."\n"; + // Send email to admin if necessary. + if ($emailpending) { + self::send_backup_status_to_admin($admin); } - - //Build the message subject - $site = get_site(); - $prefix = format_string($site->shortname, true, array('context' => context_course::instance(SITEID))).": "; - if ($haserrors) { - $prefix .= "[".strtoupper(get_string('error'))."] "; - } - $subject = $prefix.get_string('automatedbackupstatus', 'backup'); - - //Send the message - $eventdata = new \core\message\message(); - $eventdata->courseid = SITEID; - $eventdata->modulename = 'moodle'; - $eventdata->userfrom = $admin; - $eventdata->userto = $admin; - $eventdata->subject = $subject; - $eventdata->fullmessage = $message; - $eventdata->fullmessageformat = FORMAT_PLAIN; - $eventdata->fullmessagehtml = ''; - $eventdata->smallmessage = ''; - - $eventdata->component = 'moodle'; - $eventdata->name = 'backup'; - - message_send($eventdata); + } finally { + // Everything is finished release lock. + $lock->release(); + mtrace('Automated backups complete.'); } - - //Everything is finished stop backup_auto_running - backup_cron_automated_helper::set_state_running(false); - - mtrace('Automated backups complete.'); - - return $status; } /** * Gets the results from the last automated backup that was run based upon * the statuses of the courses that were looked at. * - * @global moodle_database $DB * @return array */ public static function get_backup_status_array() { @@ -339,10 +160,14 @@ abstract class backup_cron_automated_helper { self::BACKUP_STATUS_UNFINISHED => 0, self::BACKUP_STATUS_SKIPPED => 0, self::BACKUP_STATUS_WARNING => 0, - self::BACKUP_STATUS_NOTYETRUN => 0 + self::BACKUP_STATUS_NOTYETRUN => 0, + self::BACKUP_STATUS_QUEUED => 0, ); - $statuses = $DB->get_records_sql('SELECT DISTINCT bc.laststatus, COUNT(bc.courseid) AS statuscount FROM {backup_courses} bc GROUP BY bc.laststatus'); + $statuses = $DB->get_records_sql('SELECT DISTINCT bc.laststatus, + COUNT(bc.courseid) AS statuscount + FROM {backup_courses} bc + GROUP BY bc.laststatus'); foreach ($statuses as $status) { if (empty($status->statuscount)) { @@ -354,6 +179,219 @@ abstract class backup_cron_automated_helper { return $result; } + /** + * Collect details for all statuses of the courses + * and send report to admin. + * + * @param stdClass $admin + * @return array + */ + private static function send_backup_status_to_admin($admin) { + global $DB, $CFG; + + mtrace("Sending email to admin"); + $message = ""; + + $count = self::get_backup_status_array(); + $haserrors = ($count[self::BACKUP_STATUS_ERROR] != 0 || $count[self::BACKUP_STATUS_UNFINISHED] != 0); + + // Build the message text. + // Summary. + $message .= get_string('summary') . "\n"; + $message .= "==================================================\n"; + $message .= ' ' . get_string('courses') . ': ' . array_sum($count) . "\n"; + $message .= ' ' . get_string('ok') . ': ' . $count[self::BACKUP_STATUS_OK] . "\n"; + $message .= ' ' . get_string('skipped') . ': ' . $count[self::BACKUP_STATUS_SKIPPED] . "\n"; + $message .= ' ' . get_string('error') . ': ' . $count[self::BACKUP_STATUS_ERROR] . "\n"; + $message .= ' ' . get_string('unfinished') . ': ' . $count[self::BACKUP_STATUS_UNFINISHED] . "\n"; + $message .= ' ' . get_string('backupadhocpending') . ': ' . $count[self::BACKUP_STATUS_QUEUED] . "\n"; + $message .= ' ' . get_string('warning') . ': ' . $count[self::BACKUP_STATUS_WARNING] . "\n"; + $message .= ' ' . get_string('backupnotyetrun') . ': ' . $count[self::BACKUP_STATUS_NOTYETRUN]."\n\n"; + + // Reference. + if ($haserrors) { + $message .= " ".get_string('backupfailed')."\n\n"; + $desturl = "$CFG->wwwroot/report/backups/index.php"; + $message .= " ".get_string('backuptakealook', '', $desturl)."\n\n"; + // Set message priority. + $admin->priority = 1; + // Reset error and unfinished statuses to ok if longer than 24 hours. + $sql = "laststatus IN (:statuserror,:statusunfinished) AND laststarttime < :yesterday"; + $params = [ + 'statuserror' => self::BACKUP_STATUS_ERROR, + 'statusunfinished' => self::BACKUP_STATUS_UNFINISHED, + 'yesterday' => time() - 86400, + ]; + $DB->set_field_select('backup_courses', 'laststatus', self::BACKUP_STATUS_OK, $sql, $params); + } else { + $message .= " ".get_string('backupfinished')."\n"; + } + + // Build the message subject. + $site = get_site(); + $prefix = format_string($site->shortname, true, array('context' => context_course::instance(SITEID))).": "; + if ($haserrors) { + $prefix .= "[".strtoupper(get_string('error'))."] "; + } + $subject = $prefix.get_string('automatedbackupstatus', 'backup'); + + // Send the message. + $eventdata = new \core\message\message(); + $eventdata->courseid = SITEID; + $eventdata->modulename = 'moodle'; + $eventdata->userfrom = $admin; + $eventdata->userto = $admin; + $eventdata->subject = $subject; + $eventdata->fullmessage = $message; + $eventdata->fullmessageformat = FORMAT_PLAIN; + $eventdata->fullmessagehtml = ''; + $eventdata->smallmessage = ''; + + $eventdata->component = 'moodle'; + $eventdata->name = 'backup'; + + return message_send($eventdata); + } + + /** + * Loop through courses and push to course ad-hoc task if required + * + * @param \record_set $courses + * @param stdClass $admin + * @return boolean + */ + private static function check_and_push_automated_backups($courses, $admin) { + global $DB; + + $now = time(); + $emailpending = false; + + $nextstarttime = self::calculate_next_automated_backup(null, $now); + $showtime = "undefined"; + if ($nextstarttime > 0) { + $showtime = date('r', $nextstarttime); + } + + foreach ($courses as $course) { + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $course->id)); + if (!$backupcourse) { + $backupcourse = new stdClass; + $backupcourse->courseid = $course->id; + $backupcourse->laststatus = self::BACKUP_STATUS_NOTYETRUN; + $DB->insert_record('backup_courses', $backupcourse); + $backupcourse = $DB->get_record('backup_courses', array('courseid' => $course->id)); + } + + // Check if we are going to be running the backup now. + $shouldrunnow = ($backupcourse->nextstarttime > 0 && $backupcourse->nextstarttime < $now); + + // Check if the course is not scheduled to run right now, or it has been put in queue. + if (!$shouldrunnow || $backupcourse->laststatus == self::BACKUP_STATUS_QUEUED) { + $backupcourse->nextstarttime = $nextstarttime; + $DB->update_record('backup_courses', $backupcourse); + mtrace('Skipping ' . $course->fullname . ' (Not scheduled for backup until ' . $showtime . ')'); + } else { + $skipped = self::should_skip_course_backup($backupcourse, $course, $nextstarttime); + if (!$skipped) { // If it should not be skipped. + + // Only make the backup if laststatus isn't 2-UNFINISHED (uncontrolled error or being backed up). + if ($backupcourse->laststatus != self::BACKUP_STATUS_UNFINISHED) { + // Add every non-skipped courses to backup adhoc task queue. + mtrace('Putting backup of ' . $course->fullname . ' in adhoc task queue ...'); + + // We have to send an email because we have included at least one backup. + $emailpending = true; + // Create adhoc task for backup. + self::push_course_backup_adhoc_task($backupcourse, $admin); + mtrace("complete - next execution: $showtime"); + } + } + } + } + + return $emailpending; + } + + /** + * Check if we can skip this course backup. + * + * @param stdClass $backupcourse + * @param stdClass $course + * @param int $nextstarttime + * @return boolean + */ + private static function should_skip_course_backup($backupcourse, $course, $nextstarttime) { + global $DB; + + $config = get_config('backup'); + $now = time(); + // Assume that we are not skipping anything. + $skipped = false; + $skippedmessage = ''; + + // The last backup is considered as successful when OK or SKIPPED. + $lastbackupwassuccessful = ($backupcourse->laststatus == self::BACKUP_STATUS_SKIPPED || + $backupcourse->laststatus == self::BACKUP_STATUS_OK) && ( + $backupcourse->laststarttime > 0 && $backupcourse->lastendtime > 0); + + // If config backup_auto_skip_hidden is set to true, skip courses that are not visible. + if ($config->backup_auto_skip_hidden) { + $skipped = ($config->backup_auto_skip_hidden && !$course->visible); + $skippedmessage = 'Not visible'; + } + + // If config backup_auto_skip_modif_days is set to true, skip courses + // that have not been modified since the number of days defined. + if (!$skipped && $lastbackupwassuccessful && $config->backup_auto_skip_modif_days) { + $timenotmodifsincedays = $now - ($config->backup_auto_skip_modif_days * DAYSECS); + // Check log if there were any modifications to the course content. + $logexists = self::is_course_modified($course->id, $timenotmodifsincedays); + $skipped = ($course->timemodified <= $timenotmodifsincedays && !$logexists); + $skippedmessage = 'Not modified in the past '.$config->backup_auto_skip_modif_days.' days'; + } + + // If config backup_auto_skip_modif_prev is set to true, skip courses + // that have not been modified since previous backup. + if (!$skipped && $lastbackupwassuccessful && $config->backup_auto_skip_modif_prev) { + // Check log if there were any modifications to the course content. + $logexists = self::is_course_modified($course->id, $backupcourse->laststarttime); + $skipped = ($course->timemodified <= $backupcourse->laststarttime && !$logexists); + $skippedmessage = 'Not modified since previous backup'; + } + + if ($skipped) { // Must have been skipped for a reason. + $backupcourse->laststatus = self::BACKUP_STATUS_SKIPPED; + $backupcourse->nextstarttime = $nextstarttime; + $DB->update_record('backup_courses', $backupcourse); + mtrace('Skipping ' . $course->fullname . ' (' . $skippedmessage . ')'); + mtrace('Backup of \'' . $course->fullname . '\' is scheduled on ' . date('r', $nextstarttime)); + } + + return $skipped; + } + + /** + * Create course backup adhoc task + * + * @param stdClass $backupcourse + * @param stdClass $admin + * @return void + */ + private static function push_course_backup_adhoc_task($backupcourse, $admin) { + global $DB; + + $asynctask = new \core\task\course_backup_task(); + $asynctask->set_blocking(false); + $asynctask->set_custom_data(array( + 'courseid' => $backupcourse->courseid, + 'adminid' => $admin->id + )); + \core\task\manager::queue_adhoc_task($asynctask); + + $backupcourse->laststatus = self::BACKUP_STATUS_QUEUED; + $DB->update_record('backup_courses', $backupcourse); + } + /** * Works out the next time the automated backup should be run. * @@ -508,7 +546,6 @@ abstract class backup_cron_automated_helper { * Removes deleted courses fromn the backup_courses table so that we don't * waste time backing them up. * - * @global moodle_database $DB * @return int */ public static function remove_deleted_courses_from_schedule() { @@ -517,8 +554,8 @@ abstract class backup_cron_automated_helper { $sql = "SELECT bc.courseid FROM {backup_courses} bc WHERE bc.courseid NOT IN (SELECT c.id FROM {course} c)"; $rs = $DB->get_recordset_sql($sql); foreach ($rs as $deletedcourse) { - //Doesn't exist, so delete from backup tables - $DB->delete_records('backup_courses', array('courseid'=>$deletedcourse->courseid)); + // Doesn't exist, so delete from backup tables. + $DB->delete_records('backup_courses', array('courseid' => $deletedcourse->courseid)); $skipped++; } $rs->close(); @@ -526,57 +563,35 @@ abstract class backup_cron_automated_helper { } /** - * Gets the state of the automated backup system. + * Try to get lock for automated backup. + * @param int $rundirective * - * @global moodle_database $DB - * @return int One of self::STATE_* + * @return \core\lock\lock|boolean - An instance of \core\lock\lock if the lock was obtained, or false. */ - public static function get_automated_backup_state($rundirective = self::RUN_ON_SCHEDULE) { - global $DB; - + public static function get_automated_backup_lock($rundirective = self::RUN_ON_SCHEDULE) { $config = get_config('backup'); $active = (int)$config->backup_auto_active; $weekdays = (string)$config->backup_auto_weekdays; + mtrace("Checking automated backup status", '...'); + $locktype = 'automated_backup'; + $resource = 'queue_backup_jobs_running'; + $lockfactory = \core\lock\lock_config::get_lock_factory($locktype); + // In case of automated backup also check that it is scheduled for at least one weekday. if ($active === self::AUTO_BACKUP_DISABLED || ($rundirective == self::RUN_ON_SCHEDULE && $active === self::AUTO_BACKUP_MANUAL) || ($rundirective == self::RUN_ON_SCHEDULE && strpos($weekdays, '1') === false)) { - return self::STATE_DISABLED; - } else if (!empty($config->backup_auto_running)) { - // Detect if the backup_auto_running semaphore is a valid one - // by looking for recent activity in the backup_controllers table - // for backups of type backup::MODE_AUTOMATED - $timetosee = 60 * 90; // Time to consider in order to clean the semaphore - $params = array( 'purpose' => backup::MODE_AUTOMATED, 'timetolook' => (time() - $timetosee)); - if ($DB->record_exists_select('backup_controllers', - "operation = 'backup' AND type = 'course' AND purpose = :purpose AND timemodified > :timetolook", $params)) { - return self::STATE_RUNNING; // Recent activity found, still running - } else { - // No recent activity found, let's clean the semaphore - mtrace('Automated backups activity not found in last ' . (int)$timetosee/60 . ' minutes. Cleaning running status'); - backup_cron_automated_helper::set_state_running(false); - } + mtrace('INACTIVE'); + return false; } - return self::STATE_OK; - } - /** - * Sets the state of the automated backup system. - * - * @param bool $running - * @return bool - */ - public static function set_state_running($running = true) { - if ($running === true) { - if (self::get_automated_backup_state() === self::STATE_RUNNING) { - throw new backup_helper_exception('backup_automated_already_running'); - } - set_config('backup_auto_running', '1', 'backup'); - } else { - unset_config('backup_auto_running', 'backup'); + if (!$lock = $lockfactory->get_lock($resource, 10)) { + return false; } - return true; + + mtrace('OK'); + return $lock; } /** diff --git a/lang/en/moodle.php b/lang/en/moodle.php index 2b7061fe3e4..0b950d13382 100644 --- a/lang/en/moodle.php +++ b/lang/en/moodle.php @@ -174,6 +174,7 @@ $string['backtohome'] = 'Back to the site home'; $string['backtopageyouwereon'] = 'Back to the page you were on'; $string['backup'] = 'Backup'; $string['backupactivehelp'] = 'Choose whether or not to do automated backups.'; +$string['backupadhocpending'] = 'Course backup adhoc task pending'; $string['backupcancelled'] = 'Backup cancelled'; $string['backupcoursefileshelp'] = 'If enabled then course files will be included in automated backups'; $string['backupdate'] = 'Backup date'; @@ -1627,6 +1628,7 @@ $string['publicsitefileswarning2'] = 'Note: Files placed here can be accessed by $string['publicsitefileswarning3'] = 'Note: Files placed here can be accessed by anyone who knows (or can guess) the URL.
For security reasons, backup files should be saved in the secure backupdata folder only.'; $string['question'] = 'Question'; $string['questionsinthequestionbank'] = 'Questions in the question bank'; +$string['queued'] = 'Queued'; $string['quotausage'] = 'You have currently used {$a->used} of your {$a->total} limit.'; $string['readinginfofrombackup'] = 'Reading info from backup'; $string['readme'] = 'README'; diff --git a/lib/classes/task/automated_backup_task.php b/lib/classes/task/automated_backup_task.php index 6bbeba7a6e5..447e430f337 100644 --- a/lib/classes/task/automated_backup_task.php +++ b/lib/classes/task/automated_backup_task.php @@ -43,7 +43,6 @@ class automated_backup_task extends scheduled_task { */ public function execute() { global $CFG; - // Run automated backups if required - these may take a long time to execute. require_once($CFG->dirroot.'/backup/util/includes/backup_includes.php'); require_once($CFG->dirroot.'/backup/util/helper/backup_cron_helper.class.php'); diff --git a/lib/classes/task/course_backup_task.php b/lib/classes/task/course_backup_task.php new file mode 100644 index 00000000000..b2a51176555 --- /dev/null +++ b/lib/classes/task/course_backup_task.php @@ -0,0 +1,97 @@ +. + +/** + * Adhoc task that performs single automated course backup. + * + * @package core + * @copyright 2019 John Yao + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\task; +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); +require_once($CFG->dirroot . '/backup/util/helper/backup_cron_helper.class.php'); + +/** + * Adhoc task that performs single automated course backup. + * + * @package core + * @copyright 2019 John Yao + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class course_backup_task extends \core\task\adhoc_task { + + /** + * Run the adhoc task and preform the backup. + */ + public function execute() { + global $DB; + + $lockfactory = \core\lock\lock_config::get_lock_factory('course_backup_adhoc'); + $courseid = $this->get_custom_data()->courseid; + + try { + $course = $DB->get_record('course', array('id' => $courseid), '*', MUST_EXIST); + } catch (moodle_exception $e) { + mtrace('Invalid course id: ' . $courseid . ', task aborted.'); + return; + } + + if (!$lock = $lockfactory->get_lock('course_backup_adhoc_task_' . $courseid, 10)) { + mtrace('Backup adhoc task for: ' . $course->fullname . 'is already running.'); + return; + } else { + mtrace('Processing automated backup for course: ' . $course->fullname); + } + + try { + $backupcourse = $DB->get_record('backup_courses', array( + 'courseid' => $courseid, + 'laststatus' => \backup_cron_automated_helper::BACKUP_STATUS_QUEUED + ), '*', MUST_EXIST); + + $adminid = $this->get_custom_data()->adminid; + $backupcourse->laststarttime = time(); + $backupcourse->laststatus = \backup_cron_automated_helper::BACKUP_STATUS_UNFINISHED; + $DB->update_record('backup_courses', $backupcourse); + + $backupcourse->laststatus = \backup_cron_automated_helper::launch_automated_backup($course, time(), $adminid); + if ($backupcourse->laststatus == \backup_cron_automated_helper::BACKUP_STATUS_ERROR || + $backupcourse->laststatus == \backup_cron_automated_helper::BACKUP_STATUS_UNFINISHED) { + mtrace('Automated backup for course: ' . $course->fullname . ' failed.'); + // Reset unfinished to error. + $backupcourse->laststatus = \backup_cron_automated_helper::BACKUP_STATUS_ERROR; + } + + // Remove excess backups. + $removedcount = \backup_cron_automated_helper::remove_excess_backups($course, time()); + $backupcourse->lastendtime = time(); + $backupcourse->nextstarttime = \backup_cron_automated_helper::calculate_next_automated_backup(null, time()); + $DB->update_record('backup_courses', $backupcourse); + } catch (moodle_exception $e) { + mtrace('Automated backup for course: ' . $course->fullname . ' encounters an error.'); + mtrace('Exception: ' . $e->getMessage()); + mtrace('Debug: ' . $e->debuginfo); + } finally { + // Everything is finished release lock. + $lock->release(); + mtrace('Automated backup for course: ' . $course->fullname . ' completed.'); + } + } +} diff --git a/report/backups/index.php b/report/backups/index.php index 39dadc41fa4..91090a72d16 100644 --- a/report/backups/index.php +++ b/report/backups/index.php @@ -45,6 +45,7 @@ $strunfinished = get_string('unfinished'); $strskipped = get_string('skipped'); $strwarning = get_string('warning'); $strnotyetrun = get_string('backupnotyetrun'); +$strqueued = get_string('queued'); if ($courseid) { $course = $DB->get_record('course', array('id' => $courseid), 'id, fullname', MUST_EXIST); @@ -152,6 +153,9 @@ foreach ($rs as $backuprow) { } else if ($backuprow->laststatus == backup_cron_automated_helper::BACKUP_STATUS_NOTYETRUN) { $status = $strnotyetrun; $statusclass = 'backup-notyetrun'; + } else if ($backuprow->laststatus == backup_cron_automated_helper::BACKUP_STATUS_QUEUED) { + $status = $strqueued; + $statusclass = 'backup-queued'; } else { $status = $strerror; $statusclass = 'backup-error'; // Red. diff --git a/theme/boost/scss/moodle/admin.scss b/theme/boost/scss/moodle/admin.scss index 047d52e7673..323e9ff5b16 100644 --- a/theme/boost/scss/moodle/admin.scss +++ b/theme/boost/scss/moodle/admin.scss @@ -88,7 +88,8 @@ #page-admin-report-backups-index .backup-skipped, #page-admin-report-backups-index .backup-ok, -#page-admin-report-backups-index .backup-notyetrun { +#page-admin-report-backups-index .backup-notyetrun, +#page-admin-report-backups-index .backup-queued { @extend .tag-success; } diff --git a/theme/boost/style/moodle.css b/theme/boost/style/moodle.css index bf77d330fea..3fdc12ac439 100644 --- a/theme/boost/style/moodle.css +++ b/theme/boost/style/moodle.css @@ -9491,13 +9491,16 @@ a.dimmed_text:visited, .tag-success, .statusok, #page-admin-report-backups-index .backup-skipped, #page-admin-report-backups-index .backup-ok, -#page-admin-report-backups-index .backup-notyetrun { +#page-admin-report-backups-index .backup-notyetrun, +#page-admin-report-backups-index .backup-queued { background-color: #5cb85c; } .tag-success[href]:hover, .statusok[href]:hover, #page-admin-report-backups-index .backup-skipped[href]:hover, #page-admin-report-backups-index .backup-ok[href]:hover, - #page-admin-report-backups-index .backup-notyetrun[href]:hover, .tag-success[href]:focus, .statusok[href]:focus, #page-admin-report-backups-index .backup-skipped[href]:focus, + #page-admin-report-backups-index .backup-notyetrun[href]:hover, + #page-admin-report-backups-index .backup-queued[href]:hover, .tag-success[href]:focus, .statusok[href]:focus, #page-admin-report-backups-index .backup-skipped[href]:focus, #page-admin-report-backups-index .backup-ok[href]:focus, - #page-admin-report-backups-index .backup-notyetrun[href]:focus { + #page-admin-report-backups-index .backup-notyetrun[href]:focus, + #page-admin-report-backups-index .backup-queued[href]:focus { background-color: #449d44; } .tag-info, #page-admin-index .adminwarning.availableupdatesinfo .moodleupdateinfo.maturity200 .info.release { diff --git a/theme/classic/style/moodle.css b/theme/classic/style/moodle.css index 3fa11e1c32a..608aad450a5 100644 --- a/theme/classic/style/moodle.css +++ b/theme/classic/style/moodle.css @@ -9739,13 +9739,16 @@ a.dimmed_text:visited, .tag-success, .statusok, #page-admin-report-backups-index .backup-skipped, #page-admin-report-backups-index .backup-ok, -#page-admin-report-backups-index .backup-notyetrun { +#page-admin-report-backups-index .backup-notyetrun, +#page-admin-report-backups-index .backup-queued { background-color: #5cb85c; } .tag-success[href]:hover, .statusok[href]:hover, #page-admin-report-backups-index .backup-skipped[href]:hover, #page-admin-report-backups-index .backup-ok[href]:hover, - #page-admin-report-backups-index .backup-notyetrun[href]:hover, .tag-success[href]:focus, .statusok[href]:focus, #page-admin-report-backups-index .backup-skipped[href]:focus, + #page-admin-report-backups-index .backup-notyetrun[href]:hover, + #page-admin-report-backups-index .backup-queued[href]:hover, .tag-success[href]:focus, .statusok[href]:focus, #page-admin-report-backups-index .backup-skipped[href]:focus, #page-admin-report-backups-index .backup-ok[href]:focus, - #page-admin-report-backups-index .backup-notyetrun[href]:focus { + #page-admin-report-backups-index .backup-notyetrun[href]:focus, + #page-admin-report-backups-index .backup-queued[href]:focus { background-color: #449d44; } .tag-info, #page-admin-index .adminwarning.availableupdatesinfo .moodleupdateinfo.maturity200 .info.release {