diff --git a/admin/settings/courses.php b/admin/settings/courses.php index f214ab44d69..99892d68dd9 100644 --- a/admin/settings/courses.php +++ b/admin/settings/courses.php @@ -458,4 +458,28 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) { $ADMIN->add('backups', $temp); + // Create a page for asynchronous backup and restore configuration and defaults. + if (!empty($CFG->enableasyncbackup)) { // Only add settings if async mode is enable at site level. + $temp = new admin_settingpage('asyncgeneralsettings', new lang_string('asyncgeneralsettings', 'backup')); + + $temp->add(new admin_setting_configcheckbox( + 'backup/backup_async_message_users', + new lang_string('asyncemailenable', 'backup'), + new lang_string('asyncemailenabledetail', 'backup'), 0)); + + $temp->add(new admin_setting_configtext( + 'backup/backup_async_message_subject', + new lang_string('asyncmessagesubject', 'backup'), + new lang_string('asyncmessagesubjectdetail', 'backup'), + new lang_string('asyncmessagesubjectdefault', 'backup'))); + + $temp->add(new admin_setting_confightmleditor( + 'backup/backup_async_message', + new lang_string('asyncmessagebody', 'backup'), + new lang_string('asyncmessagebodydetail', 'backup'), + new lang_string('asyncmessagebodydefault', 'backup'))); + + $ADMIN->add('backups', $temp); + } + } diff --git a/admin/settings/subsystems.php b/admin/settings/subsystems.php index e5c660b570a..0b627494d65 100644 --- a/admin/settings/subsystems.php +++ b/admin/settings/subsystems.php @@ -51,4 +51,7 @@ if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page $optionalsubsystems->add(new admin_setting_configcheckbox('enablecoursepublishing', new lang_string('enablecoursepublishing', 'hub'), new lang_string('enablecoursepublishing_help', 'hub'), 0)); + + $optionalsubsystems->add(new admin_setting_configcheckbox('enableasyncbackup', new lang_string('enableasyncbackup', 'backup'), + new lang_string('enableasyncbackup_help', 'backup'), 0, 1, 0)); } diff --git a/admin/tool/analytics/classes/output/form/edit_model.php b/admin/tool/analytics/classes/output/form/edit_model.php index 2c0be865102..31d0f578606 100644 --- a/admin/tool/analytics/classes/output/form/edit_model.php +++ b/admin/tool/analytics/classes/output/form/edit_model.php @@ -74,6 +74,7 @@ class edit_model extends \moodleform { ); $mform->addElement('autocomplete', 'indicators', get_string('indicators', 'tool_analytics'), $indicators, $options); $mform->setType('indicators', PARAM_ALPHANUMEXT); + $mform->addHelpButton('indicators', 'indicators', 'tool_analytics'); $timesplittings = array('' => ''); foreach ($this->_customdata['timesplittings'] as $classname => $timesplitting) { diff --git a/admin/tool/analytics/classes/output/helper.php b/admin/tool/analytics/classes/output/helper.php index 6543022315d..861d6794f83 100644 --- a/admin/tool/analytics/classes/output/helper.php +++ b/admin/tool/analytics/classes/output/helper.php @@ -92,4 +92,17 @@ class helper { $PAGE->set_title($title); $PAGE->set_heading($title); } + + /** + * Resets the current page. + * + * Note that this function can only be used by analytics pages that work at the system context. + * + * @return null + */ + public static function reset_page() { + global $PAGE; + $PAGE->reset_theme_and_output(); + $PAGE->set_context(\context_system::instance()); + } } diff --git a/admin/tool/analytics/classes/output/invalid_analysables.php b/admin/tool/analytics/classes/output/invalid_analysables.php index 0bb39025747..dbdaa00ef41 100644 --- a/admin/tool/analytics/classes/output/invalid_analysables.php +++ b/admin/tool/analytics/classes/output/invalid_analysables.php @@ -76,13 +76,17 @@ class invalid_analysables implements \renderable, \templatable { $offset = $this->page * $this->perpage; - $analysables = $this->model->get_analyser(['notimesplitting' => true])->get_analysables(); + $analysables = $this->model->get_analyser(['notimesplitting' => true])->get_analysables_iterator(); $skipped = 0; $enoughresults = false; $morepages = false; $results = array(); - foreach ($analysables as $key => $analysable) { + foreach ($analysables as $analysable) { + + if (!$analysable) { + continue; + } $validtraining = $this->model->get_target()->is_valid_analysable($analysable, true); if ($validtraining === true) { @@ -117,8 +121,6 @@ class invalid_analysables implements \renderable, \templatable { $morepages = true; break; } - - unset($analysables[$key]); } // Prepare the context object. diff --git a/admin/tool/analytics/classes/task/predict_models.php b/admin/tool/analytics/classes/task/predict_models.php index 83894beb68b..91f3c850688 100644 --- a/admin/tool/analytics/classes/task/predict_models.php +++ b/admin/tool/analytics/classes/task/predict_models.php @@ -60,6 +60,10 @@ class predict_models extends \core\task\scheduled_task { foreach ($models as $model) { $result = $model->predict(); + + // Reset the page as some indicators may call external functions that overwrite the page context. + \tool_analytics\output\helper::reset_page(); + if ($result) { echo $OUTPUT->heading(get_string('modelresults', 'tool_analytics', $model->get_target()->get_name())); $renderer = $PAGE->get_renderer('tool_analytics'); diff --git a/admin/tool/analytics/classes/task/train_models.php b/admin/tool/analytics/classes/task/train_models.php index b017e9a812f..3c0c3c96019 100644 --- a/admin/tool/analytics/classes/task/train_models.php +++ b/admin/tool/analytics/classes/task/train_models.php @@ -71,6 +71,10 @@ class train_models extends \core\task\scheduled_task { } $result = $model->train(); + + // Reset the page as some indicators may call external functions that overwrite the page context. + \tool_analytics\output\helper::reset_page(); + if ($result) { echo $OUTPUT->heading(get_string('modelresults', 'tool_analytics', $model->get_target()->get_name())); diff --git a/admin/tool/analytics/cli/evaluate_model.php b/admin/tool/analytics/cli/evaluate_model.php index 2836ca76a49..8473aba6ab0 100644 --- a/admin/tool/analytics/cli/evaluate_model.php +++ b/admin/tool/analytics/cli/evaluate_model.php @@ -111,6 +111,9 @@ $analyseroptions = array( // Evaluate its suitability to predict accurately. $results = $model->evaluate($analyseroptions); +// Reset the page as some indicators may call external functions that overwrite the page context. +\tool_analytics\output\helper::reset_page(); + $renderer = $PAGE->get_renderer('tool_analytics'); echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs()); diff --git a/admin/tool/analytics/lang/en/tool_analytics.php b/admin/tool/analytics/lang/en/tool_analytics.php index dc0c5c90b67..b3a8ba42cf1 100644 --- a/admin/tool/analytics/lang/en/tool_analytics.php +++ b/admin/tool/analytics/lang/en/tool_analytics.php @@ -78,6 +78,8 @@ $string['getpredictions'] = 'Get predictions'; $string['goodmodel'] = 'This is a good model for using to obtain predictions. Enable it to start obtaining predictions.'; $string['importmodel'] = 'Import model'; $string['indicators'] = 'Indicators'; +$string['indicators_help'] = 'The indicators are what you think will lead to an accurate prediction of the target.'; +$string['indicators_link'] = 'Indicators'; $string['info'] = 'Info'; $string['ignoreversionmismatches'] = 'Ignore version mismatches'; $string['ignoreversionmismatchescheckbox'] = 'Ignore the differences between this site version and the original site version.'; @@ -111,6 +113,7 @@ $string['samestartdate'] = 'Current start date is good'; $string['sameenddate'] = 'Current end date is good'; $string['target'] = 'Target'; $string['target_help'] = 'The target is what the model will predict.'; +$string['target_link'] = 'Targets'; $string['timesplittingnotdefined'] = 'Time splitting is not defined.'; $string['timesplittingnotdefined_help'] = 'You need to select a time-splitting method before enabling the model.'; $string['trainandpredictmodel'] = 'Training model and calculating predictions'; diff --git a/admin/tool/analytics/model.php b/admin/tool/analytics/model.php index 3cca64ebda4..ab2fe61796a 100644 --- a/admin/tool/analytics/model.php +++ b/admin/tool/analytics/model.php @@ -156,8 +156,6 @@ switch ($action) { case 'evaluate': confirm_sesskey(); - echo $OUTPUT->header(); - if ($model->is_static()) { throw new moodle_exception('errornostaticevaluate', 'tool_analytics'); } @@ -176,6 +174,12 @@ switch ($action) { $options['mode'] = 'trainedmodel'; } $results = $model->evaluate($options); + + // We reset the theme and the output as some indicators may be using external functions + // which reset $PAGE. + \tool_analytics\output\helper::reset_page(); + echo $OUTPUT->header(); + $renderer = $PAGE->get_renderer('tool_analytics'); echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs()); break; @@ -183,8 +187,6 @@ switch ($action) { case 'getpredictions': confirm_sesskey(); - echo $OUTPUT->header(); - if ($onlycli) { throw new moodle_exception('erroronlycli', 'tool_analytics'); } @@ -202,6 +204,11 @@ switch ($action) { $predictlogs = array(); } + // We reset the theme and the output as some indicators may be using external functions + // which reset $PAGE. + \tool_analytics\output\helper::reset_page(); + echo $OUTPUT->header(); + $renderer = $PAGE->get_renderer('tool_analytics'); echo $renderer->render_get_predictions_results($trainresults, $trainlogs, $predictresults, $predictlogs); break; diff --git a/admin/tool/mobile/classes/external.php b/admin/tool/mobile/classes/external.php index 17956350a89..e9d1c672021 100644 --- a/admin/tool/mobile/classes/external.php +++ b/admin/tool/mobile/classes/external.php @@ -262,6 +262,8 @@ class external extends external_api { /** * Creates an auto-login key for the current user. Is created only in https sites and is restricted by time and ip address. * + * Please note that it only works if the request comes from the Moodle mobile or desktop app. + * * @param string $privatetoken the user private token for validating the request * @return array with the settings and warnings * @since Moodle 3.2 @@ -284,6 +286,12 @@ class external extends external_api { } } + // Only requests from the Moodle mobile or desktop app. This enhances security to avoid any type of XSS attack. + // This code goes intentionally here and not inside the check_autologin_prerequisites() function because it + // is used by other PHP scripts that can be opened in any browser. + if (!\core_useragent::is_moodle_app()) { + throw new moodle_exception('apprequired', 'tool_mobile'); + } api::check_autologin_prerequisites($USER->id); if (isset($_GET['privatetoken']) or empty($privatetoken)) { diff --git a/admin/tool/mobile/db/services.php b/admin/tool/mobile/db/services.php index 0b50d6853cf..d53f7b41a7f 100644 --- a/admin/tool/mobile/db/services.php +++ b/admin/tool/mobile/db/services.php @@ -57,7 +57,8 @@ $functions = array( 'classname' => 'tool_mobile\external', 'methodname' => 'get_autologin_key', 'description' => 'Creates an auto-login key for the current user. - Is created only in https sites and is restricted by time and ip address.', + Is created only in https sites and is restricted by time, ip address and only works if the request + comes from the Moodle mobile or desktop app.', 'type' => 'write', 'services' => array(MOODLE_OFFICIAL_MOBILE_SERVICE), ), diff --git a/admin/tool/mobile/lang/en/tool_mobile.php b/admin/tool/mobile/lang/en/tool_mobile.php index a3738785296..0b01cd02a57 100644 --- a/admin/tool/mobile/lang/en/tool_mobile.php +++ b/admin/tool/mobile/lang/en/tool_mobile.php @@ -27,6 +27,7 @@ $string['androidappid'] = 'Android app\'s unique identifier'; $string['androidappid_desc'] = 'This setting may be left as default unless you have a custom Android app.'; $string['apppolicy'] = 'App policy URL'; $string['apppolicy_help'] = 'The URL of a policy for app users which is listed on the About page in the app. If the field is left empty, the site policy URL will be used instead.'; +$string['apprequired'] = 'This functionality is only available when accessed via the Moodle mobile or desktop app.'; $string['autologinkeygenerationlockout'] = 'Auto-login key generation is blocked. You need to wait 6 minutes between requests.'; $string['autologinnotallowedtoadmins'] = 'Auto-login is not allowed for site admins.'; $string['cachedef_plugininfo'] = 'This stores the list of plugins with mobile addons'; diff --git a/admin/tool/mobile/tests/externallib_test.php b/admin/tool/mobile/tests/externallib_test.php index 4045ab14d26..1fee731019c 100644 --- a/admin/tool/mobile/tests/externallib_test.php +++ b/admin/tool/mobile/tests/externallib_test.php @@ -213,6 +213,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase { // Enable requeriments. $_GET['wstoken'] = $token->token; // Mock parameters. + // Fake the app. + core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' . + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile'); + // Even if we force the password change for the current user we should be able to retrieve the key. set_user_preference('auth_forcepasswordchange', 1, $user->id); @@ -240,6 +244,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase { global $CFG; $this->resetAfterTest(true); + // Fake the app. + core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' . + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile'); + // Need to disable webservices to verify that's checked. $CFG->enablewebservices = 0; $CFG->enablemobilewebservice = 0; @@ -256,6 +264,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase { public function test_get_autologin_key_missing_https() { global $CFG; + // Fake the app. + core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' . + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile'); + // Need to simulate a non HTTPS site here. $CFG->wwwroot = str_replace('https:', 'http:', $CFG->wwwroot); @@ -276,6 +288,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase { $this->resetAfterTest(true); $this->setAdminUser(); + // Fake the app. + core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' . + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile'); + $this->expectException('moodle_exception'); $this->expectExceptionMessage(get_string('autologinnotallowedtoadmins', 'tool_mobile')); $result = external::get_autologin_key(''); @@ -296,6 +312,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase { $token = external_generate_token_for_current_user($service); $_GET['wstoken'] = $token->token; // Mock parameters. + // Fake the app. + core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' . + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile'); + $result = external::get_autologin_key($token->privatetoken); $result = external_api::clean_returnvalue(external::get_autologin_key_returns(), $result); @@ -311,6 +331,20 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase { $result = external::get_autologin_key($token->privatetoken); } + /** + * Test get_autologin_key missing app_request. + */ + public function test_get_autologin_key_missing_app_request() { + global $CFG; + + $this->resetAfterTest(true); + $this->setAdminUser(); + + $this->expectException('moodle_exception'); + $this->expectExceptionMessage(get_string('apprequired', 'tool_mobile')); + $result = external::get_autologin_key(''); + } + /** * Test get_content. */ diff --git a/admin/tool/mobile/upgrade.txt b/admin/tool/mobile/upgrade.txt index ec720c3491a..d39aa5fdc0c 100644 --- a/admin/tool/mobile/upgrade.txt +++ b/admin/tool/mobile/upgrade.txt @@ -4,6 +4,10 @@ Information provided here is intended especially for developers. === 3.7 === * New external function tool_mobile::tool_mobile_call_external_function allows calling multiple external functions and returns all responses. + * External function tool_mobile::get_autologin_key now only works if the request comes from the Moodle mobile or desktop app. + This increases confidence that requests did originate from the mobile app, decreasing the likelihood of an XSS attack. + If you want to use this functionality, please override the Web Service via the override_webservice_execution callback although + this is not recommended or encouraged. === 3.5 === diff --git a/admin/tool/recyclebin/classes/category_bin.php b/admin/tool/recyclebin/classes/category_bin.php index b08b6a5f5bb..6477c4d25c2 100644 --- a/admin/tool/recyclebin/classes/category_bin.php +++ b/admin/tool/recyclebin/classes/category_bin.php @@ -108,6 +108,16 @@ class category_bin extends base_bin { require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); + // As far as recycle bin is using MODE_AUTOMATED, it observes the backup_auto_storage + // setting (storing backups @ real location. For recycle bin we want to ensure that + // backup files are always stored in Moodle file area. In order to achieve that, we + // hack the setting here via $CFG->forced_plugin_settings, so it won't interfere other + // operations. See MDL-65218 for more information. + // This hack will be removed once recycle bin switches to use its own backup mode, with + // own preferences and 100% appart from MODLE_AUTOMATED. + // TODO: Remove this as part of MDL-65228. + $CFG->forced_plugin_settings['backup'] = ['backup_auto_storage' => 0]; + // Backup the course. $user = get_admin(); $controller = new \backup_controller( @@ -115,11 +125,15 @@ class category_bin extends base_bin { $course->id, \backup::FORMAT_MOODLE, \backup::INTERACTIVE_NO, - \backup::MODE_GENERAL, + \backup::MODE_AUTOMATED, $user->id ); $controller->execute_plan(); + // We don't need the forced setting anymore, hence unsetting it. + // TODO: Remove this as part of MDL-65228. + unset($CFG->forced_plugin_settings['backup']); + // Grab the result. $result = $controller->get_results(); if (!isset($result['backup_destination'])) { @@ -233,7 +247,7 @@ class category_bin extends base_bin { $tempdir, $course->id, \backup::INTERACTIVE_NO, - \backup::MODE_GENERAL, + \backup::MODE_AUTOMATED, $user->id, \backup::TARGET_NEW_COURSE ); diff --git a/admin/tool/recyclebin/classes/course_bin.php b/admin/tool/recyclebin/classes/course_bin.php index d58d1a77f7c..deec65c1cb2 100644 --- a/admin/tool/recyclebin/classes/course_bin.php +++ b/admin/tool/recyclebin/classes/course_bin.php @@ -112,6 +112,16 @@ class course_bin extends base_bin { return; } + // As far as recycle bin is using MODE_AUTOMATED, it observes the backup_auto_storage + // setting (storing backups @ real location. For recycle bin we want to ensure that + // backup files are always stored in Moodle file area. In order to achieve that, we + // hack the setting here via $CFG->forced_plugin_settings, so it won't interfere other + // operations. See MDL-65218 for more information. + // This hack will be removed once recycle bin switches to use its own backup mode, with + // own preferences and 100% appart from MODLE_AUTOMATED. + // TODO: Remove this as part of MDL-65228. + $CFG->forced_plugin_settings['backup'] = ['backup_auto_storage' => 0]; + // Backup the activity. $user = get_admin(); $controller = new \backup_controller( @@ -124,6 +134,10 @@ class course_bin extends base_bin { ); $controller->execute_plan(); + // We don't need the forced setting anymore, hence unsetting it. + // TODO: Remove this as part of MDL-65228. + unset($CFG->forced_plugin_settings['backup']); + // Grab the result. $result = $controller->get_results(); if (!isset($result['backup_destination'])) { @@ -344,4 +358,4 @@ class course_bin extends base_bin { $context = \context_course::instance($this->_courseid); return has_capability('tool/recyclebin:deleteitems', $context); } -} \ No newline at end of file +} diff --git a/admin/tool/recyclebin/tests/category_bin_test.php b/admin/tool/recyclebin/tests/category_bin_test.php index bada3d63853..0e24c311ca1 100644 --- a/admin/tool/recyclebin/tests/category_bin_test.php +++ b/admin/tool/recyclebin/tests/category_bin_test.php @@ -176,4 +176,116 @@ class tool_recyclebin_category_bin_tests extends advanced_testcase { $course = reset($courses); $this->assertEquals('Test course 2', $course->fullname); } + + /** + * Provider for test_course_restore_with_userdata() and test_course_restore_without_userdata() + * + * Used to verify that recycle bin is immune to various settings. Provides plugin, name, value for + * direct usage with set_config() + */ + public function recycle_bin_settings_provider() { + return [ + 'backup/backup_auto_storage moodle' => [[ + (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 0], + ]], + + 'backup/backup_auto_storage external' => [[ + (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 1], + (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true], + ]], + + 'backup/backup_auto_storage mixed' => [[ + (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 2], + (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true], + ]], + ]; + } + + /** + * Tests that user data is restored when course is restored. + * + * @dataProvider recycle_bin_settings_provider + * @param array $settings array of plugin, name, value stdClass(). + */ + public function test_course_restore_with_userdata($settings) { + global $DB; + + // Force configuration changes from provider. + foreach ($settings as $setting) { + // Need to create a directory for backup_auto_destination. + if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) { + $setting->value = make_request_directory(); + } + set_config($setting->name, $setting->value, $setting->plugin); + } + + // We want user data to be included for this test. + set_config('backup_auto_users', true, 'backup'); + + $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student'); + + // Delete course. + delete_course($this->course, false); + $this->assertFalse($DB->record_exists('course', ['id' => $this->course->id])); + + // Verify there is now a backup @ cat recycle bin file area. + $recyclebin = new \tool_recyclebin\category_bin($this->course->category); + $this->assertEquals(1, count($recyclebin->get_items())); + + // Restore the recycle bin item. + $recyclebin->restore_item(current($recyclebin->get_items())); + + // Get the new course. + $newcourse = $DB->get_record('course', ['shortname' => $this->course->shortname], '*', MUST_EXIST); + + // Check that it was removed from the recycle bin. + $this->assertEquals(0, count($recyclebin->get_items())); + + // Verify that student DOES continue enrolled. + $this->assertTrue(is_enrolled(context_course::instance($newcourse->id), $student->id)); + } + + /** + * Tests that user data is not restored when course is restored. + * + * @dataProvider recycle_bin_settings_provider + * @param array $settings array of plugin, name, value stdClass(). + */ + public function test_course_restore_without_userdata($settings) { + global $DB; + + // Force configuration changes from provider. + foreach ($settings as $setting) { + // Need to create a directory for backup_auto_destination. + if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) { + $setting->value = make_request_directory(); + } + set_config($setting->name, $setting->value, $setting->plugin); + } + + // We want user data to be included for this test. + set_config('backup_auto_users', false, 'backup'); + + $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student'); + + // Delete course. + delete_course($this->course, false); + $this->assertFalse($DB->record_exists('course', ['id' => $this->course->id])); + + // Verify there is now a backup @ cat recycle bin file area. + $recyclebin = new \tool_recyclebin\category_bin($this->course->category); + $this->assertEquals(1, count($recyclebin->get_items())); + + // Restore the recycle bin item. + $recyclebin->restore_item(current($recyclebin->get_items())); + + // Get the new course. + $newcourse = $DB->get_record('course', ['shortname' => $this->course->shortname], '*', MUST_EXIST); + + // Check that it was removed from the recycle bin. + $this->assertEquals(0, count($recyclebin->get_items())); + + // Verify that student DOES NOT continue enrolled. + $this->assertFalse(is_enrolled(context_course::instance($newcourse->id), $student->id)); + } } diff --git a/admin/tool/recyclebin/tests/course_bin_test.php b/admin/tool/recyclebin/tests/course_bin_test.php index 9d993260c36..44f9db6cc06 100644 --- a/admin/tool/recyclebin/tests/course_bin_test.php +++ b/admin/tool/recyclebin/tests/course_bin_test.php @@ -175,9 +175,45 @@ class tool_recyclebin_course_bin_tests extends advanced_testcase { } /** - * Tests that user data is restored when module is restored. + * Provider for test_coursemodule_restore_with_userdata() and test_coursemodule_restore_without_userdata() + * + * Used to verify that recycle bin is immune to various settings. Provides plugin, name, value for + * direct usage with set_config() */ - public function test_coursemodule_restore_with_userdata() { + public function recycle_bin_settings_provider() { + return [ + 'backup/backup_auto_storage moodle' => [[ + (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 0], + ]], + + 'backup/backup_auto_storage external' => [[ + (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 1], + (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true], + ]], + + 'backup/backup_auto_storage mixed' => [[ + (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 2], + (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true], + ]], + ]; + } + + /** + * Tests that user data is restored when module is restored. + * + * @dataProvider recycle_bin_settings_provider + * @param array $settings array of plugin, name, value stdClass(). + */ + public function test_coursemodule_restore_with_userdata($settings) { + // Force configuration changes from provider. + foreach ($settings as $setting) { + // Need to create a directory for backup_auto_destination. + if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) { + $setting->value = make_request_directory(); + } + set_config($setting->name, $setting->value, $setting->plugin); + } + $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student'); $this->setUser($student); @@ -211,8 +247,20 @@ class tool_recyclebin_course_bin_tests extends advanced_testcase { /** * Tests that user data is not restored when module is restored. + * + * @dataProvider recycle_bin_settings_provider + * @param array $settings array of plugin, name, value stdClass(). */ - public function test_coursemodule_restore_without_userdata() { + public function test_coursemodule_restore_without_userdata($settings) { + // Force configuration changes from provider. + foreach ($settings as $setting) { + // Need to create a directory for backup_auto_destination. + if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) { + $setting->value = make_request_directory(); + } + set_config($setting->name, $setting->value, $setting->plugin); + } + $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student'); $this->setUser($student); diff --git a/analytics/classes/analysable.php b/analytics/classes/analysable.php index e9dcaae46de..97faf596e8e 100644 --- a/analytics/classes/analysable.php +++ b/analytics/classes/analysable.php @@ -29,10 +29,6 @@ defined('MOODLE_INTERNAL') || die(); /** * Any element analysers can analyse. * - * Analysers get_analysers method return all analysable elements in the site; - * it is important that analysable elements implement lazy loading to avoid - * big memory footprints. See \core_analytics\course example. - * * @package core_analytics * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later diff --git a/analytics/classes/analysis.php b/analytics/classes/analysis.php new file mode 100644 index 00000000000..bcb621c6f0b --- /dev/null +++ b/analytics/classes/analysis.php @@ -0,0 +1,894 @@ +. + +/** + * Runs an analysis of the site. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Runs an analysis of the site. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class analysis { + + /** + * @var \core_analytics\local\analyser\base + */ + private $analyser; + + /** + * @var bool Whether to calculate the target or not in this run. + */ + private $includetarget; + + /** + * @var \core_analytics\local\analysis\result + */ + private $result; + + /** + * @var \core\lock\lock + */ + private $lock; + + /** + * Constructor. + * + * @param \core_analytics\local\analyser\base $analyser + * @param bool $includetarget Whether to calculate the target or not. + * @param \core_analytics\local\analysis\result $result + */ + public function __construct(\core_analytics\local\analyser\base $analyser, bool $includetarget, + \core_analytics\local\analysis\result $result) { + $this->analyser = $analyser; + $this->includetarget = $includetarget; + $this->result = $result; + } + + /** + * Runs the analysis. + * + * @return null + */ + public function run() { + + $options = $this->analyser->get_options(); + + // Time limit control. + $modeltimelimit = intval(get_config('analytics', 'modeltimelimit')); + + $filesbytimesplitting = array(); + + $alreadyprocessedanalysables = $this->get_processed_analysables(); + + if ($this->includetarget) { + $action = 'training'; + } else { + $action = 'prediction'; + } + $analysables = $this->analyser->get_analysables_iterator($action); + + $inittime = microtime(true); + foreach ($analysables as $analysable) { + $processed = false; + + if (!$analysable) { + continue; + } + + $analysableresults = $this->process_analysable($analysable); + if ($analysableresults) { + $processed = $this->result->add_analysable_results($analysableresults); + if (!$processed) { + $errors = array(); + foreach ($analysableresults as $timesplittingid => $result) { + $str = ''; + if (count($analysableresults) > 1) { + $str .= $timesplittingid . ': '; + } + $str .= $result->message; + $errors[] = $str; + } + + $a = new \stdClass(); + $a->analysableid = $analysable->get_name(); + $a->errors = implode(', ', $errors); + $this->analyser->add_log(get_string('analysablenotused', 'analytics', $a)); + } + } + + // Updated regardless of how well the analysis went. + if ($this->analyser->get_target()->always_update_analysis_time() || $processed) { + $this->update_analysable_analysed_time($alreadyprocessedanalysables, $analysable->get_id()); + } + + // Apply time limit. + if (!$options['evaluation']) { + $timespent = microtime(true) - $inittime; + if ($modeltimelimit <= $timespent) { + break; + } + } + } + } + + /** + * Get analysables that have been already processed. + * + * @return \stdClass[] + */ + protected function get_processed_analysables(): array { + global $DB; + + $params = array('modelid' => $this->analyser->get_modelid()); + $params['action'] = ($this->includetarget) ? 'training' : 'prediction'; + $select = 'modelid = :modelid and action = :action'; + + // Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid). + return $DB->get_records_select('analytics_used_analysables', $select, + $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey'); + } + + /** + * Processes an analysable + * + * This method returns the general analysable status, an array of files by time splitting method and + * an error message if there is any problem. + * + * @param \core_analytics\analysable $analysable + * @return \stdClass[] Results objects by time splitting method + */ + public function process_analysable(\core_analytics\analysable $analysable): array { + + // Target instances scope is per-analysable (it can't be lower as calculations run once per + // analysable, not time splitting method nor time range). + $target = call_user_func(array($this->analyser->get_target(), 'instance')); + + // We need to check that the analysable is valid for the target even if we don't include targets + // as we still need to discard invalid analysables for the target. + $isvalidresult = $target->is_valid_analysable($analysable, $this->includetarget); + if ($isvalidresult !== true) { + $a = new \stdClass(); + $a->analysableid = $analysable->get_name(); + $a->result = $isvalidresult; + $this->analyser->add_log(get_string('analysablenotvalidfortarget', 'analytics', $a)); + return array(); + } + + // Process all provided time splitting methods. + $results = array(); + foreach ($this->analyser->get_timesplittings() as $timesplitting) { + + $cachedresult = $this->result->retrieve_cached_result($timesplitting, $analysable); + if ($cachedresult) { + $result = new \stdClass(); + $result->result = $cachedresult; + $results[$timesplitting->get_id()] = $result; + continue; + } + + $results[$timesplitting->get_id()] = $this->process_time_splitting($timesplitting, $analysable, $target); + } + + return $results; + } + + /** + * Processes the analysable samples using the provided time splitting method. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\analysable $analysable + * @param \core_analytics\local\target\base $target + * @return \stdClass Results object. + */ + protected function process_time_splitting(\core_analytics\local\time_splitting\base $timesplitting, + \core_analytics\analysable $analysable, \core_analytics\local\target\base $target): \stdClass { + + $options = $this->analyser->get_options(); + + $result = new \stdClass(); + + if (!$timesplitting->is_valid_analysable($analysable)) { + $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; + $result->message = get_string('invalidanalysablefortimesplitting', 'analytics', + $timesplitting->get_name()); + return $result; + } + $timesplitting->set_analysable($analysable); + + if (CLI_SCRIPT && !PHPUNIT_TEST) { + mtrace('Analysing id "' . $analysable->get_id() . '" with "' . $timesplitting->get_name() . + '" time splitting method...'); + } + + // What is a sample is defined by the analyser, it can be an enrolment, a course, a user, a question + // attempt... it is on what we will base indicators calculations. + list($sampleids, $samplesdata) = $this->analyser->get_all_samples($analysable); + + if (count($sampleids) === 0) { + $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; + $result->message = get_string('nodata', 'analytics'); + return $result; + } + + if ($this->includetarget) { + // All ranges are used when we are calculating data for training. + $ranges = $timesplitting->get_training_ranges(); + } else { + // The latest range that has not yet been used for prediction (it depends on the time range where we are right now). + $ranges = $timesplitting->get_most_recent_prediction_range(); + } + + // There is no need to keep track of the evaluated samples and ranges as we always evaluate the whole dataset. + if ($options['evaluation'] === false) { + + if (empty($ranges)) { + $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; + $result->message = get_string('noranges', 'analytics'); + return $result; + } + + // We skip all samples that are already part of a training dataset, even if they have not been used for prediction. + if (!$target::based_on_assumptions()) { + // Targets based on assumptions can not be trained. + $this->filter_out_train_samples($sampleids, $timesplitting); + } + + if (count($sampleids) === 0) { + $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; + $result->message = get_string('nonewdata', 'analytics'); + return $result; + } + + // Only when processing data for predictions. + if (!$this->includetarget) { + // We also filter out samples and ranges that have already been used for predictions. + $predictsamplesrecord = $this->filter_out_prediction_samples_and_ranges($sampleids, $ranges, $timesplitting); + } + + if (count($sampleids) === 0) { + $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; + $result->message = get_string('nonewdata', 'analytics'); + return $result; + } + + if (count($ranges) === 0) { + $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; + $result->message = get_string('nonewranges', 'analytics'); + return $result; + } + } + + // Flag the model + analysable + timesplitting as being analysed (prevent concurrent executions). + if (!$this->init_analysable_analysis($timesplitting->get_id(), $analysable->get_id())) { + // If this model + analysable + timesplitting combination is being analysed we skip this process. + $result->status = \core_analytics\model::NO_DATASET; + $result->message = get_string('analysisinprogress', 'analytics'); + return $result; + } + + // Remove samples the target consider invalid. + try { + $target->add_sample_data($samplesdata); + $target->filter_out_invalid_samples($sampleids, $analysable, $this->includetarget); + } catch (\Throwable $e) { + $this->finish_analysable_analysis(); + throw $e; + } + + if (!$sampleids) { + $result->status = \core_analytics\model::NO_DATASET; + $result->message = get_string('novalidsamples', 'analytics'); + $this->finish_analysable_analysis(); + return $result; + } + + try { + $indicators = $this->analyser->get_indicators(); + foreach ($indicators as $key => $indicator) { + // The analyser attaches the main entities the sample depends on and are provided to the + // indicator to calculate the sample. + $indicators[$key]->add_sample_data($samplesdata); + } + + // Here we start the memory intensive process that will last until $data var is + // unset (until the method is finished basically). + $data = $this->calculate($timesplitting, $sampleids, $ranges, $target); + } catch (\Throwable $e) { + $this->finish_analysable_analysis(); + throw $e; + } + + if (!$data) { + $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; + $result->message = get_string('novaliddata', 'analytics'); + $this->finish_analysable_analysis(); + return $result; + } + + try { + // No need to keep track of analysed stuff when evaluating. + if ($options['evaluation'] === false) { + // Save the samples that have been already analysed so they are not analysed again in future. + + if ($this->includetarget) { + $this->save_train_samples($sampleids, $timesplitting); + } else { + // The variable $predictsamplesrecord will always be set as filter_out_prediction_samples_and_ranges + // will always be called before it (no evaluation mode and no includetarget). + $this->save_prediction_samples($sampleids, $ranges, $timesplitting, $predictsamplesrecord); + } + } + + // We need to pass all the analysis data. + $formattedresult = $this->result->format_result($data, $target, $timesplitting, $analysable); + + } catch (\Throwable $e) { + $this->finish_analysable_analysis(); + throw $e; + } + + if (!$formattedresult) { + $this->finish_analysable_analysis(); + throw new \moodle_exception('errorcannotwritedataset', 'analytics'); + } + + $result->status = \core_analytics\model::OK; + $result->message = get_string('successfullyanalysed', 'analytics'); + $result->result = $formattedresult; + + // Flag the model + analysable + timesplitting as analysed. + $this->finish_analysable_analysis(); + + return $result; + } + + /** + * Calculates indicators and targets. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param array $sampleids + * @param array $ranges + * @param \core_analytics\local\target\base $target + * @return array|null + */ + public function calculate(\core_analytics\local\time_splitting\base $timesplitting, array &$sampleids, + array $ranges, \core_analytics\local\target\base $target): ?array { + + $calculatedtarget = null; + if ($this->includetarget) { + // We first calculate the target because analysable data may still be invalid or none + // of the analysable samples may be valid. + $calculatedtarget = $target->calculate($sampleids, $timesplitting->get_analysable()); + + // We remove samples we can not calculate their target. + $sampleids = array_filter($sampleids, function($sampleid) use ($calculatedtarget) { + if (is_null($calculatedtarget[$sampleid])) { + return false; + } + return true; + }); + } + + // No need to continue calculating if the target couldn't be calculated for any sample. + if (empty($sampleids)) { + return null; + } + + $dataset = $this->calculate_indicators($timesplitting, $sampleids, $ranges); + + if (empty($dataset)) { + return null; + } + + // Now that we have the indicators in place we can add the time range indicators (and target if provided) to each of them. + $this->fill_dataset($timesplitting, $dataset, $calculatedtarget); + + $this->add_context_metadata($timesplitting, $dataset, $target); + + if (!PHPUNIT_TEST && CLI_SCRIPT) { + echo PHP_EOL; + } + + return $dataset; + } + + /** + * Calculates indicators. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param array $sampleids + * @param array $ranges + * @return array + */ + protected function calculate_indicators(\core_analytics\local\time_splitting\base $timesplitting, array $sampleids, + array $ranges): array { + global $DB; + + $options = $this->analyser->get_options(); + + $dataset = array(); + + // Faster to run 1 db query per range. + $existingcalculations = array(); + if ($timesplitting->cache_indicator_calculations()) { + foreach ($ranges as $rangeindex => $range) { + // Load existing calculations. + $existingcalculations[$rangeindex] = \core_analytics\manager::get_indicator_calculations( + $timesplitting->get_analysable(), $range['start'], $range['end'], $this->analyser->get_samples_origin()); + } + } + + // Here we store samples which calculations are not all null. + $notnulls = array(); + + // Fill the dataset samples with indicators data. + $newcalculations = array(); + foreach ($this->analyser->get_indicators() as $indicator) { + + // Hook to allow indicators to store analysable-dependant data. + $indicator->fill_per_analysable_caches($timesplitting->get_analysable()); + + // Per-range calculations. + foreach ($ranges as $rangeindex => $range) { + + // Indicator instances are per-range. + $rangeindicator = clone $indicator; + + $prevcalculations = array(); + if (!empty($existingcalculations[$rangeindex][$rangeindicator->get_id()])) { + $prevcalculations = $existingcalculations[$rangeindex][$rangeindicator->get_id()]; + } + + // Calculate the indicator for each sample in this time range. + list($samplesfeatures, $newindicatorcalculations, $indicatornotnulls) = $rangeindicator->calculate($sampleids, + $this->analyser->get_samples_origin(), $range['start'], $range['end'], $prevcalculations); + + // Copy the features data to the dataset. + foreach ($samplesfeatures as $analysersampleid => $features) { + + $uniquesampleid = $timesplitting->append_rangeindex($analysersampleid, $rangeindex); + + if (!isset($notnulls[$uniquesampleid]) && !empty($indicatornotnulls[$analysersampleid])) { + $notnulls[$uniquesampleid] = $uniquesampleid; + } + + // Init the sample if it is still empty. + if (!isset($dataset[$uniquesampleid])) { + $dataset[$uniquesampleid] = array(); + } + + // Append the features indicator features at the end of the sample. + $dataset[$uniquesampleid] = array_merge($dataset[$uniquesampleid], $features); + } + + if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations()) { + $timecreated = time(); + foreach ($newindicatorcalculations as $sampleid => $calculatedvalue) { + // Prepare the new calculations to be stored into DB. + + $indcalc = new \stdClass(); + $indcalc->contextid = $timesplitting->get_analysable()->get_context()->id; + $indcalc->starttime = $range['start']; + $indcalc->endtime = $range['end']; + $indcalc->sampleid = $sampleid; + $indcalc->sampleorigin = $this->analyser->get_samples_origin(); + $indcalc->indicator = $rangeindicator->get_id(); + $indcalc->value = $calculatedvalue; + $indcalc->timecreated = $timecreated; + $newcalculations[] = $indcalc; + } + } + } + + if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations()) { + $batchsize = self::get_insert_batch_size(); + if (count($newcalculations) > $batchsize) { + // We don't want newcalculations array to grow too much as we already keep the + // system memory busy storing $dataset contents. + + // Insert from the beginning. + $remaining = array_splice($newcalculations, $batchsize); + + // Sorry mssql and oracle, this will be slow. + $DB->insert_records('analytics_indicator_calc', $newcalculations); + $newcalculations = $remaining; + } + } + } + + if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations() && $newcalculations) { + // Insert the remaining records. + $DB->insert_records('analytics_indicator_calc', $newcalculations); + } + + // Delete rows where all calculations are null. + // We still store the indicator calculation and we still store the sample id as + // processed so we don't have to process this sample again, but we exclude it + // from the dataset because it is not useful. + $nulls = array_diff_key($dataset, $notnulls); + foreach ($nulls as $uniqueid => $ignoredvalues) { + unset($dataset[$uniqueid]); + } + + return $dataset; + } + + /** + * Adds time range indicators and the target to each sample. + * + * This will identify the sample as belonging to a specific range. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param array $dataset + * @param array|null $calculatedtarget + * @return null + */ + protected function fill_dataset(\core_analytics\local\time_splitting\base $timesplitting, + array &$dataset, ?array $calculatedtarget = null) { + + $nranges = count($timesplitting->get_distinct_ranges()); + + foreach ($dataset as $uniquesampleid => $unmodified) { + + list($analysersampleid, $rangeindex) = $timesplitting->infer_sample_info($uniquesampleid); + + // No need to add range features if this time splitting method only defines one time range. + if ($nranges > 1) { + + // 1 column for each range. + $timeindicators = array_fill(0, $nranges, 0); + + $timeindicators[$rangeindex] = 1; + + $dataset[$uniquesampleid] = array_merge($timeindicators, $dataset[$uniquesampleid]); + } + + if ($calculatedtarget) { + // Add this sampleid's calculated target and the end. + $dataset[$uniquesampleid][] = $calculatedtarget[$analysersampleid]; + + } else { + // Add this sampleid, it will be used to identify the prediction that comes back from + // the predictions processor. + array_unshift($dataset[$uniquesampleid], $uniquesampleid); + } + } + } + + /** + * Updates the analysable analysis time. + * + * @param array $processedanalysables + * @param int $analysableid + * @return null + */ + protected function update_analysable_analysed_time(array $processedanalysables, int $analysableid) { + global $DB; + + if (!empty($processedanalysables[$analysableid])) { + $obj = $processedanalysables[$analysableid]; + + $obj->id = $obj->primarykey; + unset($obj->primarykey); + + $obj->timeanalysed = time(); + $DB->update_record('analytics_used_analysables', $obj); + + } else { + + $obj = new \stdClass(); + $obj->modelid = $this->analyser->get_modelid(); + $obj->action = ($this->includetarget) ? 'training' : 'prediction'; + $obj->analysableid = $analysableid; + $obj->timeanalysed = time(); + + $DB->insert_record('analytics_used_analysables', $obj); + } + } + + /** + * Adds dataset context info. + * + * The final dataset document will look like this: + * ---------------------------------------------------- + * metadata1,metadata2,metadata3,..... + * value1, value2, value3,..... + * + * header1,header2,header3,header4,..... + * stud1value1,stud1value2,stud1value3,stud1value4,..... + * stud2value1,stud2value2,stud2value3,stud2value4,..... + * ..... + * ---------------------------------------------------- + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param array $dataset + * @param \core_analytics\local\target\base $target + * @return null + */ + protected function add_context_metadata(\core_analytics\local\time_splitting\base $timesplitting, array &$dataset, + \core_analytics\local\target\base $target) { + $headers = $this->get_headers($timesplitting, $target); + + // This will also reset samples' dataset keys. + array_unshift($dataset, $headers); + } + + /** + * Returns the headers for the csv file based on the indicators and the target. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\local\target\base $target + * @return string[] + */ + public function get_headers(\core_analytics\local\time_splitting\base $timesplitting, + \core_analytics\local\target\base $target): array { + // 3rd column will contain the indicator ids. + $headers = array(); + + if (!$this->includetarget) { + // The first column is the sampleid. + $headers[] = 'sampleid'; + } + + // We always have 1 column for each time splitting method range, it does not depend on how + // many ranges we calculated. + $ranges = $timesplitting->get_distinct_ranges(); + if (count($ranges) > 1) { + foreach ($ranges as $rangeindex) { + $headers[] = 'range/' . $rangeindex; + } + } + + // Model indicators. + foreach ($this->analyser->get_indicators() as $indicator) { + $headers = array_merge($headers, $indicator::get_feature_headers()); + } + + // The target as well. + if ($this->includetarget) { + $headers[] = $target->get_id(); + } + + return $headers; + } + + /** + * Filters out samples that have already been used for training. + * + * @param int[] $sampleids + * @param \core_analytics\local\time_splitting\base $timesplitting + * @return null + */ + protected function filter_out_train_samples(array &$sampleids, \core_analytics\local\time_splitting\base $timesplitting) { + global $DB; + + $params = array('modelid' => $this->analyser->get_modelid(), 'analysableid' => $timesplitting->get_analysable()->get_id(), + 'timesplitting' => $timesplitting->get_id()); + + $trainingsamples = $DB->get_records('analytics_train_samples', $params); + + // Skip each file trained samples. + foreach ($trainingsamples as $trainingfile) { + + $usedsamples = json_decode($trainingfile->sampleids, true); + + if (!empty($usedsamples)) { + // Reset $sampleids to $sampleids minus this file's $usedsamples. + $sampleids = array_diff_key($sampleids, $usedsamples); + } + } + } + + /** + * Filters out samples that have already been used for prediction. + * + * @param int[] $sampleids + * @param array $ranges + * @param \core_analytics\local\time_splitting\base $timesplitting + * @return \stdClass|null The analytics_predict_samples record or null + */ + protected function filter_out_prediction_samples_and_ranges(array &$sampleids, array &$ranges, + \core_analytics\local\time_splitting\base $timesplitting) { + + if (count($ranges) > 1) { + throw new \coding_exception('$ranges argument should only contain one range'); + } + + $rangeindex = key($ranges); + $predictedrange = $this->get_predict_samples_record($timesplitting, $rangeindex); + + if (!$predictedrange) { + // Nothing to filter out. + return null; + } + + $predictedrange->sampleids = json_decode($predictedrange->sampleids, true); + $missingsamples = array_diff_key($sampleids, $predictedrange->sampleids); + if (count($missingsamples) === 0) { + // All samples already calculated. + unset($ranges[$rangeindex]); + return null; + } + + // Replace the list of samples by the one excluding samples that already got predictions at this range. + $sampleids = $missingsamples; + + return $predictedrange; + } + + /** + * Returns a predict samples record. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param int $rangeindex + * @return \stdClass|false + */ + private function get_predict_samples_record(\core_analytics\local\time_splitting\base $timesplitting, int $rangeindex) { + global $DB; + + $params = array('modelid' => $this->analyser->get_modelid(), 'analysableid' => $timesplitting->get_analysable()->get_id(), + 'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex); + $predictedrange = $DB->get_record('analytics_predict_samples', $params); + + return $predictedrange; + } + + /** + * Saves samples that have just been used for training. + * + * @param int[] $sampleids + * @param \core_analytics\local\time_splitting\base $timesplitting + * @return null + */ + protected function save_train_samples(array $sampleids, \core_analytics\local\time_splitting\base $timesplitting) { + global $DB; + + $trainingsamples = new \stdClass(); + $trainingsamples->modelid = $this->analyser->get_modelid(); + $trainingsamples->analysableid = $timesplitting->get_analysable()->get_id(); + $trainingsamples->timesplitting = $timesplitting->get_id(); + + $trainingsamples->sampleids = json_encode($sampleids); + $trainingsamples->timecreated = time(); + + $DB->insert_record('analytics_train_samples', $trainingsamples); + } + + /** + * Saves samples that have just been used for prediction. + * + * @param int[] $sampleids + * @param array $ranges + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \stdClass|null $predictsamplesrecord The existing record or null if there is no record yet. + * @return null + */ + protected function save_prediction_samples(array $sampleids, array $ranges, + \core_analytics\local\time_splitting\base $timesplitting, ?\stdClass $predictsamplesrecord = null) { + global $DB; + + if (count($ranges) > 1) { + throw new \coding_exception('$ranges argument should only contain one range'); + } + + $rangeindex = key($ranges); + + if ($predictsamplesrecord) { + // Append the new samples used for prediction. + $predictsamplesrecord->sampleids = json_encode($predictsamplesrecord->sampleids + $sampleids); + $predictsamplesrecord->timemodified = time(); + $DB->update_record('analytics_predict_samples', $predictsamplesrecord); + } else { + $predictsamplesrecord = (object)[ + 'modelid' => $this->analyser->get_modelid(), + 'analysableid' => $timesplitting->get_analysable()->get_id(), + 'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex + ]; + $predictsamplesrecord->sampleids = json_encode($sampleids); + $predictsamplesrecord->timecreated = time(); + $predictsamplesrecord->timemodified = $predictsamplesrecord->timecreated; + $DB->insert_record('analytics_predict_samples', $predictsamplesrecord); + } + } + + /** + * Flags the analysable element as in-analysis and stores a lock for it. + * + * @param string $timesplittingid + * @param int $analysableid + * @return bool Success or not + */ + private function init_analysable_analysis(string $timesplittingid, int $analysableid) { + + // Do not include $this->includetarget as we don't want the same analysable to be analysed for training + // and prediction at the same time. + $lockkey = 'modelid:' . $this->analyser->get_modelid() . '-analysableid:' . $analysableid . + '-timesplitting:' . self::clean_time_splitting_id($timesplittingid); + + // Large timeout as processes may be quite long. + $lockfactory = \core\lock\lock_config::get_lock_factory('core_analytics'); + + // If it is not ready in 10 secs skip this model + analysable + timesplittingmethod combination + // it will attempt it again during next cron run. + if (!$this->lock = $lockfactory->get_lock($lockkey, 10)) { + return false; + } + return true; + } + + + /** + * Remove all possibly problematic chars from the time splitting method id (id = its full class name). + * + * @param string $timesplittingid + * @return string + */ + public static function clean_time_splitting_id($timesplittingid) { + $timesplittingid = str_replace('\\', '-', $timesplittingid); + return clean_param($timesplittingid, PARAM_ALPHANUMEXT); + } + + /** + * Mark the currently analysed analysable+timesplitting as analysed. + * + * @return null + */ + private function finish_analysable_analysis() { + $this->lock->release(); + } + + /** + * Returns the batch size used for insert_records. + * + * This method tries to find the best batch size without getting + * into dml internals. Maximum 1000 records to save memory. + * + * @return int + */ + private static function get_insert_batch_size(): int { + global $DB; + + $dbconfig = $DB->export_dbconfig(); + + // 500 is pgsql default so using 1000 is fine, no other db driver uses a hardcoded value. + if (empty($dbconfig) || empty($dbconfig->dboptions) || empty($dbconfig->dboptions['bulkinsertsize'])) { + return 1000; + } + + $bulkinsert = $dbconfig->dboptions['bulkinsertsize']; + if ($bulkinsert < 1000) { + return $bulkinsert; + } + + while ($bulkinsert > 1000) { + $bulkinsert = round($bulkinsert / 2, 0); + } + + return (int)$bulkinsert; + } +} diff --git a/analytics/classes/course.php b/analytics/classes/course.php index 919f36bd690..6e372cfb733 100644 --- a/analytics/classes/course.php +++ b/analytics/classes/course.php @@ -132,12 +132,11 @@ class course implements \core_analytics\analysable { * Use self::instance() instead to get cached copies of the course. Instances obtained * through this constructor will not be cached. * - * Lazy load of course data, students and teachers. - * - * @param int|\stdClass $course Course id + * @param int|\stdClass $course Course id or mdl_course record + * @param \context|null $context * @return void */ - public function __construct($course) { + public function __construct($course, ?\context $context = null) { if (is_scalar($course)) { $this->course = new \stdClass(); @@ -145,6 +144,10 @@ class course implements \core_analytics\analysable { } else { $this->course = $course; } + + if (!is_null($context)) { + $this->coursecontext = $context; + } } /** @@ -153,9 +156,10 @@ class course implements \core_analytics\analysable { * Lazy load of course data, students and teachers. * * @param int|\stdClass $course Course object or course id + * @param \context|null $context * @return \core_analytics\course */ - public static function instance($course) { + public static function instance($course, ?\context $context = null) { $courseid = $course; if (!is_scalar($courseid)) { @@ -166,7 +170,7 @@ class course implements \core_analytics\analysable { return self::$cachedinstance; } - $cachedinstance = new \core_analytics\course($course); + $cachedinstance = new \core_analytics\course($course, $context); self::$cachedinstance = $cachedinstance; self::$cachedid = (int)$courseid; return self::$cachedinstance; diff --git a/analytics/classes/dataset_manager.php b/analytics/classes/dataset_manager.php index 00b571a5725..811bf7cca6e 100644 --- a/analytics/classes/dataset_manager.php +++ b/analytics/classes/dataset_manager.php @@ -109,31 +109,8 @@ class dataset_manager { $this->modelid = $modelid; $this->analysableid = $analysableid; $this->timesplittingid = $timesplittingid; - $this->evaluation = $evaluation; $this->filearea = $filearea; - } - - /** - * Mark the analysable as being analysed. - * - * @return bool Could we get the lock or not. - */ - public function init_process() { - - // Do not include $this->includetarget as we don't want the same analysable to be analysed for training - // and prediction at the same time. - $lockkey = 'modelid:' . $this->modelid . '-analysableid:' . $this->analysableid . - '-timesplitting:' . self::clean_time_splitting_id($this->timesplittingid); - - // Large timeout as processes may be quite long. - $lockfactory = \core\lock\lock_config::get_lock_factory('core_analytics'); - - // If it is not ready in 10 secs skip this model + analysable + timesplittingmethod combination - // it will attempt it again during next cron run. - if (!$this->lock = $lockfactory->get_lock($lockkey, 10)) { - return false; - } - return true; + $this->evaluation = $evaluation; } /** @@ -152,7 +129,8 @@ class dataset_manager { 'filearea' => $this->filearea, 'itemid' => $this->modelid, 'contextid' => \context_system::instance()->id, - 'filepath' => '/analysable/' . $this->analysableid . '/' . self::clean_time_splitting_id($this->timesplittingid) . '/', + 'filepath' => '/analysable/' . $this->analysableid . '/' . + \core_analytics\analysis::clean_time_splitting_id($this->timesplittingid) . '/', 'filename' => self::get_filename($this->evaluation) ]; @@ -167,8 +145,7 @@ class dataset_manager { $filepath = make_request_directory() . DIRECTORY_SEPARATOR . $filerecord['filename']; $fh = fopen($filepath, 'w+'); if (!$fh) { - $this->close_process(); - throw new \moodle_exception('errorcannotwritedataset', 'analytics', '', $filepath); + return false; } foreach ($data as $line) { fputcsv($fh, $line); @@ -178,15 +155,6 @@ class dataset_manager { return $fs->create_file_from_pathname($filerecord, $filepath); } - /** - * Mark as analysed. - * - * @return void - */ - public function close_process() { - $this->lock->release(); - } - /** * Returns the previous evaluation file. * @@ -200,8 +168,9 @@ class dataset_manager { public static function get_previous_evaluation_file($modelid, $timesplittingid) { $fs = get_file_storage(); // Evaluation data is always labelled. + $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/'; return $fs->get_file(\context_system::instance()->id, 'analytics', self::LABELLED_FILEAREA, $modelid, - '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/', self::EVALUATION_FILENAME); + $filepath, self::EVALUATION_FILENAME); } /** @@ -233,7 +202,7 @@ class dataset_manager { $filesbytimesplitting = array(); foreach ($timesplittingids as $timesplittingid) { - $filepath = '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/'; + $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/'; $files = $fs->get_directory_files(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath); foreach ($files as $file) { @@ -291,7 +260,8 @@ class dataset_manager { // Always evaluation.csv and labelled as it is an evaluation file. $filearea = self::LABELLED_FILEAREA; $filename = self::get_filename(true); - $filepath = '/analysable/' . $analysableid . '/' . self::clean_time_splitting_id($timesplittingid) . '/'; + $filepath = '/analysable/' . $analysableid . '/' . + \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/'; return $fs->get_file(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath, $filename); } @@ -372,7 +342,7 @@ class dataset_manager { 'filearea' => $filearea, 'itemid' => $modelid, 'contextid' => \context_system::instance()->id, - 'filepath' => '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/', + 'filepath' => '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/', 'filename' => self::get_filename($evaluation) ]; @@ -393,7 +363,7 @@ class dataset_manager { $fs = get_file_storage(); $contextid = \context_system::instance()->id; - $filepath = '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/'; + $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/'; $files = $fs->get_directory_files($contextid, 'analytics', self::LABELLED_FILEAREA, $modelid, $filepath, true, false); @@ -467,17 +437,6 @@ class dataset_manager { return $fs->delete_area_files(\context_system::instance()->id, 'analytics', false, $modelid); } - /** - * Remove all possibly problematic chars from the time splitting method id (id = its full class name). - * - * @param string $timesplittingid - * @return string - */ - protected static function clean_time_splitting_id($timesplittingid) { - $timesplittingid = str_replace('\\', '-', $timesplittingid); - return clean_param($timesplittingid, PARAM_ALPHANUMEXT); - } - /** * Returns the file name to be used. * diff --git a/analytics/classes/insights_generator.php b/analytics/classes/insights_generator.php new file mode 100644 index 00000000000..24fdfaf1d8d --- /dev/null +++ b/analytics/classes/insights_generator.php @@ -0,0 +1,236 @@ +. + +/** + * Insights generator. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/lib/messagelib.php'); + +/** + * Insights generator. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class insights_generator { + + /** + * @var int + */ + private $modelid; + + /** + * @var \core_analytics\local\target\base + */ + private $target; + + /** + * @var int[] + */ + private $contextcourseids; + + /** + * Constructor. + * + * @param int $modelid + * @param \core_analytics\local\target\base $target + */ + public function __construct(int $modelid, \core_analytics\local\target\base $target) { + $this->modelid = $modelid; + $this->target = $target; + } + + /** + * Generates insight notifications. + * + * @param array $samplecontexts The contexts these predictions belong to + * @param \core_analytics\prediction[] $predictions The prediction records + * @return null + */ + public function generate($samplecontexts, $predictions) { + global $OUTPUT; + + $analyserclass = $this->target->get_analyser_class(); + + // We will need to restore it later. + $actuallanguage = current_language(); + + if ($analyserclass::one_sample_per_analysable()) { + + // Iterate through the predictions and the users in each prediction (likely to be just one). + foreach ($predictions as $prediction) { + + $context = $samplecontexts[$prediction->get_prediction_data()->contextid]; + + $users = $this->target->get_insights_users($context); + foreach ($users as $user) { + + $this->set_notification_language($user); + list($insighturl, $fullmessage, $fullmessagehtml) = $this->prediction_info($prediction); + $this->notification($context, $user, $insighturl, $fullmessage, $fullmessagehtml); + } + } + + } else { + + // Iterate through the context and the users in each context. + foreach ($samplecontexts as $context) { + + $users = $this->target->get_insights_users($context); + foreach ($users as $user) { + + $this->set_notification_language($user); + + $insighturl = $this->target->get_insight_context_url($this->modelid, $context); + + $fullmessage = get_string('insightinfomessage', 'analytics', $insighturl->out(false)); + $fullmessagehtml = $OUTPUT->render_from_template('core_analytics/insight_info_message', + ['url' => $insighturl->out(false)] + ); + + $this->notification($context, $user, $insighturl, $fullmessage, $fullmessagehtml); + } + } + } + + force_current_language($actuallanguage); + } + + /** + * Generates a insight notification for the user. + * + * @param \context $context + * @param \stdClass $user + * @param \moodle_url $insighturl The insight URL + * @param string $fullmessage + * @param string $fullmessagehtml + * @return null + */ + private function notification(\context $context, \stdClass $user, \moodle_url $insighturl, string $fullmessage, string $fullmessagehtml) { + + $message = new \core\message\message(); + $message->component = 'moodle'; + $message->name = 'insights'; + + $message->userfrom = \core_user::get_noreply_user(); + $message->userto = $user; + + $message->subject = $this->target->get_insight_subject($this->modelid, $context); + + // Same than the subject. + $message->contexturlname = $message->subject; + $message->courseid = $this->get_context_courseid($context); + + $message->fullmessage = $fullmessage; + $message->fullmessageformat = FORMAT_PLAIN; + $message->fullmessagehtml = $fullmessagehtml; + $message->smallmessage = $fullmessage; + $message->contexturl = $insighturl->out(false); + + message_send($message); + } + + /** + * Returns the course context of the provided context reading an internal cache first. + * + * @param \context $context + * @return int + */ + private function get_context_courseid(\context $context) { + + if (empty($this->contextcourseids[$context->id])) { + + $coursecontext = $context->get_course_context(false); + if (!$coursecontext) { + // Default to the frontpage course context. + $coursecontext = \context_course::instance(SITEID); + } + $this->contextcourseids[$context->id] = $coursecontext->instanceid; + } + + return $this->contextcourseids[$context->id]; + } + + /** + * Extracts info from the prediction for display purposes. + * + * @param \core_analytics\prediction $prediction + * @return array Three items array with formats [\moodle_url, string, string] + */ + private function prediction_info(\core_analytics\prediction $prediction) { + global $OUTPUT; + + $predictionactions = $this->target->prediction_actions($prediction, true, true); + + // For FORMAT_PLAIN. + $fullmessageplaintext = ''; + + // For FORMAT_HTML. + $messageactions = []; + $insighturl = null; + foreach ($predictionactions as $action) { + $actionurl = $action->get_url(); + if (!$actionurl->get_param('forwardurl')) { + + $actiondoneurl = new \moodle_url('/report/insights/done.php'); + // Set the forward url to the 'done' script. + $actionurl->param('forwardurl', $actiondoneurl->out(false)); + } + + if (empty($insighturl)) { + // We use the primary action url as insight url so we log that the user followed the provided link. + $insighturl = $action->get_url(); + } + $actiondata = (object)['url' => $action->get_url()->out(false), 'text' => $action->get_text()]; + $fullmessageplaintext .= get_string('insightinfomessageaction', 'analytics', $actiondata) . PHP_EOL; + $messageactions[] = $actiondata; + } + + $fullmessagehtml = $OUTPUT->render_from_template('core_analytics/insight_info_message_prediction', + ['actions' => $messageactions]); + return [$insighturl, $fullmessageplaintext, $fullmessagehtml]; + } + + /** + * Sets the session language to the language used by the notification receiver. + * + * @param \stdClass $user The user who will receive the message + * @return null + */ + private function set_notification_language($user) { + global $CFG; + + // Copied from current_language(). + if (!empty($user->lang)) { + $lang = $user->lang; + } else if (isset($CFG->lang)) { + $lang = $CFG->lang; + } else { + $lang = 'en'; + } + force_current_language($lang); + } +} diff --git a/analytics/classes/local/analyser/base.php b/analytics/classes/local/analyser/base.php index 06b7468e847..1c65e2c6273 100644 --- a/analytics/classes/local/analyser/base.php +++ b/analytics/classes/local/analyser/base.php @@ -47,13 +47,6 @@ abstract class base { */ protected $target; - /** - * A $this->$target copy loaded with the ongoing analysis analysable. - * - * @var \core_analytics\local\target\base - */ - protected $analysabletarget; - /** * The model indicators. * @@ -118,9 +111,37 @@ abstract class base { * \core_analytics\local\analyser\by_course and \core_analytics\local\analyser\sitewide are implementing * this method returning site courses (by_course) and the whole system (sitewide) as analysables. * + * @todo MDL-65284 This will be removed in Moodle 4.1 + * @deprecated + * @see get_analysables_iterator + * @throws \coding_exception * @return \core_analytics\analysable[] Array of analysable elements using the analysable id as array key. */ - abstract public function get_analysables(); + public function get_analysables() { + // This function should only be called from get_analysables_iterator and we keep it here until Moodle 4.1 + // for backwards compatibility. + throw new \coding_exception('This method is deprecated in favour of get_analysables_iterator.'); + } + + /** + * Returns the list of analysable elements available on the site. + * + * A relatively complex SQL query should be set so that we take into account which analysable elements + * have already been processed and the order in which they have been processed. Helper methods are available + * to ease to implementation of get_analysables_iterator: get_iterator_sql and order_sql. + * + * @param string|null $action 'prediction', 'training' or null if no specific action needed. + * @return \Iterator + */ + public function get_analysables_iterator(?string $action = null) { + + debugging('Please overwrite get_analysables_iterator with your own implementation, we only keep this default + implementation for backwards compatibility purposes with get_analysables(). note that $action param will + be ignored so the analysable elements will be processed using get_analysables order, regardless of the + last time they were processed.'); + + return new \ArrayIterator($this->get_analysables()); + } /** * This function returns this analysable list of samples. @@ -128,7 +149,7 @@ abstract class base { * @param \core_analytics\analysable $analysable * @return array array[0] = int[] (sampleids) and array[1] = array (samplesdata) */ - abstract protected function get_all_samples(\core_analytics\analysable $analysable); + abstract public function get_all_samples(\core_analytics\analysable $analysable); /** * This function returns the samples data from a list of sample ids. @@ -174,70 +195,46 @@ abstract class base { abstract public function sample_description($sampleid, $contextid, $sampledata); /** - * Main analyser method which processes the site analysables. - * - * @param bool $includetarget - * @return \stored_file[] + * Model id getter. + * @return int */ - public function get_analysable_data($includetarget) { - global $DB; + public function get_modelid(): int { + return $this->modelid; + } - // Time limit control. - $modeltimelimit = intval(get_config('analytics', 'modeltimelimit')); + /** + * Options getter. + * @return array + */ + public function get_options(): array { + return $this->options; + } - $filesbytimesplitting = array(); + /** + * Returns the analysed target. + * + * @return \core_analytics\local\target\base + */ + public function get_target(): \core_analytics\local\target\base { + return $this->target; + } - list($analysables, $processedanalysables) = $this->get_sorted_analysables($includetarget); + /** + * Getter for time splittings. + * + * @return \core_analytics\local\time_splitting\base + */ + public function get_timesplittings(): array { + return $this->timesplittings; + } - $inittime = time(); - foreach ($analysables as $key => $analysable) { - - $files = $this->process_analysable($analysable, $includetarget); - - // Later we will need to aggregate data by time splitting method. - foreach ($files as $timesplittingid => $file) { - $filesbytimesplitting[$timesplittingid][] = $file; - } - - $this->update_analysable_analysed_time($processedanalysables, $analysable->get_id(), $includetarget); - - // Apply time limit. - if (!$this->options['evaluation']) { - $timespent = time() - $inittime; - if ($modeltimelimit <= $timespent) { - break; - } - } - - unset($analysables[$key]); - } - - if ($this->options['evaluation'] === false) { - // Look for previous training and prediction files we generated and couldn't be used - // by machine learning backends because they weren't big enough. - - $pendingfiles = \core_analytics\dataset_manager::get_pending_files($this->modelid, $includetarget, - array_keys($filesbytimesplitting)); - foreach ($pendingfiles as $timesplittingid => $files) { - foreach ($files as $file) { - $filesbytimesplitting[$timesplittingid][] = $file; - } - } - } - - // We join the datasets by time splitting method. - $timesplittingfiles = $this->merge_analysable_files($filesbytimesplitting, $includetarget); - - if (!empty($pendingfiles)) { - // We must remove them now as they are already part of another dataset. - foreach ($pendingfiles as $timesplittingid => $files) { - foreach ($files as $file) { - $file->delete(); - } - } - } - - return $timesplittingfiles; + /** + * Getter for indicators. + * + * @return \core_analytics\local\indicator\base + */ + public function get_indicators(): array { + return $this->indicators; } /** @@ -252,19 +249,39 @@ abstract class base { /** * Returns labelled data (training and evaluation). * - * @return array + * @return \stored_file[] */ public function get_labelled_data() { - return $this->get_analysable_data(true); + // Delegates all processing to the analysis. + $result = new \core_analytics\local\analysis\result_file($this->get_modelid(), true, $this->get_options()); + $analysis = new \core_analytics\analysis($this, true, $result); + $analysis->run(); + return $result->get(); } /** * Returns unlabelled data (prediction). * - * @return array + * @return \stored_file[] */ public function get_unlabelled_data() { - return $this->get_analysable_data(false); + // Delegates all processing to the analysis. + $result = new \core_analytics\local\analysis\result_file($this->get_modelid(), false, $this->get_options()); + $analysis = new \core_analytics\analysis($this, false, $result); + $analysis->run(); + return $result->get(); + } + + /** + * Returns indicator calculations as an array. + * @return array + */ + public function get_static_data() { + // Delegates all processing to the analysis. + $result = new \core_analytics\local\analysis\result_array($this->get_modelid(), false, $this->get_options()); + $analysis = new \core_analytics\analysis($this, false, $result); + $analysis->run(); + return $result->get(); } /** @@ -284,36 +301,6 @@ abstract class base { } } - /** - * Merges analysable dataset files into 1. - * - * @param array $filesbytimesplitting - * @param bool $includetarget - * @return \stored_file[] - */ - protected function merge_analysable_files($filesbytimesplitting, $includetarget) { - - $timesplittingfiles = array(); - foreach ($filesbytimesplitting as $timesplittingid => $files) { - - if ($this->options['evaluation'] === true) { - // Delete the previous copy. Only when evaluating. - \core_analytics\dataset_manager::delete_previous_evaluation_file($this->modelid, $timesplittingid); - } - - // Merge all course files into one. - if ($includetarget) { - $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA; - } else { - $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA; - } - $timesplittingfiles[$timesplittingid] = \core_analytics\dataset_manager::merge_datasets($files, - $this->modelid, $timesplittingid, $filearea, $this->options['evaluation']); - } - - return $timesplittingfiles; - } - /** * Checks that this analyser satisfies the provided indicator requirements. * @@ -338,83 +325,6 @@ abstract class base { return $missingrequired; } - /** - * Processes an analysable - * - * This method returns the general analysable status, an array of files by time splitting method and - * an error message if there is any problem. - * - * @param \core_analytics\analysable $analysable - * @param bool $includetarget - * @return \stored_file[] Files by time splitting method - */ - public function process_analysable($analysable, $includetarget) { - - // Default returns. - $files = array(); - $message = null; - - // Target instances scope is per-analysable (it can't be lower as calculations run once per - // analysable, not time splitting method nor time range). - $this->analysabletarget = call_user_func(array($this->target, 'instance')); - - // We need to check that the analysable is valid for the target even if we don't include targets - // as we still need to discard invalid analysables for the target. - $result = $this->analysabletarget->is_valid_analysable($analysable, $includetarget); - if ($result !== true) { - $a = new \stdClass(); - $a->analysableid = $analysable->get_name(); - $a->result = $result; - $this->add_log(get_string('analysablenotvalidfortarget', 'analytics', $a)); - return array(); - } - - // Process all provided time splitting methods. - $results = array(); - foreach ($this->timesplittings as $timesplitting) { - - // For evaluation purposes we don't need to be that strict about how updated the data is, - // if this analyser was analysed less that 1 week ago we skip generating a new one. This - // helps scale the evaluation process as sites with tons of courses may a lot of time to - // complete an evaluation. - if (!empty($this->options['evaluation']) && !empty($this->options['reuseprevanalysed'])) { - - $previousanalysis = \core_analytics\dataset_manager::get_evaluation_analysable_file($this->modelid, - $analysable->get_id(), $timesplitting->get_id()); - // 1 week is a partly random time interval, no need to worry about DST. - $boundary = time() - WEEKSECS; - if ($previousanalysis && $previousanalysis->get_timecreated() > $boundary) { - // Recover the previous analysed file and avoid generating a new one. - - // Don't bother filling a result object as it is only useful when there are no files generated. - $files[$timesplitting->get_id()] = $previousanalysis; - continue; - } - } - - $result = $this->process_time_splitting($timesplitting, $analysable, $includetarget); - - if (!empty($result->file)) { - $files[$timesplitting->get_id()] = $result->file; - } - $results[] = $result; - } - - if (empty($files)) { - $errors = array(); - foreach ($results as $timesplittingid => $result) { - $errors[] = $timesplittingid . ': ' . $result->message; - } - - $a = new \stdClass(); - $a->analysableid = $analysable->get_name(); - $a->errors = implode(', ', $errors); - $this->add_log(get_string('analysablenotused', 'analytics', $a)); - } - - return $files; - } - /** * Adds a register to the analysis log. * @@ -482,427 +392,83 @@ abstract class base { } /** - * Processes the analysable samples using the provided time splitting method. + * Do this analyser's analysables have 1 single sample each? * - * @param \core_analytics\local\time_splitting\base $timesplitting - * @param \core_analytics\analysable $analysable - * @param bool $includetarget - * @return \stdClass Results object. + * Overwrite and return true if your analysables only have + * one sample. The insights generated by models using this + * analyser will then include the suggested actions in the + * notification. + * + * @return bool */ - protected function process_time_splitting($timesplitting, $analysable, $includetarget = false) { - - $result = new \stdClass(); - - if (!$timesplitting->is_valid_analysable($analysable)) { - $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; - $result->message = get_string('invalidanalysablefortimesplitting', 'analytics', - $timesplitting->get_name()); - return $result; - } - $timesplitting->set_analysable($analysable); - - if (CLI_SCRIPT && !PHPUNIT_TEST) { - mtrace('Analysing id "' . $analysable->get_id() . '" with "' . $timesplitting->get_name() . - '" time splitting method...'); - } - - // What is a sample is defined by the analyser, it can be an enrolment, a course, a user, a question - // attempt... it is on what we will base indicators calculations. - list($sampleids, $samplesdata) = $this->get_all_samples($analysable); - - if (count($sampleids) === 0) { - $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; - $result->message = get_string('nodata', 'analytics'); - return $result; - } - - if ($includetarget) { - // All ranges are used when we are calculating data for training. - $ranges = $timesplitting->get_all_ranges(); - } else { - // The latest range that has not yet been used for prediction (it depends on the time range where we are right now). - $ranges = $this->get_most_recent_prediction_range($timesplitting); - } - - // There is no need to keep track of the evaluated samples and ranges as we always evaluate the whole dataset. - if ($this->options['evaluation'] === false) { - - if (empty($ranges)) { - $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; - $result->message = get_string('noranges', 'analytics'); - return $result; - } - - // We skip all samples that are already part of a training dataset, even if they have not been used for prediction. - $this->filter_out_train_samples($sampleids, $timesplitting); - - if (count($sampleids) === 0) { - $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; - $result->message = get_string('nonewdata', 'analytics'); - return $result; - } - - // Only when processing data for predictions. - if (!$includetarget) { - // We also filter out samples and ranges that have already been used for predictions. - $this->filter_out_prediction_samples_and_ranges($sampleids, $ranges, $timesplitting); - } - - if (count($sampleids) === 0) { - $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; - $result->message = get_string('nonewdata', 'analytics'); - return $result; - } - - if (count($ranges) === 0) { - $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; - $result->message = get_string('nonewranges', 'analytics'); - return $result; - } - } - - if (!empty($includetarget)) { - $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA; - } else { - $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA; - } - $dataset = new \core_analytics\dataset_manager($this->modelid, $analysable->get_id(), $timesplitting->get_id(), - $filearea, $this->options['evaluation']); - - // Flag the model + analysable + timesplitting as being analysed (prevent concurrent executions). - if (!$dataset->init_process()) { - // If this model + analysable + timesplitting combination is being analysed we skip this process. - $result->status = \core_analytics\model::NO_DATASET; - $result->message = get_string('analysisinprogress', 'analytics'); - return $result; - } - - try { - // Remove samples the target consider invalid. - $this->analysabletarget->add_sample_data($samplesdata); - $this->analysabletarget->filter_out_invalid_samples($sampleids, $analysable, $includetarget); - } catch (\Throwable $e) { - $dataset->close_process(); - throw $e; - } - - if (!$sampleids) { - $result->status = \core_analytics\model::NO_DATASET; - $result->message = get_string('novalidsamples', 'analytics'); - $dataset->close_process(); - return $result; - } - - try { - foreach ($this->indicators as $key => $indicator) { - // The analyser attaches the main entities the sample depends on and are provided to the - // indicator to calculate the sample. - $this->indicators[$key]->add_sample_data($samplesdata); - } - - // Here we start the memory intensive process that will last until $data var is - // unset (until the method is finished basically). - if ($includetarget) { - $data = $timesplitting->calculate($sampleids, $this->get_samples_origin(), $this->indicators, $ranges, - $this->analysabletarget); - } else { - $data = $timesplitting->calculate($sampleids, $this->get_samples_origin(), $this->indicators, $ranges); - } - } catch (\Throwable $e) { - $dataset->close_process(); - throw $e; - } - - if (!$data) { - $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD; - $result->message = get_string('novaliddata', 'analytics'); - $dataset->close_process(); - return $result; - } - - try { - // Add extra metadata. - $this->add_model_metadata($data); - - // Write all calculated data to a file. - $file = $dataset->store($data); - } catch (\Throwable $e) { - $dataset->close_process(); - throw $e; - } - - // Flag the model + analysable + timesplitting as analysed. - $dataset->close_process(); - - // No need to keep track of analysed stuff when evaluating. - if ($this->options['evaluation'] === false) { - // Save the samples that have been already analysed so they are not analysed again in future. - - if ($includetarget) { - $this->save_train_samples($sampleids, $timesplitting, $file); - } else { - $this->save_prediction_samples($sampleids, $ranges, $timesplitting); - } - } - - $result->status = \core_analytics\model::OK; - $result->message = get_string('successfullyanalysed', 'analytics'); - $result->file = $file; - return $result; + public static function one_sample_per_analysable() { + return false; } /** - * Returns the most recent range that can be used to predict. + * Get the sql of a default implementation of the iterator. * - * @param \core_analytics\local\time_splitting\base $timesplitting - * @return array + * This method only works for analysers that return analysable elements which ids map to a context instance ids. + * + * @param string $tablename The name of the table + * @param int $contextlevel The context level of the analysable + * @param string|null $action + * @param string|null $tablealias The table alias + * @return array [0] => sql and [1] => params array */ - protected function get_most_recent_prediction_range($timesplitting) { + protected function get_iterator_sql(string $tablename, int $contextlevel, ?string $action = null, ?string $tablealias = null) { - $now = time(); - $ranges = $timesplitting->get_all_ranges(); - - // Opposite order as we are interested in the last range that can be used for prediction. - krsort($ranges); - - // We already provided the analysable to the time splitting method, there is no need to feed it back. - foreach ($ranges as $rangeindex => $range) { - if ($timesplitting->ready_to_predict($range)) { - // We need to maintain the same indexes. - return array($rangeindex => $range); - } + if (!$tablealias) { + $tablealias = 'analysable'; } - return array(); + $params = ['contextlevel' => $contextlevel, 'modelid' => $this->get_modelid()]; + $select = $tablealias . '.*, ' . \context_helper::get_preload_record_columns_sql('ctx'); + + // We add the action filter on ON instead of on WHERE because otherwise records are not returned if there are existing + // records for another action or model. + $usedanalysablesjoin = ' LEFT JOIN {analytics_used_analysables} aua ON ' . $tablealias . '.id = aua.analysableid AND ' . + '(aua.modelid = :modelid OR aua.modelid IS NULL)'; + + if ($action) { + $usedanalysablesjoin .= " AND aua.action = :action"; + $params = $params + ['action' => $action]; + } + + // Adding the 1 = 1 just to have the WHERE part so that all further conditions added by callers can be + // appended to $sql with and ' AND'. + $sql = 'SELECT ' . $select . ' + FROM {' . $tablename . '} ' . $tablealias . ' + ' . $usedanalysablesjoin . ' + JOIN {context} ctx ON (ctx.contextlevel = :contextlevel AND ctx.instanceid = ' . $tablealias . '.id) + WHERE 1 = 1'; + + return [$sql, $params]; } /** - * Filters out samples that have already been used for training. + * Returns the order by clause. * - * @param int[] $sampleids - * @param \core_analytics\local\time_splitting\base $timesplitting + * @param string|null $fieldname The field name + * @param string $order 'ASC' or 'DESC' + * @param string|null $tablealias The table alias of the field + * @return string */ - protected function filter_out_train_samples(&$sampleids, $timesplitting) { - global $DB; + protected function order_sql(?string $fieldname = null, string $order = 'ASC', ?string $tablealias = null) { - $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(), - 'timesplitting' => $timesplitting->get_id()); - - $trainingsamples = $DB->get_records('analytics_train_samples', $params); - - // Skip each file trained samples. - foreach ($trainingsamples as $trainingfile) { - - $usedsamples = json_decode($trainingfile->sampleids, true); - - if (!empty($usedsamples)) { - // Reset $sampleids to $sampleids minus this file's $usedsamples. - $sampleids = array_diff_key($sampleids, $usedsamples); - } - } - } - - /** - * Filters out samples that have already been used for prediction. - * - * @param int[] $sampleids - * @param array $ranges - * @param \core_analytics\local\time_splitting\base $timesplitting - */ - protected function filter_out_prediction_samples_and_ranges(&$sampleids, &$ranges, $timesplitting) { - global $DB; - - if (count($ranges) > 1) { - throw new \coding_exception('$ranges argument should only contain one range'); + if (!$tablealias) { + $tablealias = 'analysable'; } - $rangeindex = key($ranges); - - $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(), - 'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex); - $predictedrange = $DB->get_record('analytics_predict_samples', $params); - - if (!$predictedrange) { - // Nothing to filter out. - return; + if ($order != 'ASC' && $order != 'DESC') { + throw new \coding_exception('The order can only be ASC or DESC'); } - $predictedrange->sampleids = json_decode($predictedrange->sampleids, true); - $missingsamples = array_diff_key($sampleids, $predictedrange->sampleids); - if (count($missingsamples) === 0) { - // All samples already calculated. - unset($ranges[$rangeindex]); - return; + $ordersql = ' ORDER BY (CASE WHEN aua.timeanalysed IS NULL THEN 0 ELSE aua.timeanalysed END) ASC'; + if ($fieldname) { + $ordersql .= ', ' . $tablealias . '.' . $fieldname .' ' . $order; } - // Replace the list of samples by the one excluding samples that already got predictions at this range. - $sampleids = $missingsamples; - } - - /** - * Saves samples that have just been used for training. - * - * @param int[] $sampleids - * @param \core_analytics\local\time_splitting\base $timesplitting - * @param \stored_file $file - * @return void - */ - protected function save_train_samples($sampleids, $timesplitting, $file) { - global $DB; - - $trainingsamples = new \stdClass(); - $trainingsamples->modelid = $this->modelid; - $trainingsamples->analysableid = $timesplitting->get_analysable()->get_id(); - $trainingsamples->timesplitting = $timesplitting->get_id(); - $trainingsamples->fileid = $file->get_id(); - - $trainingsamples->sampleids = json_encode($sampleids); - $trainingsamples->timecreated = time(); - - $DB->insert_record('analytics_train_samples', $trainingsamples); - } - - /** - * Saves samples that have just been used for prediction. - * - * @param int[] $sampleids - * @param array $ranges - * @param \core_analytics\local\time_splitting\base $timesplitting - * @return void - */ - protected function save_prediction_samples($sampleids, $ranges, $timesplitting) { - global $DB; - - if (count($ranges) > 1) { - throw new \coding_exception('$ranges argument should only contain one range'); - } - - $rangeindex = key($ranges); - - $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(), - 'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex); - if ($predictionrange = $DB->get_record('analytics_predict_samples', $params)) { - // Append the new samples used for prediction. - $prevsamples = json_decode($predictionrange->sampleids, true); - $predictionrange->sampleids = json_encode($prevsamples + $sampleids); - $predictionrange->timemodified = time(); - $DB->update_record('analytics_predict_samples', $predictionrange); - } else { - $predictionrange = (object)$params; - $predictionrange->sampleids = json_encode($sampleids); - $predictionrange->timecreated = time(); - $predictionrange->timemodified = $predictionrange->timecreated; - $DB->insert_record('analytics_predict_samples', $predictionrange); - } - } - - /** - * Adds target metadata to the dataset. - * - * @param array $data - * @return void - */ - protected function add_model_metadata(&$data) { - global $CFG; - - $metadata = array( - 'moodleversion' => $CFG->version, - 'targetcolumn' => $this->analysabletarget->get_id() - ); - if ($this->analysabletarget->is_linear()) { - $metadata['targettype'] = 'linear'; - $metadata['targetmin'] = $this->analysabletarget::get_min_value(); - $metadata['targetmax'] = $this->analysabletarget::get_max_value(); - } else { - $metadata['targettype'] = 'discrete'; - $metadata['targetclasses'] = json_encode($this->analysabletarget::get_classes()); - } - - foreach ($metadata as $varname => $value) { - $data[0][] = $varname; - $data[1][] = $value; - } - } - - /** - * Returns the list of analysables sorted in processing priority order. - * - * It will first return analysables that have never been analysed before - * and it will continue with the ones we have already seen by timeanalysed DESC - * order. - * - * @param bool $includetarget - * @return array(0 => \core_analytics\analysable[], 1 => \stdClass[]) - */ - protected function get_sorted_analysables($includetarget) { - - $analysables = $this->get_analysables(); - - // Get the list of analysables that have been already processed. - $processedanalysables = $this->get_processed_analysables($includetarget); - - // We want to start processing analysables we have not yet processed and later continue - // with analysables that we already processed. - $unseen = array_diff_key($analysables, $processedanalysables); - - // Var $processed first as we want to respect its timeanalysed DESC order so analysables that - // have recently been processed are on the bottom of the stack. - $seen = array_intersect_key($processedanalysables, $analysables); - array_walk($seen, function(&$value, $analysableid) use ($analysables) { - // We replace the analytics_used_analysables record by the analysable object. - $value = $analysables[$analysableid]; - }); - - return array($unseen + $seen, $processedanalysables); - } - - /** - * Get analysables that have been already processed. - * - * @param bool $includetarget - * @return \stdClass[] - */ - protected function get_processed_analysables($includetarget) { - global $DB; - - $params = array('modelid' => $this->modelid); - $params['action'] = ($includetarget) ? 'training' : 'prediction'; - $select = 'modelid = :modelid and action = :action'; - - // Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid). - return $DB->get_records_select('analytics_used_analysables', $select, - $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey'); - } - - /** - * Updates the analysable analysis time. - * - * @param array $processedanalysables - * @param int $analysableid - * @param bool $includetarget - * @return null - */ - protected function update_analysable_analysed_time($processedanalysables, $analysableid, $includetarget) { - global $DB; - - if (!empty($processedanalysables[$analysableid])) { - $obj = $processedanalysables[$analysableid]; - - $obj->id = $obj->primarykey; - unset($obj->primarykey); - - $obj->timeanalysed = time(); - $DB->update_record('analytics_used_analysables', $obj); - - } else { - - $obj = new \stdClass(); - $obj->modelid = $this->modelid; - $obj->action = ($includetarget) ? 'training' : 'prediction'; - $obj->analysableid = $analysableid; - $obj->timeanalysed = time(); - - $DB->insert_record('analytics_used_analysables', $obj); - } + return $ordersql; } } diff --git a/analytics/classes/local/analyser/by_course.php b/analytics/classes/local/analyser/by_course.php index bbda69a1abf..99e70c1e6a1 100644 --- a/analytics/classes/local/analyser/by_course.php +++ b/analytics/classes/local/analyser/by_course.php @@ -38,34 +38,43 @@ abstract class by_course extends base { /** * Return the list of courses to analyse. * - * @return \core_analytics\course[] + * @param string|null $action 'prediction', 'training' or null if no specific action needed. + * @return \Iterator */ - public function get_analysables() { + public function get_analysables_iterator(?string $action = null) { + global $DB; - // Default to all system courses. + list($sql, $params) = $this->get_iterator_sql('course', CONTEXT_COURSE, $action, 'c'); + + // This will be updated to filter by context as part of MDL-64739. if (!empty($this->options['filter'])) { $courses = array(); foreach ($this->options['filter'] as $courseid) { $courses[$courseid] = new \stdClass(); $courses[$courseid]->id = $courseid; } - } else { - // Iterate through all potentially valid courses. - $courses = get_courses('all', 'c.sortorder ASC', 'c.id'); - } - unset($courses[SITEID]); - $analysables = array(); - foreach ($courses as $course) { - // Skip the frontpage course. - $analysable = \core_analytics\course::instance($course->id); - $analysables[$analysable->get_id()] = $analysable; + list($coursesql, $courseparams) = $DB->get_in_or_equal($courses, SQL_PARAMS_NAMED); + $sql .= " AND c.id IN $coursesql"; + $params = $params + $courseparams; } - if (empty($analysables)) { - $this->log[] = get_string('nocourses', 'analytics'); + $ordersql = $this->order_sql('sortorder', 'ASC', 'c'); + + $recordset = $DB->get_recordset_sql($sql . $ordersql, $params); + + if (!$recordset->valid()) { + $this->add_log(get_string('nocourses', 'analytics')); + return new \ArrayIterator([]); } - return $analysables; + return new \core\dml\recordset_walk($recordset, function($record) { + + if ($record->id == SITEID) { + return false; + } + $context = \context_helper::preload_from_record($record); + return \core_analytics\course::instance($record, $context); + }); } -} +} \ No newline at end of file diff --git a/analytics/classes/local/analyser/sitewide.php b/analytics/classes/local/analyser/sitewide.php index 075b930e2fb..da6ea71db82 100644 --- a/analytics/classes/local/analyser/sitewide.php +++ b/analytics/classes/local/analyser/sitewide.php @@ -36,12 +36,13 @@ defined('MOODLE_INTERNAL') || die(); abstract class sitewide extends base { /** - * Returns one single analysable element, the site. + * Return the list of analysables to analyse. * - * @return \core_analytics\analysable[] + * @param string|null $action 'prediction', 'training' or null if no specific action needed. + * @return \Iterator */ - public function get_analysables() { - $analysable = new \core_analytics\site(); - return array(SYSCONTEXTID => $analysable); + public function get_analysables_iterator(?string $action = null) { + // We can safely ignore $action as we have 1 single analysable element in this analyser. + return new \ArrayIterator([new \core_analytics\site()]); } } diff --git a/analytics/classes/local/analysis/result.php b/analytics/classes/local/analysis/result.php new file mode 100644 index 00000000000..d82926117b4 --- /dev/null +++ b/analytics/classes/local/analysis/result.php @@ -0,0 +1,102 @@ +. + +/** + * Keeps track of the analysis results. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics\local\analysis; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Keeps track of the analysis results. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +abstract class result { + + /** + * @var int + */ + protected $modelid; + + /** + * @var bool + */ + protected $includetarget; + + /** + * @var array Analysis options + */ + protected $options; + + /** + * Stores analysis data at instance level. + * @param int $modelid + * @param bool $includetarget + * @param array $options + */ + public function __construct(int $modelid, bool $includetarget, array $options) { + $this->modelid = $modelid; + $this->includetarget = $includetarget; + $this->options = $options; + } + + /** + * Retrieves cached results during evaluation. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\analysable $analysable + * @return mixed It can be in whatever format the result uses. + */ + public function retrieve_cached_result(\core_analytics\local\time_splitting\base $timesplitting, + \core_analytics\analysable $analysable) { + return false; + } + + /** + * Stores the analysis results. + * + * @param array $results + * @return bool True if anything was successfully analysed + */ + abstract public function add_analysable_results(array $results): bool; + + /** + * Formats the result. + * + * @param array $data + * @param \core_analytics\local\target\base $target + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\analysable $analysable + * @return mixed It can be in whatever format the result uses + */ + abstract public function format_result(array $data, \core_analytics\local\target\base $target, + \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable); + + /** + * Returns the results of the analysis. + * @return array + */ + abstract public function get(): array; +} \ No newline at end of file diff --git a/analytics/classes/local/analysis/result_array.php b/analytics/classes/local/analysis/result_array.php new file mode 100644 index 00000000000..0a8e21ee3e1 --- /dev/null +++ b/analytics/classes/local/analysis/result_array.php @@ -0,0 +1,100 @@ +. + +/** + * Keeps track of the analysis results by storing the results in an array. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics\local\analysis; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Keeps track of the analysis results by storing the results in an array. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class result_array extends result { + + /** + * Stores the analysis results by time-splitting method. + * @var array + */ + private $resultsbytimesplitting = []; + + /** + * Stores the analysis results. + * @param array $results + * @return bool True if anything was successfully analysed + */ + public function add_analysable_results(array $results): bool { + + $any = false; + + // Process all provided time splitting methods. + foreach ($results as $timesplittingid => $result) { + if (!empty($result->result)) { + if (empty($this->resultsbytimesplitting[$timesplittingid])) { + $this->resultsbytimesplitting[$timesplittingid] = []; + } + $this->resultsbytimesplitting[$timesplittingid] += $result->result; + $any = true; + } + } + if (empty($any)) { + return false; + } + return true; + } + + /** + * Formats the result. + * + * @param array $data + * @param \core_analytics\local\target\base $target + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\analysable $analysable + * @return mixed The data as it comes + */ + public function format_result(array $data, \core_analytics\local\target\base $target, + \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable) { + return $data; + } + + /** + * Returns the results of the analysis. + * @return array + */ + public function get(): array { + + // We join the datasets by time splitting method. + $timesplittingresults = array(); + foreach ($this->resultsbytimesplitting as $timesplittingid => $results) { + if (empty($timesplittingresults[$timesplittingid])) { + $timesplittingresults[$timesplittingid] = []; + } + $timesplittingresults[$timesplittingid] += $results; + } + + return $timesplittingresults; + } +} \ No newline at end of file diff --git a/analytics/classes/local/analysis/result_file.php b/analytics/classes/local/analysis/result_file.php new file mode 100644 index 00000000000..7a6b61e47ac --- /dev/null +++ b/analytics/classes/local/analysis/result_file.php @@ -0,0 +1,228 @@ +. + +/** + * Keeps track of the analysis results by storing the results in files. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics\local\analysis; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Keeps track of the analysis results by storing the results in files. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class result_file extends result { + + /** + * Stores the analysis results by time-splitting method. + * @var array + */ + private $filesbytimesplitting = []; + + /** + * Stores the analysis results. + * @param array $results + * @return bool True if anything was successfully analysed + */ + public function add_analysable_results(array $results): bool { + + $any = false; + + // Process all provided time splitting methods. + foreach ($results as $timesplittingid => $result) { + if (!empty($result->result)) { + $this->filesbytimesplitting[$timesplittingid][] = $result->result; + $any = true; + } + } + + if (empty($any)) { + return false; + } + return true; + } + + /** + * Retrieves cached results during evaluation. + * + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\analysable $analysable + * @return mixed A \stored_file in this case. + */ + public function retrieve_cached_result(\core_analytics\local\time_splitting\base $timesplitting, + \core_analytics\analysable $analysable) { + + // For evaluation purposes we don't need to be that strict about how updated the data is, + // if this analyser was analysed less that 1 week ago we skip generating a new one. This + // helps scale the evaluation process as sites with tons of courses may need a lot of time to + // complete an evaluation. + if (!empty($options['evaluation']) && !empty($options['reuseprevanalysed'])) { + + $previousanalysis = \core_analytics\dataset_manager::get_evaluation_analysable_file($this->analyser->get_modelid(), + $analysable->get_id(), $timesplitting->get_id()); + // 1 week is a partly random time interval, no need to worry about DST. + $boundary = time() - WEEKSECS; + if ($previousanalysis && $previousanalysis->get_timecreated() > $boundary) { + // Recover the previous analysed file and avoid generating a new one. + return $previousanalysis; + } + } + + return false; + } + + /** + * Formats the result. + * + * @param array $data + * @param \core_analytics\local\target\base $target + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\analysable $analysable + * @return mixed A \stored_file in this case + */ + public function format_result(array $data, \core_analytics\local\target\base $target, + \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable) { + + if (!empty($this->includetarget)) { + $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA; + } else { + $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA; + } + $dataset = new \core_analytics\dataset_manager($this->modelid, $analysable->get_id(), + $timesplitting->get_id(), $filearea, $this->options['evaluation']); + + // Add extra metadata. + $this->add_model_metadata($data, $timesplitting, $target); + + // Write all calculated data to a file. + if (!$result = $dataset->store($data)) { + return false; + } + + return $result; + } + + /** + * Returns the results of the analysis. + * @return array + */ + public function get(): array { + + if ($this->options['evaluation'] === false) { + // Look for previous training and prediction files we generated and couldn't be used + // by machine learning backends because they weren't big enough. + + $pendingfiles = \core_analytics\dataset_manager::get_pending_files($this->modelid, $this->includetarget, + array_keys($this->filesbytimesplitting)); + foreach ($pendingfiles as $timesplittingid => $files) { + foreach ($files as $file) { + $this->filesbytimesplitting[$timesplittingid][] = $file; + } + } + } + + // We join the datasets by time splitting method. + $timesplittingfiles = array(); + foreach ($this->filesbytimesplitting as $timesplittingid => $files) { + + if ($this->options['evaluation'] === true) { + // Delete the previous copy. Only when evaluating. + \core_analytics\dataset_manager::delete_previous_evaluation_file($this->modelid, $timesplittingid); + } + + // Merge all course files into one. + if ($this->includetarget) { + $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA; + } else { + $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA; + } + $timesplittingfiles[$timesplittingid] = \core_analytics\dataset_manager::merge_datasets($files, + $this->modelid, $timesplittingid, $filearea, $this->options['evaluation']); + } + + if (!empty($pendingfiles)) { + // We must remove them now as they are already part of another dataset. + foreach ($pendingfiles as $timesplittingid => $files) { + foreach ($files as $file) { + $file->delete(); + } + } + } + + return $timesplittingfiles; + } + + /** + * Adds target metadata to the dataset. + * + * The final dataset document will look like this: + * ---------------------------------------------------- + * metadata1,metadata2,metadata3,..... + * value1, value2, value3,..... + * + * header1,header2,header3,header4,..... + * stud1value1,stud1value2,stud1value3,stud1value4,..... + * stud2value1,stud2value2,stud2value3,stud2value4,..... + * ..... + * ---------------------------------------------------- + * + * @param array $data + * @param \core_analytics\local\time_splitting\base $timesplitting + * @param \core_analytics\local\target\base $target + * @return null + */ + private function add_model_metadata(array &$data, \core_analytics\local\time_splitting\base $timesplitting, + \core_analytics\local\target\base $target) { + global $CFG; + + // If no target the first column is the sampleid, if target the last column is the target. + // This will need to be updated when we support unsupervised learning models. + $metadata = array( + 'timesplitting' => $timesplitting->get_id(), + 'nfeatures' => count(current($data)) - 1, + 'moodleversion' => $CFG->version, + 'targetcolumn' => $target->get_id() + ); + if ($target->is_linear()) { + $metadata['targettype'] = 'linear'; + $metadata['targetmin'] = $target::get_min_value(); + $metadata['targetmax'] = $target::get_max_value(); + } else { + $metadata['targettype'] = 'discrete'; + $metadata['targetclasses'] = json_encode($target::get_classes()); + } + + // The first 2 samples will be used to store metadata about the dataset. + $metadatacolumns = []; + $metadatavalues = []; + foreach ($metadata as $key => $value) { + $metadatacolumns[] = $key; + $metadatavalues[] = $value; + } + + // This will also reset samples' dataset keys. + array_unshift($data, $metadatacolumns, $metadatavalues); + } +} diff --git a/analytics/classes/local/target/base.php b/analytics/classes/local/target/base.php index 142518ec3e2..0ea5e8b8e29 100644 --- a/analytics/classes/local/target/base.php +++ b/analytics/classes/local/target/base.php @@ -104,14 +104,29 @@ abstract class base extends \core_analytics\calculable { return false; } + /** + * Update the last analysis time on analysable processed or always. + * + * If you overwrite this method to return false the last analysis time + * will only be recorded in DB when the element successfully analysed. You can + * safely return false for lightweight targets. + * + * @return bool + */ + public function always_update_analysis_time(): bool { + return true; + } + /** * Suggested actions for a user. * * @param \core_analytics\prediction $prediction * @param bool $includedetailsaction + * @param bool $isinsightuser * @return \core_analytics\prediction_action[] */ - public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false) { + public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false, + $isinsightuser = false) { global $PAGE; $predictionid = $prediction->get_prediction_data()->id; @@ -122,12 +137,12 @@ abstract class base extends \core_analytics\calculable { if ($includedetailsaction) { - $predictionurl = new \moodle_url('/report/insights/prediction.php', - array('id' => $predictionid)); + $predictionurl = new \moodle_url('/report/insights/prediction.php', array('id' => $predictionid)); + $detailstext = $this->get_view_details_text(); $actions[] = new \core_analytics\prediction_action(\core_analytics\prediction::ACTION_PREDICTION_DETAILS, $prediction, - $predictionurl, new \pix_icon('t/preview', get_string('viewprediction', 'analytics')), - get_string('viewprediction', 'analytics')); + $predictionurl, new \pix_icon('t/preview', $detailstext), + $detailstext); } // Flag as fixed / solved. @@ -151,6 +166,25 @@ abstract class base extends \core_analytics\calculable { return $actions; } + /** + * Returns the view details link text. + * @return string + */ + private function get_view_details_text() { + if ($this->based_on_assumptions()) { + $analyserclass = $this->get_analyser_class(); + if ($analyserclass::one_sample_per_analysable()) { + $detailstext = get_string('viewinsightdetails', 'analytics'); + } else { + $detailstext = get_string('viewdetails', 'analytics'); + } + } else { + $detailstext = get_string('viewprediction', 'analytics'); + } + + return $detailstext; + } + /** * Callback to execute once a prediction has been returned from the predictions processor. * @@ -173,61 +207,28 @@ abstract class base extends \core_analytics\calculable { * * @param int $modelid * @param \context[] $samplecontexts + * @param \core_analytics\prediction[] $predictions * @return void */ - public function generate_insight_notifications($modelid, $samplecontexts) { - - foreach ($samplecontexts as $context) { - - $insightinfo = new \stdClass(); - $insightinfo->insightname = $this->get_name(); - $insightinfo->contextname = $context->get_context_name(); - $subject = get_string('insightmessagesubject', 'analytics', $insightinfo); - - $users = $this->get_insights_users($context); - - if (!$coursecontext = $context->get_course_context(false)) { - $coursecontext = \context_course::instance(SITEID); - } - - foreach ($users as $user) { - - $message = new \core\message\message(); - $message->component = 'moodle'; - $message->name = 'insights'; - - $message->userfrom = \core_user::get_noreply_user(); - $message->userto = $user; - - $insighturl = new \moodle_url('/report/insights/insights.php?modelid=' . $modelid . '&contextid=' . $context->id); - $message->subject = $subject; - // Same than the subject. - $message->contexturlname = $message->subject; - $message->courseid = $coursecontext->instanceid; - - $message->fullmessage = get_string('insightinfomessage', 'analytics', $insighturl->out(false)); - $message->fullmessageformat = FORMAT_PLAIN; - $message->fullmessagehtml = get_string('insightinfomessagehtml', 'analytics', $insighturl->out()); - $message->smallmessage = get_string('insightinfomessage', 'analytics', $insighturl->out(false)); - $message->contexturl = $insighturl->out(false); - - message_send($message); - } - } - + public function generate_insight_notifications($modelid, $samplecontexts, array $predictions = []) { + // Delegate the processing of insights to the insights_generator. + $insightsgenerator = new \core_analytics\insights_generator($modelid, $this); + $insightsgenerator->generate($samplecontexts, $predictions); } /** * Returns the list of users that will receive insights notifications. * * Feel free to overwrite if you need to but keep in mind that moodle/analytics:listinsights - * capability is required to access the list of insights. + * or moodle/analytics:listowninsights capability is required to access the list of insights. * * @param \context $context * @return array */ - protected function get_insights_users(\context $context) { - if ($context->contextlevel >= CONTEXT_COURSE) { + public function get_insights_users(\context $context) { + if ($context->contextlevel === CONTEXT_USER) { + $users = [$context->instanceid => \core_user::get_user($context->instanceid)]; + } else if ($context->contextlevel >= CONTEXT_COURSE) { // At course level or below only enrolled users although this is not ideal for // teachers assigned at category level. $users = get_enrolled_users($context, 'moodle/analytics:listinsights'); @@ -237,6 +238,30 @@ abstract class base extends \core_analytics\calculable { return $users; } + /** + * URL to the insight. + * + * @param int $modelid + * @param \context $context + * @return \moodle_url + */ + public function get_insight_context_url($modelid, $context) { + return new \moodle_url('/report/insights/insights.php?modelid=' . $modelid . '&contextid=' . $context->id); + } + + /** + * The insight notification subject. + * + * This is just a default message, you should overwrite it for a custom insight message. + * + * @param int $modelid + * @param \context $context + * @return string + */ + public function get_insight_subject(int $modelid, \context $context) { + return get_string('insightmessagesubject', 'analytics', $context->get_context_name()); + } + /** * Returns an instance of the child class. * diff --git a/analytics/classes/local/target/binary.php b/analytics/classes/local/target/binary.php index f8c3b04dc1d..97e9a3f9bb5 100644 --- a/analytics/classes/local/target/binary.php +++ b/analytics/classes/local/target/binary.php @@ -60,7 +60,7 @@ abstract class binary extends discrete { * * @return array */ - protected function ignored_predicted_classes() { + public function ignored_predicted_classes() { // Zero-value class is usually ignored in binary classifiers. return array(0); } diff --git a/analytics/classes/local/target/discrete.php b/analytics/classes/local/target/discrete.php index 26044c7ac5d..8332ee324c5 100644 --- a/analytics/classes/local/target/discrete.php +++ b/analytics/classes/local/target/discrete.php @@ -145,7 +145,7 @@ abstract class discrete extends base { * * @return array List of values that will be ignored (array keys are ignored). */ - protected function ignored_predicted_classes() { + public function ignored_predicted_classes() { // Coding exception as this will only be called if this target have non-linear values. throw new \coding_exception('Overwrite ignored_predicted_classes() and return an array with the classes that should not ' . 'trigger the callback'); diff --git a/analytics/classes/local/time_splitting/base.php b/analytics/classes/local/time_splitting/base.php index 1ee6dfb6e22..3eef63c04cd 100644 --- a/analytics/classes/local/time_splitting/base.php +++ b/analytics/classes/local/time_splitting/base.php @@ -40,42 +40,29 @@ abstract class base { */ protected $id; + /** + * The model id. + * + * @var int + */ + protected $modelid; + /** * @var \core_analytics\analysable */ protected $analysable; - - /** - * @var int[] - */ - protected $sampleids; - - /** - * @var string - */ - protected $samplesorigin; - /** * @var array */ protected $ranges = []; - /** - * @var \core_analytics\local\indicator\base - */ - protected static $indicators = []; - - /** - * @var bool - */ - protected $evaluation = false; - /** * Define the time splitting methods ranges. * * 'time' value defines when predictions are executed, their values will be compared with - * the current time in ready_to_predict + * the current time in ready_to_predict. The ranges should be sorted by 'time' in + * ascending order. * * @return array('start' => time(), 'end' => time(), 'time' => time()) */ @@ -101,24 +88,6 @@ abstract class base { return '\\' . get_class($this); } - /** - * Returns current evaluation value. - * - * @return bool - */ - public function is_evaluating() { - return $this->evaluation; - } - - /** - * Sets the evaluation flag. - * - * @param bool $evaluation - */ - public function set_evaluating($evaluation) { - $this->evaluation = (bool)$evaluation; - } - /** * Assigns the analysable and updates the time ranges according to the analysable start and end dates. * @@ -164,252 +133,17 @@ abstract class base { } /** - * Calculates indicators and targets. + * Should we use this time range for training? * - * @param array $sampleids - * @param string $samplesorigin - * @param \core_analytics\local\indicator\base[] $indicators - * @param array $ranges - * @param \core_analytics\local\target\base $target - * @return array|bool + * @param array $range + * @return bool */ - public function calculate(&$sampleids, $samplesorigin, $indicators, $ranges, $target = false) { - - $calculatedtarget = false; - if ($target) { - // We first calculate the target because analysable data may still be invalid or none - // of the analysable samples may be valid ($sampleids is also passed by reference). - $calculatedtarget = $target->calculate($sampleids, $this->analysable); - - // We remove samples we can not calculate their target. - $sampleids = array_filter($sampleids, function($sampleid) use ($calculatedtarget) { - if (is_null($calculatedtarget[$sampleid])) { - return false; - } - return true; - }); + public function ready_to_train($range) { + $now = time(); + if ($range['time'] <= $now && $range['end'] <= $now) { + return true; } - - // No need to continue calculating if the target couldn't be calculated for any sample. - if (empty($sampleids)) { - return false; - } - - $dataset = $this->calculate_indicators($sampleids, $samplesorigin, $indicators, $ranges); - - if (empty($dataset)) { - return false; - } - - // Now that we have the indicators in place we can add the time range indicators (and target if provided) to each of them. - $this->fill_dataset($dataset, $calculatedtarget); - - $this->add_metadata($dataset, $indicators, $target); - - if (!PHPUNIT_TEST && CLI_SCRIPT) { - echo PHP_EOL; - } - - return $dataset; - } - - /** - * Calculates indicators. - * - * @param array $sampleids - * @param string $samplesorigin - * @param \core_analytics\local\indicator\base[] $indicators - * @param array $ranges - * @return array - */ - protected function calculate_indicators($sampleids, $samplesorigin, $indicators, $ranges) { - global $DB; - - $dataset = array(); - - // Faster to run 1 db query per range. - $existingcalculations = array(); - foreach ($ranges as $rangeindex => $range) { - // Load existing calculations. - $existingcalculations[$rangeindex] = \core_analytics\manager::get_indicator_calculations($this->analysable, - $range['start'], $range['end'], $samplesorigin); - } - - // Here we store samples which calculations are not all null. - $notnulls = array(); - - // Fill the dataset samples with indicators data. - $newcalculations = array(); - foreach ($indicators as $indicator) { - - // Hook to allow indicators to store analysable-dependant data. - $indicator->fill_per_analysable_caches($this->analysable); - - // Per-range calculations. - foreach ($ranges as $rangeindex => $range) { - - // Indicator instances are per-range. - $rangeindicator = clone $indicator; - - $prevcalculations = array(); - if (!empty($existingcalculations[$rangeindex][$rangeindicator->get_id()])) { - $prevcalculations = $existingcalculations[$rangeindex][$rangeindicator->get_id()]; - } - - // Calculate the indicator for each sample in this time range. - list($samplesfeatures, $newindicatorcalculations, $indicatornotnulls) = $rangeindicator->calculate($sampleids, - $samplesorigin, $range['start'], $range['end'], $prevcalculations); - - // Copy the features data to the dataset. - foreach ($samplesfeatures as $analysersampleid => $features) { - - $uniquesampleid = $this->append_rangeindex($analysersampleid, $rangeindex); - - if (!isset($notnulls[$uniquesampleid]) && !empty($indicatornotnulls[$analysersampleid])) { - $notnulls[$uniquesampleid] = $uniquesampleid; - } - - // Init the sample if it is still empty. - if (!isset($dataset[$uniquesampleid])) { - $dataset[$uniquesampleid] = array(); - } - - // Append the features indicator features at the end of the sample. - $dataset[$uniquesampleid] = array_merge($dataset[$uniquesampleid], $features); - } - - if (!$this->is_evaluating()) { - $timecreated = time(); - foreach ($newindicatorcalculations as $sampleid => $calculatedvalue) { - // Prepare the new calculations to be stored into DB. - - $indcalc = new \stdClass(); - $indcalc->contextid = $this->analysable->get_context()->id; - $indcalc->starttime = $range['start']; - $indcalc->endtime = $range['end']; - $indcalc->sampleid = $sampleid; - $indcalc->sampleorigin = $samplesorigin; - $indcalc->indicator = $rangeindicator->get_id(); - $indcalc->value = $calculatedvalue; - $indcalc->timecreated = $timecreated; - $newcalculations[] = $indcalc; - } - } - } - - if (!$this->is_evaluating()) { - $batchsize = self::get_insert_batch_size(); - if (count($newcalculations) > $batchsize) { - // We don't want newcalculations array to grow too much as we already keep the - // system memory busy storing $dataset contents. - - // Insert from the beginning. - $remaining = array_splice($newcalculations, $batchsize); - - // Sorry mssql and oracle, this will be slow. - $DB->insert_records('analytics_indicator_calc', $newcalculations); - $newcalculations = $remaining; - } - } - } - - if (!$this->is_evaluating() && $newcalculations) { - // Insert the remaining records. - $DB->insert_records('analytics_indicator_calc', $newcalculations); - } - - // Delete rows where all calculations are null. - // We still store the indicator calculation and we still store the sample id as - // processed so we don't have to process this sample again, but we exclude it - // from the dataset because it is not useful. - $nulls = array_diff_key($dataset, $notnulls); - foreach ($nulls as $uniqueid => $ignoredvalues) { - unset($dataset[$uniqueid]); - } - - return $dataset; - } - - /** - * Adds time range indicators and the target to each sample. - * - * This will identify the sample as belonging to a specific range. - * - * @param array $dataset - * @param array $calculatedtarget - * @return void - */ - protected function fill_dataset(&$dataset, $calculatedtarget = false) { - - $nranges = count($this->get_all_ranges()); - - foreach ($dataset as $uniquesampleid => $unmodified) { - - list($analysersampleid, $rangeindex) = $this->infer_sample_info($uniquesampleid); - - // No need to add range features if this time splitting method only defines one time range. - if ($nranges > 1) { - - // 1 column for each range. - $timeindicators = array_fill(0, $nranges, 0); - - $timeindicators[$rangeindex] = 1; - - $dataset[$uniquesampleid] = array_merge($timeindicators, $dataset[$uniquesampleid]); - } - - if ($calculatedtarget) { - // Add this sampleid's calculated target and the end. - $dataset[$uniquesampleid][] = $calculatedtarget[$analysersampleid]; - - } else { - // Add this sampleid, it will be used to identify the prediction that comes back from - // the predictions processor. - array_unshift($dataset[$uniquesampleid], $uniquesampleid); - } - } - } - - /** - * Adds dataset context info. - * - * The final dataset document will look like this: - * ---------------------------------------------------- - * metadata1,metadata2,metadata3,..... - * value1, value2, value3,..... - * - * indicator1,indicator2,indicator3,indicator4,..... - * stud1value1,stud1value2,stud1value3,stud1value4,..... - * stud2value1,stud2value2,stud2value3,stud2value4,..... - * ..... - * ---------------------------------------------------- - * - * @param array $dataset - * @param \core_analytics\local\indicator\base[] $indicators - * @param \core_analytics\local\target\base|false $target - * @return void - */ - protected function add_metadata(&$dataset, $indicators, $target = false) { - - $metadata = array( - 'timesplitting' => $this->get_id(), - // If no target the first column is the sampleid, if target the last column is the target. - // This will need to be updated when we support unsupervised learning models. - 'nfeatures' => count(current($dataset)) - 1 - ); - - // The first 2 samples will be used to store metadata about the dataset. - $metadatacolumns = []; - $metadatavalues = []; - foreach ($metadata as $key => $value) { - $metadatacolumns[] = $key; - $metadatavalues[] = $value; - } - - $headers = $this->get_headers($indicators, $target); - - // This will also reset samples' dataset keys. - array_unshift($dataset, $metadatacolumns, $metadatavalues, $headers); + return false; } /** @@ -421,6 +155,53 @@ abstract class base { return $this->ranges; } + /** + * By default all ranges are for training. + * + * @return array + */ + public function get_training_ranges() { + return $this->ranges; + } + + /** + * Returns the distinct range indexes in this time splitting method. + * + * @return int[] + */ + public function get_distinct_ranges() { + if ($this->include_range_info_in_training_data()) { + return array_keys($this->ranges); + } else { + return [0]; + } + } + + /** + * Returns the most recent range that can be used to predict. + * + * This method is only called when calculating predictions. + * + * @return array + */ + public function get_most_recent_prediction_range() { + + $ranges = $this->get_all_ranges(); + + // Opposite order as we are interested in the last range that can be used for prediction. + krsort($ranges); + + // We already provided the analysable to the time splitting method, there is no need to feed it back. + foreach ($ranges as $rangeindex => $range) { + if ($this->ready_to_predict($range)) { + // We need to maintain the same indexes. + return array($rangeindex => $range); + } + } + + return array(); + } + /** * Returns range data by its index. * @@ -441,7 +222,7 @@ abstract class base { * @param int $rangeindex * @return string */ - public function append_rangeindex($sampleid, $rangeindex) { + public final function append_rangeindex($sampleid, $rangeindex) { return $sampleid . '-' . $rangeindex; } @@ -451,46 +232,47 @@ abstract class base { * @param string $uniquesampleid * @return array array($sampleid, $rangeindex) */ - public function infer_sample_info($uniquesampleid) { + public final function infer_sample_info($uniquesampleid) { return explode('-', $uniquesampleid); } /** - * Returns the headers for the csv file based on the indicators and the target. + * Whether to include the range index in the training data or not. * - * @param \core_analytics\local\indicator\base[] $indicators - * @param \core_analytics\local\target\base|false $target - * @return string[] + * By default, we consider that the different time ranges included in a time splitting method may not be + * compatible between them (i.e. the indicators calculated at the end of the course can easily + * differ from indicators calculated at the beginning of the course). So we include the range index as + * one of the variables that the machine learning backend uses to generate predictions. + * + * If the indicators calculated using the different time ranges available in this time splitting method + * are comparable you can overwrite this method to return false. + * + * Note that: + * - This is only relevant for models whose predictions are not based on assumptions + * (i.e. the ones using a machine learning backend to generate predictions). + * - The ranges can only be included in the training data when + * we know the final number of ranges the time splitting method will have. E.g. + * We can not know the final number of ranges of a 'daily' time splitting method + * as we will have one new range every day. + * @return bool */ - protected function get_headers($indicators, $target = false) { - // 3rd column will contain the indicator ids. - $headers = array(); + public function include_range_info_in_training_data() { + return true; + } - if (!$target) { - // The first column is the sampleid. - $headers[] = 'sampleid'; - } - - // We always have 1 column for each time splitting method range, it does not depend on how - // many ranges we calculated. - $ranges = $this->get_all_ranges(); - if (count($ranges) > 1) { - foreach ($ranges as $rangeindex => $range) { - $headers[] = 'range/' . $rangeindex; - } - } - - // Model indicators. - foreach ($indicators as $indicator) { - $headers = array_merge($headers, $indicator::get_feature_headers()); - } - - // The target as well. - if ($target) { - $headers[] = $target->get_id(); - } - - return $headers; + /** + * Whether to cache or not the indicator calculations. + * + * Indicator calculations are stored to be reused across models. The calculations + * are indexed by the calculation start and end time, and these times depend on the + * time-splitting method. You should overwrite this method and return false if the time + * frames generated by your time-splitting method are unique and / or can hardly be + * reused by further models. + * + * @return bool + */ + public function cache_indicator_calculations(): bool { + return true; } /** @@ -508,32 +290,4 @@ abstract class base { } } } - - /** - * Returns the batch size used for insert_records. - * - * This method tries to find the best batch size without getting - * into dml internals. Maximum 1000 records to save memory. - * - * @return int - */ - private static function get_insert_batch_size() { - global $DB; - - // 500 is pgsql default so using 1000 is fine, no other db driver uses a hardcoded value. - if (empty($DB->dboptions['bulkinsertsize'])) { - return 1000; - } - - $bulkinsert = $DB->dboptions['bulkinsertsize']; - if ($bulkinsert < 1000) { - return $bulkinsert; - } - - while ($bulkinsert > 1000) { - $bulkinsert = round($bulkinsert / 2, 0); - } - - return (int)$bulkinsert; - } } diff --git a/analytics/classes/local/time_splitting/periodic.php b/analytics/classes/local/time_splitting/periodic.php new file mode 100644 index 00000000000..4a52370c375 --- /dev/null +++ b/analytics/classes/local/time_splitting/periodic.php @@ -0,0 +1,143 @@ +. + +/** + * Time splitting method that generates predictions regularly. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics\local\time_splitting; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Time splitting method that generates predictions periodically. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +abstract class periodic extends base { + + /** + * The periodicity of the predictions / training data generation. + * + * @return \DateInterval + */ + abstract protected function periodicity(); + + /** + * Returns whether the analysable can be processed by this time splitting method or not. + * + * @param \core_analytics\analysable $analysable + * @return bool + */ + public function is_valid_analysable(\core_analytics\analysable $analysable) { + if (!$analysable->get_start()) { + return false; + } + return true; + } + + /** + * define_ranges + * + * @return array + */ + protected function define_ranges() { + + $periodicity = $this->periodicity(); + + $now = new \DateTimeImmutable('now', \core_date::get_server_timezone_object()); + + if ($this->analysable->get_end()) { + $end = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_end()); + } + $next = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_start()); + + $ranges = []; + while ($next < $now && + (empty($end) || $next < $end)) { + $range = $this->get_next_range($next); + if ($range) { + $ranges[] = $range; + } + $next = $next->add($periodicity); + } + + $nextrange = $this->get_next_range($next); + if ($this->ready_to_predict($nextrange) && (empty($end) || $next < $end)) { + // Add the next one if we have not reached the analysable end yet. + // It will be used to get predictions. + $ranges[] = $nextrange; + } + + return $ranges; + } + + /** + * Overwritten as all generated rows are comparable. + * + * @return bool + */ + public function include_range_info_in_training_data() { + return false; + } + + /** + * Overwritting as the last range may be for prediction. + * + * @return array + */ + public function get_training_ranges() { + // Cloning the array. + $trainingranges = $this->ranges; + + foreach ($trainingranges as $rangeindex => $range) { + if (!$this->ready_to_train($range)) { + unset($trainingranges[$rangeindex]); + } + } + + return $trainingranges; + } + + /** + * The next range is based on the past period. + * + * @param \DateTimeImmutable $next + * @return array + */ + protected function get_next_range(\DateTimeImmutable $next) { + + $end = $next->getTimestamp(); + $start = $next->sub($this->periodicity())->getTimestamp(); + + if ($start < $this->analysable->get_start()) { + // We skip the first range generated as its start is prior to the analysable start. + return false; + } + + return [ + 'start' => $start, + 'end' => $end, + 'time' => $end + ]; + } +} diff --git a/analytics/classes/local/time_splitting/upcoming_periodic.php b/analytics/classes/local/time_splitting/upcoming_periodic.php new file mode 100644 index 00000000000..d8b2598f735 --- /dev/null +++ b/analytics/classes/local/time_splitting/upcoming_periodic.php @@ -0,0 +1,62 @@ +. + +/** + * Time splitting method that generates predictions periodically. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics\local\time_splitting; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Time splitting method that generates predictions periodically. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +abstract class upcoming_periodic extends periodic { + + /** + * The next range indicator calculations should be based on upcoming dates. + * + * @param \DateTimeImmutable $next + * @return array + */ + protected function get_next_range(\DateTimeImmutable $next) { + + $start = $next->getTimestamp(); + $end = $next->add($this->periodicity())->getTimestamp(); + return [ + 'start' => $start, + 'end' => $end, + 'time' => $start + ]; + } + + /** + * Whether to cache or not the indicator calculations. + * @return bool + */ + public function cache_indicator_calculations(): bool { + return false; + } +} diff --git a/analytics/classes/manager.php b/analytics/classes/manager.php index da9d26e20d2..3e5bbb8d3e9 100644 --- a/analytics/classes/manager.php +++ b/analytics/classes/manager.php @@ -80,10 +80,23 @@ class manager { * * @throws \required_capability_exception * @param \context $context + * @param bool $return The method returns a bool if true. * @return void */ - public static function check_can_list_insights(\context $context) { - require_capability('moodle/analytics:listinsights', $context); + public static function check_can_list_insights(\context $context, bool $return = false) { + global $USER; + + if ($context->contextlevel === CONTEXT_USER && $context->instanceid == $USER->id) { + $capability = 'moodle/analytics:listowninsights'; + } else { + $capability = 'moodle/analytics:listinsights'; + } + + if ($return) { + return has_capability($capability, $context); + } else { + require_capability($capability, $context); + } } /** @@ -552,15 +565,19 @@ class manager { $models = self::get_all_models(); foreach ($models as $model) { $analyser = $model->get_analyser(array('notimesplitting' => true)); - $analysables = $analyser->get_analysables(); - if (!$analysables) { + $analysables = $analyser->get_analysables_iterator(); + + $analysableids = []; + foreach ($analysables as $analysable) { + if (!$analysable) { + continue; + } + $analysableids[] = $analysable->get_id(); + } + if (empty($analysableids)) { continue; } - $analysableids = array_map(function($analysable) { - return $analysable->get_id(); - }, $analysables); - list($notinsql, $params) = $DB->get_in_or_equal($analysableids, SQL_PARAMS_NAMED, 'param', false); $params['modelid'] = $model->get_id(); diff --git a/analytics/classes/model.php b/analytics/classes/model.php index 5ca4f2cd266..5113a854c1a 100644 --- a/analytics/classes/model.php +++ b/analytics/classes/model.php @@ -306,12 +306,6 @@ class model { } } - if (!empty($options['evaluation'])) { - foreach ($timesplittings as $timesplitting) { - $timesplitting->set_evaluating(true); - } - } - $classname = $target->get_analyser_class(); if (!class_exists($classname)) { throw new \coding_exception($classname . ' class does not exists'); @@ -541,7 +535,7 @@ class model { $this->get_analyser()->add_log(get_string('noevaluationbasedassumptions', 'analytics')); $result = new \stdClass(); $result->status = self::NO_DATASET; - return array($this->get_time_splitting()->get_id() => $result); + return array($result); } $options['evaluation'] = true; @@ -736,50 +730,67 @@ class model { // Before get_unlabelled_data call so we get an early exception if it is not writable. $outputdir = $this->get_output_dir(array('execution')); - // Before get_unlabelled_data call so we get an early exception if it is not ready. if (!$this->is_static()) { + // Predictions using a machine learning backend. + + // Before get_unlabelled_data call so we get an early exception if it is not ready. $predictor = $this->get_predictions_processor(); - } - $samplesdata = $this->get_analyser()->get_unlabelled_data(); + $samplesdata = $this->get_analyser()->get_unlabelled_data(); - // Get the prediction samples file. - if (empty($samplesdata) || empty($samplesdata[$this->model->timesplitting])) { + // Get the prediction samples file. + if (empty($samplesdata) || empty($samplesdata[$this->model->timesplitting])) { - $result = new \stdClass(); - $result->status = self::NO_DATASET; - $result->info = $this->get_analyser()->get_logs(); - return $result; - } - $samplesfile = $samplesdata[$this->model->timesplitting]; + $result = new \stdClass(); + $result->status = self::NO_DATASET; + $result->info = $this->get_analyser()->get_logs(); + return $result; + } + $samplesfile = $samplesdata[$this->model->timesplitting]; - // We need to throw an exception if we are trying to predict stuff that was already predicted. - $params = array('modelid' => $this->model->id, 'action' => 'predicted', 'fileid' => $samplesfile->get_id()); - if ($predicted = $DB->get_record('analytics_used_files', $params)) { - throw new \moodle_exception('erroralreadypredict', 'analytics', '', $samplesfile->get_id()); - } + // We need to throw an exception if we are trying to predict stuff that was already predicted. + $params = array('modelid' => $this->model->id, 'action' => 'predicted', 'fileid' => $samplesfile->get_id()); + if ($predicted = $DB->get_record('analytics_used_files', $params)) { + throw new \moodle_exception('erroralreadypredict', 'analytics', '', $samplesfile->get_id()); + } - $indicatorcalculations = \core_analytics\dataset_manager::get_structured_data($samplesfile); + $indicatorcalculations = \core_analytics\dataset_manager::get_structured_data($samplesfile); - // Prepare the results object. - $result = new \stdClass(); - - if ($this->is_static()) { - // Prediction based on assumptions. - $result->status = self::OK; - $result->info = []; - $result->predictions = $this->get_static_predictions($indicatorcalculations); - - } else { // Estimation and classification processes run on the machine learning backend side. if ($this->get_target()->is_linear()) { $predictorresult = $predictor->estimate($this->get_unique_id(), $samplesfile, $outputdir); } else { $predictorresult = $predictor->classify($this->get_unique_id(), $samplesfile, $outputdir); } + + // Prepare the results object. + $result = new \stdClass(); $result->status = $predictorresult->status; $result->info = $predictorresult->info; $result->predictions = $this->format_predictor_predictions($predictorresult); + + } else { + // Predictions based on assumptions. + + $indicatorcalculations = $this->get_analyser()->get_static_data(); + // Get the prediction samples file. + if (empty($indicatorcalculations) || empty($indicatorcalculations[$this->model->timesplitting])) { + + $result = new \stdClass(); + $result->status = self::NO_DATASET; + $result->info = $this->get_analyser()->get_logs(); + return $result; + } + + // Same as reset($indicatorcalculations) as models based on assumptions only analyse 1 single + // time-splitting method. + $indicatorcalculations = $indicatorcalculations[$this->model->timesplitting]; + + // Prepare the results object. + $result = new \stdClass(); + $result->status = self::OK; + $result->info = []; + $result->predictions = $this->get_static_predictions($indicatorcalculations); } if ($result->status !== self::OK) { @@ -787,14 +798,17 @@ class model { } if ($result->predictions) { - $samplecontexts = $this->execute_prediction_callbacks($result->predictions, $indicatorcalculations); + list($samplecontexts, $predictionrecords) = $this->execute_prediction_callbacks($result->predictions, + $indicatorcalculations); } if (!empty($samplecontexts) && $this->uses_insights()) { - $this->trigger_insights($samplecontexts); + $this->trigger_insights($samplecontexts, $predictionrecords); } - $this->flag_file_as_used($samplesfile, 'predicted'); + if (!$this->is_static()) { + $this->flag_file_as_used($samplesfile, 'predicted'); + } return $result; } @@ -855,7 +869,7 @@ class model { * @param array $indicatorcalculations * @return array */ - protected function execute_prediction_callbacks($predictions, $indicatorcalculations) { + protected function execute_prediction_callbacks(&$predictions, $indicatorcalculations) { // Here we will store all predictions' contexts, this will be used to limit which users will see those predictions. $samplecontexts = array(); @@ -865,7 +879,6 @@ class model { // The unique sample id contains both the sampleid and the rangeindex. list($sampleid, $rangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid); - if ($this->get_target()->triggers_callback($prediction->prediction, $prediction->predictionscore)) { // Prepare the record to store the predicted values. @@ -887,19 +900,38 @@ class model { $this->save_predictions($records); } - return $samplecontexts; + return [$samplecontexts, $records]; } /** * Generates insights and updates the cache. * * @param \context[] $samplecontexts + * @param \stdClass[] $predictionrecords * @return void */ - protected function trigger_insights($samplecontexts) { + protected function trigger_insights($samplecontexts, $predictionrecords) { // Notify the target that all predictions have been processed. - $this->get_target()->generate_insight_notifications($this->model->id, $samplecontexts); + if ($this->get_analyser()::one_sample_per_analysable()) { + + // We need to do something unusual here. self::save_predictions uses the bulk-insert function (insert_records()) for + // performance reasons and that function does not return us the inserted ids. We need to retrieve them from + // the database, and we need to do it using one single database query (for performance reasons as well). + $predictionrecords = $this->add_prediction_ids($predictionrecords); + + // Get \core_analytics\prediction objects also fetching the samplesdata. This costs us + // 1 db read, but we have to pay it if we want that our insights include links to the + // suggested actions. + $predictions = array_map(function($predictionobj) { + $prediction = new \core_analytics\prediction($predictionobj, $this->prediction_sample_data($predictionobj)); + return $prediction; + }, $predictionrecords); + } else { + $predictions = []; + } + + $this->get_target()->generate_insight_notifications($this->model->id, $samplecontexts, $predictions); // Update cache. $cache = \cache::make('core', 'contextwithinsights'); @@ -926,12 +958,23 @@ class model { */ protected function get_static_predictions(&$indicatorcalculations) { + $headers = array_shift($indicatorcalculations); + + // Get rid of the sampleid header. + array_shift($headers); + // Group samples by analysable for \core_analytics\local\target::calculate. $analysables = array(); // List all sampleids together. $sampleids = array(); foreach ($indicatorcalculations as $uniquesampleid => $indicators) { + + // Get rid of the sampleid column. + unset($indicators[0]); + $indicators = array_combine($headers, $indicators); + $indicatorcalculations[$uniquesampleid] = $indicators; + list($sampleid, $rangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid); $analysable = $this->get_analyser()->get_sample_analysable($sampleid); @@ -946,6 +989,7 @@ class model { 'sampleids' => array() ]; } + // Using the sampleid as a key so we can easily merge indicators data later. $analysables[$analysableclass][$rangeindex]->indicatorsdata[$sampleid] = $indicators; // We could use indicatorsdata keys but the amount of redundant data is not that big and leaves code below cleaner. @@ -969,16 +1013,20 @@ class model { $this->get_target()->add_sample_data($data->indicatorsdata); // Append new elements (we can not get duplicates because sample-analysable relation is N-1). - $range = $this->get_time_splitting()->get_range_by_index($rangeindex); + $timesplitting = $this->get_time_splitting(); + $timesplitting->set_analysable($data->analysable); + $range = $timesplitting->get_range_by_index($rangeindex); + $this->get_target()->filter_out_invalid_samples($data->sampleids, $data->analysable, false); $calculations = $this->get_target()->calculate($data->sampleids, $data->analysable, $range['start'], $range['end']); // Missing $indicatorcalculations values in $calculations are caused by is_valid_sample. We need to remove // these $uniquesampleid from $indicatorcalculations because otherwise they will be stored as calculated // by self::save_prediction. - $indicatorcalculations = array_filter($indicatorcalculations, function($indicators, $uniquesampleid) use ($calculations) { - list($sampleid, $rangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid); - if (!isset($calculations[$sampleid])) { + $indicatorcalculations = array_filter($indicatorcalculations, function($indicators, $uniquesampleid) + use ($calculations, $rangeindex) { + list($sampleid, $indicatorsrangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid); + if ($rangeindex == $indicatorsrangeindex && !isset($calculations[$sampleid])) { return false; } return true; @@ -1642,6 +1690,44 @@ class model { $DB->update_record('analytics_models', $this->model); } + /** + * Adds the id from {analytics_predictions} db table to the prediction \stdClass objects. + * + * @param \stdClass[] $predictionrecords + * @return \stdClass[] The prediction records including their ids in {analytics_predictions} db table. + */ + private function add_prediction_ids($predictionrecords) { + global $DB; + + $firstprediction = reset($predictionrecords); + + $contextids = array_map(function($predictionobj) { + return $predictionobj->contextid; + }, $predictionrecords); + list($contextsql, $contextparams) = $DB->get_in_or_equal($contextids, SQL_PARAMS_NAMED); + + // We select the fields that will allow us to map ids to $predictionrecords. Given that we already filter by modelid + // we have enough with sampleid and rangeindex. The reason is that the sampleid relation to a site is N - 1. + $fields = 'id, sampleid, rangeindex'; + + // We include the contextid and the timecreated filter to reduce the number of records in $dbpredictions. We can not + // add as many OR conditions as records in $predictionrecords. + $sql = "SELECT $fields + FROM {analytics_predictions} + WHERE modelid = :modelid + AND contextid $contextsql + AND timecreated >= :firsttimecreated"; + $params = $contextparams + ['modelid' => $this->model->id, 'firsttimecreated' => $firstprediction->timecreated]; + $dbpredictions = $DB->get_recordset_sql($sql, $params); + foreach ($dbpredictions as $id => $dbprediction) { + // The append_rangeindex implementation is the same regardless of the time splitting method in use. + $uniqueid = $this->get_time_splitting()->append_rangeindex($dbprediction->sampleid, $dbprediction->rangeindex); + $predictionrecords[$uniqueid]->id = $dbprediction->id; + } + + return $predictionrecords; + } + /** * Purges the insights cache. */ diff --git a/analytics/classes/prediction_action.php b/analytics/classes/prediction_action.php index 2fb4281ba50..6f3ee338e13 100644 --- a/analytics/classes/prediction_action.php +++ b/analytics/classes/prediction_action.php @@ -40,6 +40,11 @@ class prediction_action { */ protected $actionname = null; + /** + * @var \moodle_url + */ + protected $url = null; + /** * @var \action_menu_link */ @@ -50,7 +55,7 @@ class prediction_action { * * @param string $actionname They should match a-zA-Z_0-9-, as we apply a PARAM_ALPHANUMEXT filter * @param \core_analytics\prediction $prediction - * @param \moodle_url $actionurl + * @param \moodle_url $actionurl The final URL where the user should be forwarded. * @param \pix_icon $icon Link icon * @param string $text Link text * @param bool $primary Primary button or secondary. @@ -61,16 +66,17 @@ class prediction_action { $text, $primary = false, $attributes = array()) { $this->actionname = $actionname; + $this->text = $text; // We want to track how effective are our suggested actions, we pass users through a script that will log these actions. $params = array('action' => $this->actionname, 'predictionid' => $prediction->get_prediction_data()->id, 'forwardurl' => $actionurl->out(false)); - $url = new \moodle_url('/report/insights/action.php', $params); + $this->url = new \moodle_url('/report/insights/action.php', $params); if ($primary === false) { - $this->actionlink = new \action_menu_link_secondary($url, $icon, $text, $attributes); + $this->actionlink = new \action_menu_link_secondary($this->url, $icon, $this->text, $attributes); } else { - $this->actionlink = new \action_menu_link_primary($url, $icon, $text, $attributes); + $this->actionlink = new \action_menu_link_primary($this->url, $icon, $this->text, $attributes); } } @@ -83,6 +89,15 @@ class prediction_action { return $this->actionname; } + /** + * Returns the url to the action. + * + * @return \moodle_url + */ + public function get_url() { + return $this->url; + } + /** * Returns the link to the action. * @@ -91,4 +106,12 @@ class prediction_action { public function get_action_link() { return $this->actionlink; } + + /** + * Returns the action text. + * @return string + */ + public function get_text() { + return $this->text; + } } diff --git a/analytics/classes/user.php b/analytics/classes/user.php new file mode 100644 index 00000000000..6dd83c73a2a --- /dev/null +++ b/analytics/classes/user.php @@ -0,0 +1,207 @@ +. + +/** + * Moodle user analysable + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_analytics; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Moodle user analysable + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class user implements \core_analytics\analysable { + + /** + * @var bool Has this user data been already loaded. + */ + protected $loaded = false; + + /** + * @var int $cachedid self::$cachedinstance analysable id. + */ + protected static $cachedid = 0; + + /** + * @var \core_analytics\user $cachedinstance + */ + protected static $cachedinstance = null; + + /** + * User object + * + * @var \stdClass + */ + protected $user = null; + + /** + * The user context. + * + * @var \context_user + */ + protected $usercontext = null; + + /** + * Constructor. + * + * Use self::instance() instead to get cached copies of the class. Instances obtained + * through this constructor will not be cached. + * + * @param int|\stdClass $user User id + * @param \context|null $context + * @return void + */ + public function __construct($user, ?\context $context = null) { + + if (is_scalar($user)) { + $this->user = new \stdClass(); + $this->user->id = $user; + } else { + $this->user = $user; + } + + if (!is_null($context)) { + $this->usercontext = $context; + } + } + + /** + * Returns an analytics user instance. + * + * Lazy load of analysable data. + * + * @param int|\stdClass $user User object or user id + * @param \context|null $context + * @return \core_analytics\user + */ + public static function instance($user, ?\context $context = null) { + + $userid = $user; + if (!is_scalar($userid)) { + $userid = $user->id; + } + + if (self::$cachedid === $userid) { + return self::$cachedinstance; + } + + $cachedinstance = new \core_analytics\user($user, $context); + self::$cachedinstance = $cachedinstance; + self::$cachedid = (int)$userid; + return self::$cachedinstance; + } + + /** + * get_id + * + * @return int + */ + public function get_id() { + return $this->user->id; + } + + /** + * Loads the analytics user object. + * + * @return void + */ + protected function load() { + + // The instance constructor could be already loaded with the full user object. Using email + // because it is a required user field. + if (empty($this->user->email)) { + $this->user = \core_user::get_user($this->user->id); + } + + $this->usercontext = $this->get_context(); + + $this->now = time(); + + // Flag the instance as loaded. + $this->loaded = true; + } + + /** + * The user full name. + * + * @return string + */ + public function get_name() { + + if (!$this->loaded) { + $this->load(); + } + return fullname($this->user); + } + + /** + * get_context + * + * @return \context + */ + public function get_context() { + if ($this->usercontext === null) { + $this->usercontext = \context_user::instance($this->user->id); + } + return $this->usercontext; + } + + /** + * Get the start timestamp. + * + * @return int + */ + public function get_start() { + + if (!$this->loaded) { + $this->load(); + } + return $this->user->timecreated; + } + + /** + * Get the end timestamp. + * + * @return int + */ + public function get_end() { + return self::MAX_TIME; + } + + /** + * Returns a user plain object. + * + * @return \stdClass + */ + public function get_user_data() { + + if (!$this->loaded) { + $this->load(); + } + + return $this->user; + } +} diff --git a/analytics/templates/insight_info_message.mustache b/analytics/templates/insight_info_message.mustache new file mode 100644 index 00000000000..54426a41b8f --- /dev/null +++ b/analytics/templates/insight_info_message.mustache @@ -0,0 +1,60 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core_analytics/insight_info_message + + HTML message for insights + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Example context (json): + { + "url": "https://moodle.org" + } +}} + + +{{#str}} insightinfomessagehtml, analytics {{/str}} +

+{{#str}} viewinsight, analytics {{/str}} \ No newline at end of file diff --git a/analytics/templates/insight_info_message_prediction.mustache b/analytics/templates/insight_info_message_prediction.mustache new file mode 100644 index 00000000000..9896031bf6d --- /dev/null +++ b/analytics/templates/insight_info_message_prediction.mustache @@ -0,0 +1,69 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core_analytics/insight_info_message_prediction + + HTML message for insights with one single prediction + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Example context (json): + { + "actions": [ + { + "url": "https://moodle.org", + "text": "Moodle" + }, { + "url": "https://en.wikipedia.org/wiki/Noodle", + "text": "Noodle" + } + ] + } +}} + +{{! Default btn-default styles. These styles are not applied to Moodle's web UI as there is a body:not(.dir-ltr):not(.dir-rtl)}} + + + +
+{{#actions}} + {{text}} +{{/actions}} diff --git a/analytics/tests/dataset_manager_test.php b/analytics/tests/dataset_manager_test.php index 760e667f138..f5c916c9ea3 100644 --- a/analytics/tests/dataset_manager_test.php +++ b/analytics/tests/dataset_manager_test.php @@ -56,10 +56,8 @@ class dataset_manager_testcase extends advanced_testcase { public function test_create_dataset() { $dataset1 = new \core_analytics\dataset_manager(1, 1, 'whatever', \core_analytics\dataset_manager::LABELLED_FILEAREA, false); - $dataset1->init_process(); $dataset1data = array_merge($this->sharedtoprows, array(array('yeah', 'yeah', 'yeah'))); $f1 = $dataset1->store($dataset1data); - $dataset1->close_process(); $f1contents = $f1->get_content(); $this->assertContains('yeah', $f1contents); @@ -76,16 +74,12 @@ class dataset_manager_testcase extends advanced_testcase { public function test_merge_datasets() { $dataset1 = new \core_analytics\dataset_manager(1, 1, 'whatever', \core_analytics\dataset_manager::LABELLED_FILEAREA, false); - $dataset1->init_process(); $dataset1data = array_merge($this->sharedtoprows, array(array('yeah', 'yeah', 'yeah'))); $f1 = $dataset1->store($dataset1data); - $dataset1->close_process(); $dataset2 = new \core_analytics\dataset_manager(1, 2, 'whatever', \core_analytics\dataset_manager::LABELLED_FILEAREA, false); - $dataset2->init_process(); $dataset2data = array_merge($this->sharedtoprows, array(array('no', 'no', 'no'))); $f2 = $dataset2->store($dataset2data); - $dataset2->close_process(); $files = array($f1, $f2); $merged = \core_analytics\dataset_manager::merge_datasets($files, 1, 'whatever', @@ -123,10 +117,8 @@ class dataset_manager_testcase extends advanced_testcase { // merged into training and prediction files). $analysabledataset = new \core_analytics\dataset_manager($fakemodelid, 1, 'whatever', \core_analytics\dataset_manager::LABELLED_FILEAREA, false); - $analysabledataset->init_process(); $analysabledatasetdata = array_merge($this->sharedtoprows, array(array('yeah', 'yeah', 'yeah'))); $file = $analysabledataset->store($analysabledatasetdata); - $analysabledataset->close_process(); // Evaluation files ignored. $evaluationdataset = \core_analytics\dataset_manager::merge_datasets(array($file), $fakemodelid, diff --git a/analytics/tests/fixtures/test_analyser.php b/analytics/tests/fixtures/test_analysis.php similarity index 79% rename from analytics/tests/fixtures/test_analyser.php rename to analytics/tests/fixtures/test_analysis.php index 736388cee78..6af04d1f2ea 100644 --- a/analytics/tests/fixtures/test_analyser.php +++ b/analytics/tests/fixtures/test_analysis.php @@ -31,18 +31,17 @@ defined('MOODLE_INTERNAL') || die(); * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ -class test_analyser extends \core\analytics\analyser\courses { +class test_analysis extends \core_analytics\analysis { /** * Overwritten to add a delay. * * @param \core_analytics\analysable $analysable - * @param mixed $includetarget - * @return null + * @return array */ - public function process_analysable($analysable, $includetarget) { - // A bit more than 1 second. - usleep(1100000); - return parent::process_analysable($analysable, $includetarget); + public function process_analysable(\core_analytics\analysable $analysable): array { + // Half a second. + usleep(500000); + return parent::process_analysable($analysable); } } diff --git a/analytics/tests/fixtures/test_indicator_null.php b/analytics/tests/fixtures/test_indicator_null.php new file mode 100644 index 00000000000..8b928477d2f --- /dev/null +++ b/analytics/tests/fixtures/test_indicator_null.php @@ -0,0 +1,62 @@ +. + +/** + * Test indicator. Always null. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Test indicator. Always null. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class test_indicator_null extends \core_analytics\local\indicator\binary { + + /** + * Returns a lang_string object representing the name for the indicator. + * + * Used as column identificator. + * + * If there is a corresponding '_help' string this will be shown as well. + * + * @return \lang_string + */ + public static function get_name() : \lang_string { + // Using a string that exists and contains a corresponding '_help' string. + return new \lang_string('allowstealthmodules'); + } + + /** + * calculate_sample + * + * @param int $sampleid + * @param string $samplesorigin + * @param int $starttime + * @param int $endtime + * @return float + */ + protected function calculate_sample($sampleid, $samplesorigin, $starttime, $endtime) { + return null; + } +} diff --git a/analytics/tests/fixtures/test_site_users_analyser.php b/analytics/tests/fixtures/test_site_users_analyser.php index 39bc1f2242e..ac0e1c0ecae 100644 --- a/analytics/tests/fixtures/test_site_users_analyser.php +++ b/analytics/tests/fixtures/test_site_users_analyser.php @@ -77,7 +77,7 @@ class test_site_users_analyser extends \core_analytics\local\analyser\sitewide { * @param \core_analytics\analysable $site * @return array */ - protected function get_all_samples(\core_analytics\analysable $site) { + public function get_all_samples(\core_analytics\analysable $site) { global $DB; $users = $DB->get_records('user'); @@ -121,7 +121,7 @@ class test_site_users_analyser extends \core_analytics\local\analyser\sitewide { * @return array array(string, \renderable) */ public function sample_description($sampleid, $contextid, $sampledata) { - $description = fullname($samplesdata['user']); + $description = fullname($sampledata['user']); $userimage = new \pix_icon('i/user', get_string('user')); return array($description, $userimage); } diff --git a/analytics/tests/fixtures/test_target_shortname.php b/analytics/tests/fixtures/test_target_shortname.php index bd3be06f034..6e9a71560ef 100644 --- a/analytics/tests/fixtures/test_target_shortname.php +++ b/analytics/tests/fixtures/test_target_shortname.php @@ -87,7 +87,7 @@ class test_target_shortname extends \core_analytics\local\target\binary { * We don't want to discard results. * @return array */ - protected function ignored_predicted_classes() { + public function ignored_predicted_classes() { return array(); } diff --git a/analytics/tests/fixtures/test_target_site_users.php b/analytics/tests/fixtures/test_target_site_users.php index 6f8b30b03b7..e78cbf584c0 100644 --- a/analytics/tests/fixtures/test_target_site_users.php +++ b/analytics/tests/fixtures/test_target_site_users.php @@ -89,7 +89,7 @@ class test_target_site_users extends \core_analytics\local\target\binary { * We don't want to discard results. * @return array */ - protected function ignored_predicted_classes() { + public function ignored_predicted_classes() { return array(); } diff --git a/analytics/tests/fixtures/test_timesplitting_seconds.php b/analytics/tests/fixtures/test_timesplitting_seconds.php new file mode 100644 index 00000000000..e4b4f70423c --- /dev/null +++ b/analytics/tests/fixtures/test_timesplitting_seconds.php @@ -0,0 +1,52 @@ +. + +/** + * Test time splitting. + * + * @package core_analytics + * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Test time splitting. + * + * @package core_analytics + * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class test_timesplitting_seconds extends \core_analytics\local\time_splitting\periodic { + + /** + * Every second. + * @return \DateInterval + */ + public function periodicity() { + return new \DateInterval('PT1S'); + } + + /** + * Just to comply with the interface. + * + * @return \lang_string + */ + public static function get_name() : \lang_string { + return new \lang_string('error'); + } +} diff --git a/analytics/tests/fixtures/test_timesplitting_weekly.php b/analytics/tests/fixtures/test_timesplitting_weekly.php new file mode 100644 index 00000000000..72403de9e0f --- /dev/null +++ b/analytics/tests/fixtures/test_timesplitting_weekly.php @@ -0,0 +1,51 @@ +. + +/** + * Time splitting method that generates weekly predictions. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Time splitting method that generates weekly predictions. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class test_timesplitting_weekly extends \core_analytics\local\time_splitting\periodic { + + /** + * The time splitting method name. + * @return \lang_string + */ + public static function get_name() : \lang_string { + return new \lang_string('error'); + } + + /** + * Once per week. + * @return \DateInterval + */ + public function periodicity() { + return new \DateInterval('P1W'); + } +} diff --git a/analytics/tests/manager_test.php b/analytics/tests/manager_test.php index e685bed20bb..f12af444615 100644 --- a/analytics/tests/manager_test.php +++ b/analytics/tests/manager_test.php @@ -348,9 +348,11 @@ class analytics_manager_testcase extends advanced_testcase { $noteaching = \core_analytics\manager::get_target('\core\analytics\target\no_teaching'); $dropout = \core_analytics\manager::get_target('\core\analytics\target\course_dropout'); + $upcomingactivities = \core_analytics\manager::get_target('\core_user\analytics\target\upcoming_activities_due'); $this->assertTrue(\core_analytics\model::exists($noteaching)); $this->assertTrue(\core_analytics\model::exists($dropout)); + $this->assertTrue(\core_analytics\model::exists($upcomingactivities)); foreach (\core_analytics\manager::get_all_models() as $model) { $model->delete(); @@ -358,14 +360,17 @@ class analytics_manager_testcase extends advanced_testcase { $this->assertFalse(\core_analytics\model::exists($noteaching)); $this->assertFalse(\core_analytics\model::exists($dropout)); + $this->assertFalse(\core_analytics\model::exists($upcomingactivities)); $updated = \core_analytics\manager::update_default_models_for_component('moodle'); - $this->assertEquals(2, count($updated)); + $this->assertEquals(3, count($updated)); + $this->assertTrue(array_pop($updated) instanceof \core_analytics\model); $this->assertTrue(array_pop($updated) instanceof \core_analytics\model); $this->assertTrue(array_pop($updated) instanceof \core_analytics\model); $this->assertTrue(\core_analytics\model::exists($noteaching)); $this->assertTrue(\core_analytics\model::exists($dropout)); + $this->assertTrue(\core_analytics\model::exists($upcomingactivities)); $repeated = \core_analytics\manager::update_default_models_for_component('moodle'); diff --git a/analytics/tests/model_test.php b/analytics/tests/model_test.php index 0e3385ad8ff..97a7e3b2ed3 100644 --- a/analytics/tests/model_test.php +++ b/analytics/tests/model_test.php @@ -30,7 +30,7 @@ require_once(__DIR__ . '/fixtures/test_indicator_fullname.php'); require_once(__DIR__ . '/fixtures/test_target_shortname.php'); require_once(__DIR__ . '/fixtures/test_static_target_shortname.php'); require_once(__DIR__ . '/fixtures/test_target_course_level_shortname.php'); -require_once(__DIR__ . '/fixtures/test_analyser.php'); +require_once(__DIR__ . '/fixtures/test_analysis.php'); /** * Unit tests for the model. @@ -292,7 +292,7 @@ class analytics_model_testcase extends advanced_testcase { $this->resetAfterTest(true); - set_config('modeltimelimit', 2, 'analytics'); + set_config('modeltimelimit', 1, 'analytics'); $courses = array(); for ($i = 0; $i < 5; $i++) { @@ -302,19 +302,22 @@ class analytics_model_testcase extends advanced_testcase { } $target = new test_target_course_level_shortname(); - $analyser = new test_analyser(1, $target, [], [], []); + $analyser = new \core\analytics\analyser\courses(1, $target, [], [], []); - // Each analysable element takes 1.1 secs, so the max (and likely) number of analysable + $result = new \core_analytics\local\analysis\result_array(1, false, []); + $analysis = new test_analysis($analyser, false, $result); + + // Each analysable element takes 0.5 secs minimum (test_analysis), so the max (and likely) number of analysable // elements that will be processed is 2. - $analyser->get_analysable_data(false); + $analysis->run(); $params = array('modelid' => 1, 'action' => 'prediction'); $this->assertLessThanOrEqual(2, $DB->count_records('analytics_used_analysables', $params)); - $analyser->get_analysable_data(false); + $analysis->run(); $this->assertLessThanOrEqual(4, $DB->count_records('analytics_used_analysables', $params)); // Check that analysable elements have been processed following the analyser order - // (course->sortorder here). We can not check this nicely after next get_analysable_data round + // (course->sortorder here). We can not check this nicely after next get_unlabelled_data round // because the first analysed element will be analysed again. $analysedelems = $DB->get_records('analytics_used_analysables', $params, 'timeanalysed ASC'); // Just a default for the first checked element. @@ -326,16 +329,21 @@ class analytics_model_testcase extends advanced_testcase { $last = $courses[$analysed->analysableid]; } - $analyser->get_analysable_data(false); - $this->assertGreaterThanOrEqual(5, $DB->count_records('analytics_used_analysables', $params)); + // No time limit now to process the rest. + set_config('modeltimelimit', 1000, 'analytics'); + + $analysis->run(); + $this->assertEquals(5, $DB->count_records('analytics_used_analysables', $params)); // New analysable elements are immediately pulled. $this->getDataGenerator()->create_course(); - $analyser->get_analysable_data(false); - $this->assertGreaterThanOrEqual(6, $DB->count_records('analytics_used_analysables', $params)); + $analysis->run(); + $this->assertEquals(6, $DB->count_records('analytics_used_analysables', $params)); // Training and prediction data do not get mixed. - $analyser->get_analysable_data(true); + $result = new \core_analytics\local\analysis\result_array(1, false, []); + $analysis = new test_analysis($analyser, false, $result); + $analysis->run(); $params = array('modelid' => 1, 'action' => 'training'); $this->assertLessThanOrEqual(2, $DB->count_records('analytics_used_analysables', $params)); } diff --git a/analytics/tests/prediction_test.php b/analytics/tests/prediction_test.php index 3dea3764393..ec85d83e0ea 100644 --- a/analytics/tests/prediction_test.php +++ b/analytics/tests/prediction_test.php @@ -27,6 +27,7 @@ defined('MOODLE_INTERNAL') || die(); global $CFG; require_once(__DIR__ . '/fixtures/test_indicator_max.php'); require_once(__DIR__ . '/fixtures/test_indicator_min.php'); +require_once(__DIR__ . '/fixtures/test_indicator_null.php'); require_once(__DIR__ . '/fixtures/test_indicator_fullname.php'); require_once(__DIR__ . '/fixtures/test_indicator_random.php'); require_once(__DIR__ . '/fixtures/test_target_shortname.php'); @@ -86,18 +87,14 @@ class core_analytics_prediction_testcase extends advanced_testcase { // 1 range for each analysable. $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id())); $this->assertCount(2, $predictedranges); - $this->assertEquals(1, $DB->count_records('analytics_used_files', - array('modelid' => $model->get_id(), 'action' => 'predicted'))); // 2 predictions for each range. $this->assertEquals(2, $DB->count_records('analytics_predictions', array('modelid' => $model->get_id()))); - // No new generated files nor records as there are no new courses available. + // No new generated records as there are no new courses available. $model->predict(); $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id())); $this->assertCount(2, $predictedranges); - $this->assertEquals(1, $DB->count_records('analytics_used_files', - array('modelid' => $model->get_id(), 'action' => 'predicted'))); $this->assertEquals(2, $DB->count_records('analytics_predictions', array('modelid' => $model->get_id()))); } @@ -553,8 +550,7 @@ class core_analytics_prediction_testcase extends advanced_testcase { public function test_not_null_samples() { $this->resetAfterTest(true); - $classname = '\core\analytics\time_splitting\quarters'; - $timesplitting = \core_analytics\manager::get_time_splitting($classname); + $timesplitting = \core_analytics\manager::get_time_splitting('\core\analytics\time_splitting\quarters'); $timesplitting->set_analysable(new \core_analytics\site()); $ranges = array( @@ -563,35 +559,49 @@ class core_analytics_prediction_testcase extends advanced_testcase { ); $samples = array(123 => 123, 321 => 321); - $indicator1 = $this->getMockBuilder('test_indicator_max') - ->setMethods(['calculate_sample']) - ->getMock(); - $indicator1->method('calculate_sample') - ->willReturn(null); + $target = \core_analytics\manager::get_target('test_target_shortname'); + $indicators = array('test_indicator_null', 'test_indicator_min'); + foreach ($indicators as $key => $indicator) { + $indicators[$key] = \core_analytics\manager::get_indicator($indicator); + } + $model = \core_analytics\model::create($target, $indicators, '\core\analytics\time_splitting\no_splitting'); - $indicator2 = \core_analytics\manager::get_indicator('test_indicator_min'); + $analyser = $model->get_analyser(); + $result = new \core_analytics\local\analysis\result_array($model->get_id(), false, $analyser->get_options()); + $analysis = new \core_analytics\analysis($analyser, false, $result); // Samples with at least 1 not null value are returned. $params = array( + $timesplitting, $samples, - 'whatever', - array($indicator1, $indicator2), $ranges ); - $dataset = phpunit_util::call_internal_method($timesplitting, 'calculate_indicators', $params, $classname); + $dataset = phpunit_util::call_internal_method($analysis, 'calculate_indicators', $params, + '\core_analytics\analysis'); $this->assertArrayHasKey('123-0', $dataset); $this->assertArrayHasKey('123-1', $dataset); $this->assertArrayHasKey('321-0', $dataset); $this->assertArrayHasKey('321-1', $dataset); + + $indicators = array('test_indicator_null'); + foreach ($indicators as $key => $indicator) { + $indicators[$key] = \core_analytics\manager::get_indicator($indicator); + } + $model = \core_analytics\model::create($target, $indicators, '\core\analytics\time_splitting\no_splitting'); + + $analyser = $model->get_analyser(); + $result = new \core_analytics\local\analysis\result_array($model->get_id(), false, $analyser->get_options()); + $analysis = new \core_analytics\analysis($analyser, false, $result); + // Samples with only null values are not returned. $params = array( + $timesplitting, $samples, - 'whatever', - array($indicator1), $ranges ); - $dataset = phpunit_util::call_internal_method($timesplitting, 'calculate_indicators', $params, $classname); + $dataset = phpunit_util::call_internal_method($analysis, 'calculate_indicators', $params, + '\core_analytics\analysis'); $this->assertArrayNotHasKey('123-0', $dataset); $this->assertArrayNotHasKey('123-1', $dataset); $this->assertArrayNotHasKey('321-0', $dataset); diff --git a/analytics/tests/privacy_test.php b/analytics/tests/privacy_test.php index eb3a9fe70ea..8b93bc180d3 100644 --- a/analytics/tests/privacy_test.php +++ b/analytics/tests/privacy_test.php @@ -34,7 +34,6 @@ require_once(__DIR__ . '/fixtures/test_indicator_max.php'); require_once(__DIR__ . '/fixtures/test_indicator_min.php'); require_once(__DIR__ . '/fixtures/test_target_site_users.php'); require_once(__DIR__ . '/fixtures/test_target_course_users.php'); -require_once(__DIR__ . '/fixtures/test_analyser.php'); /** * Unit tests for privacy. diff --git a/analytics/upgrade.txt b/analytics/upgrade.txt index d2036f20067..12aa5d9f23c 100644 --- a/analytics/upgrade.txt +++ b/analytics/upgrade.txt @@ -13,6 +13,36 @@ information provided here is intended especially for developers. has been replaced with automatic update of models provided by the core moodle component. There is no need to call this method explicitly any more. Instead, adding new models can be achieved by updating the lib/db/analytics.php file and bumping the core version. +* \core_analytics\model::execute_prediction_callbacks now returns an array with both sample's contexts + and the prediction records. +* Time splitting methods: + * \core_analytics\local\time_splitting\base::append_rangeindex and + \core_analytics\local\time_splitting\base::infer_sample_info are now marked as final and can not + be overwritten. + * Can now overwrite include_range_info_in_training_data() and + get_training_ranges() methods. They can be used to create time splitting methods with a pre-defined + number of ranges. + * Can now overwrite cache_indicator_calculations(). You should return false if the time frames generated + by your time-splitting method are unique and / or can hardly be reused by further models. + * \core_analytics\local\analyser\base::get_most_recent_prediction_range has been moved to + \core_analytics\local\time_splitting\base::get_most_recent_prediction_range and it is not overwritable + by time splitting methods. +* Targets: + * The visibility of the following methods must now be public: ignored_predicted_classes() + and get_insights_users() + * Prediction_actions() has now a 3rd parameter $isinsightuser. This parameter is true + when we are listing actions for the user that will receives the insight. + * Can now implement a always_update_analysis_time() method so analysable elements' timeanalysed is + only updated when analysable elements have been successfully evaluated. It is useful for lightweight targets. + * Can not implement two new methods to tune the insights generated by the model: get_insight_subject() + and get_insight_context_url(). +* Analysers: + * The visibility of get_all_samples() method must now be public. + * get_analysables() method has been deprecated in favour of a new get_analysables_interator() + for performance reasons. + * Can overwrite a new one_sample_per_analysable() method if the analysables they use only have + one sample. The insights generated by models will then include the suggested actions in + the notification. === 3.5 === diff --git a/backup/backup.class.php b/backup/backup.class.php index 0915c8e8d0a..a287a96b18f 100644 --- a/backup/backup.class.php +++ b/backup/backup.class.php @@ -74,6 +74,12 @@ abstract class backup implements checksumable { const MODE_AUTOMATED = 50; const MODE_CONVERTED = 60; + /** + * This mode is for asynchronous backups. + * These backups will run via adhoc scheduled tasks. + */ + const MODE_ASYNC = 70; + // Target (new/existing/current/adding/deleting) const TARGET_CURRENT_DELETING = 0; const TARGET_CURRENT_ADDING = 1; diff --git a/backup/backup.php b/backup/backup.php index c1b1f648f50..e0606d1b384 100644 --- a/backup/backup.php +++ b/backup/backup.php @@ -30,16 +30,27 @@ require_once('../config.php'); require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); require_once($CFG->dirroot . '/backup/moodle2/backup_plan_builder.class.php'); +// Backup of large courses requires extra memory. Use the amount configured +// in admin settings. +raise_memory_limit(MEMORY_EXTRA); $courseid = required_param('id', PARAM_INT); $sectionid = optional_param('section', null, PARAM_INT); $cmid = optional_param('cm', null, PARAM_INT); $cancel = optional_param('cancel', '', PARAM_ALPHA); +$previous = optional_param('previous', false, PARAM_BOOL); /** * Part of the forms in stages after initial, is POST never GET */ $backupid = optional_param('backup', false, PARAM_ALPHANUM); +// Determine if we are performing realtime for asynchronous backups. +$backupmode = backup::MODE_GENERAL; +if (async_helper::is_async_enabled()) { + $backupmode = backup::MODE_ASYNC; +} + +$courseurl = new moodle_url('/course/view.php', array('id' => $courseid)); $url = new moodle_url('/backup/backup.php', array('id'=>$courseid)); if ($sectionid !== null) { $url->param('section', $sectionid); @@ -53,6 +64,8 @@ $PAGE->set_pagelayout('admin'); $id = $courseid; $cm = null; $course = $DB->get_record('course', array('id'=>$courseid), '*', MUST_EXIST); +$coursecontext = context_course::instance($course->id); +$contextid = $coursecontext->id; $type = backup::TYPE_1COURSE; if (!is_null($sectionid)) { $section = $DB->get_record('course_sections', array('course'=>$course->id, 'id'=>$sectionid), '*', MUST_EXIST); @@ -68,11 +81,10 @@ require_login($course, false, $cm); switch ($type) { case backup::TYPE_1COURSE : - require_capability('moodle/backup:backupcourse', context_course::instance($course->id)); + require_capability('moodle/backup:backupcourse', $coursecontext); $heading = get_string('backupcourse', 'backup', $course->shortname); break; case backup::TYPE_1SECTION : - $coursecontext = context_course::instance($course->id); require_capability('moodle/backup:backupsection', $coursecontext); if ((string)$section->name !== '') { $sectionname = format_string($section->name, true, array('context' => $coursecontext)); @@ -84,102 +96,145 @@ switch ($type) { } break; case backup::TYPE_1ACTIVITY : - require_capability('moodle/backup:backupactivity', context_module::instance($cm->id)); + $activitycontext = context_module::instance($cm->id); + require_capability('moodle/backup:backupactivity', $activitycontext); + $contextid = $activitycontext->id; $heading = get_string('backupactivity', 'backup', $cm->name); break; default : print_error('unknownbackuptype'); } -// Backup of large courses requires extra memory. Use the amount configured -// in admin settings. -raise_memory_limit(MEMORY_EXTRA); - -if (!($bc = backup_ui::load_controller($backupid))) { - $bc = new backup_controller($type, $id, backup::FORMAT_MOODLE, - backup::INTERACTIVE_YES, backup::MODE_GENERAL, $USER->id); -} -$backup = new backup_ui($bc); - $PAGE->set_title($heading); $PAGE->set_heading($heading); -$renderer = $PAGE->get_renderer('core','backup'); if (empty($cancel)) { // Do not print the header if user cancelled the process, as we are going to redirect the user. echo $OUTPUT->header(); } -// Prepare a progress bar which can display optionally during long-running -// operations while setting up the UI. -$slowprogress = new \core\progress\display_if_slow(get_string('preparingui', 'backup')); +// Only let user perform a backup if we aren't in async mode, or if we are +// and there are no pending backups for this item for this user. +if (!async_helper::is_async_pending($id, 'course', 'backup')) { -$previous = optional_param('previous', false, PARAM_BOOL); -if ($backup->get_stage() == backup_ui::STAGE_SCHEMA && !$previous) { - // After schema stage, we are probably going to get to the confirmation stage, - // The confirmation stage has 2 sets of progress, so this is needed to prevent - // it showing 2 progress bars. - $twobars = true; - $slowprogress->start_progress('', 2); -} else { - $twobars = false; -} -$backup->get_controller()->set_progress($slowprogress); -$backup->process(); + // The mix of business logic and display elements below makes me sad. + // This needs to refactored into the renderer and seperated out. -if ($backup->enforce_changed_dependencies()) { - debugging('Your settings have been altered due to unmet dependencies', DEBUG_DEVELOPER); -} - -$loghtml = ''; -if ($backup->get_stage() == backup_ui::STAGE_FINAL) { - // Display an extra backup step bar so that we can show the 'processing' step first. - echo html_writer::start_div('', array('id' => 'executionprogress')); - echo $renderer->progress_bar($backup->get_progress_bar()); - $backup->get_controller()->set_progress(new \core\progress\display()); - - // Prepare logger and add to end of chain. - $logger = new core_backup_html_logger($CFG->debugdeveloper ? backup::LOG_DEBUG : backup::LOG_INFO); - $backup->get_controller()->add_logger($logger); - - // Carry out actual backup. - $backup->execute(); - - // Backup controller gets saved/loaded so the logger object changes and we - // have to retrieve it. - $logger = $backup->get_controller()->get_logger(); - while (!is_a($logger, 'core_backup_html_logger')) { - $logger = $logger->get_next(); + if (!($bc = backup_ui::load_controller($backupid))) { + $bc = new backup_controller($type, $id, backup::FORMAT_MOODLE, + backup::INTERACTIVE_YES, $backupmode, $USER->id); } - // Get HTML from logger. - if ($CFG->debugdisplay) { - $loghtml = $logger->get_html(); + // Prepare a progress bar which can display optionally during long-running + // operations while setting up the UI. + $slowprogress = new \core\progress\display_if_slow(get_string('preparingui', 'backup')); + $renderer = $PAGE->get_renderer('core', 'backup'); + $backup = new backup_ui($bc); + + if ($backup->get_stage() == backup_ui::STAGE_SCHEMA && !$previous) { + // After schema stage, we are probably going to get to the confirmation stage, + // The confirmation stage has 2 sets of progress, so this is needed to prevent + // it showing 2 progress bars. + $twobars = true; + $slowprogress->start_progress('', 2); + } else { + $twobars = false; + } + $backup->get_controller()->set_progress($slowprogress); + $backup->process(); + + if ($backup->enforce_changed_dependencies()) { + debugging('Your settings have been altered due to unmet dependencies', DEBUG_DEVELOPER); } - // Hide the progress display and first backup step bar (the 'finished' step will show next). - echo html_writer::end_div(); - echo html_writer::script('document.getElementById("executionprogress").style.display = "none";'); -} else { - $backup->save_controller(); -} + $loghtml = ''; + if ($backup->get_stage() == backup_ui::STAGE_FINAL) { -// Displaying UI can require progress reporting, so do it here before outputting -// the backup stage bar (as part of the existing progress bar, if required). -$ui = $backup->display($renderer); -if ($twobars) { - $slowprogress->end_progress(); -} + if ($backupmode != backup::MODE_ASYNC) { + // Synchronous backup handling. -echo $renderer->progress_bar($backup->get_progress_bar()); + // Display an extra backup step bar so that we can show the 'processing' step first. + echo html_writer::start_div('', array('id' => 'executionprogress')); + echo $renderer->progress_bar($backup->get_progress_bar()); + $backup->get_controller()->set_progress(new \core\progress\display()); -echo $ui; -$backup->destroy(); -unset($backup); + // Prepare logger and add to end of chain. + $logger = new core_backup_html_logger($CFG->debugdeveloper ? backup::LOG_DEBUG : backup::LOG_INFO); + $backup->get_controller()->add_logger($logger); -// Display log data if there was any. -if ($loghtml != '') { - echo $renderer->log_display($loghtml); + // Carry out actual backup. + $backup->execute(); + + // Backup controller gets saved/loaded so the logger object changes and we + // have to retrieve it. + $logger = $backup->get_controller()->get_logger(); + while (!is_a($logger, 'core_backup_html_logger')) { + $logger = $logger->get_next(); + } + + // Get HTML from logger. + if ($CFG->debugdisplay) { + $loghtml = $logger->get_html(); + } + + // Hide the progress display and first backup step bar (the 'finished' step will show next). + echo html_writer::end_div(); + echo html_writer::script('document.getElementById("executionprogress").style.display = "none";'); + } else { + // Async backup handling. + $backup->get_controller()->finish_ui(); + + echo html_writer::start_div('', array('id' => 'executionprogress')); + echo $renderer->progress_bar($backup->get_progress_bar()); + echo html_writer::end_div(); + + // Create adhoc task for backup. + $asynctask = new \core\task\asynchronous_backup_task(); + $asynctask->set_blocking(false); + $asynctask->set_custom_data(array('backupid' => $backupid)); + \core\task\manager::queue_adhoc_task($asynctask); + + // Add ajax progress bar and initiate ajax via a template. + $restoreurl = new moodle_url('/backup/restorefile.php', array('contextid' => $contextid)); + $progresssetup = array( + 'backupid' => $backupid, + 'contextid' => $contextid, + 'courseurl' => $courseurl->out(), + 'restoreurl' => $restoreurl->out(), + 'headingident' => 'backup' + ); + echo $renderer->render_from_template('core/async_backup_status', $progresssetup); + } + + } else { + $backup->save_controller(); + } + + if ($backup->get_stage() != backup_ui::STAGE_FINAL) { + + // Displaying UI can require progress reporting, so do it here before outputting + // the backup stage bar (as part of the existing progress bar, if required). + $ui = $backup->display($renderer); + if ($twobars) { + $slowprogress->end_progress(); + } + + echo $renderer->progress_bar($backup->get_progress_bar()); + echo $ui; + + // Display log data if there was any. + if ($loghtml != '' && $backupmode != backup::MODE_ASYNC) { + echo $renderer->log_display($loghtml); + } + } + + $backup->destroy(); + unset($backup); + +} else { // User has a pending async operation. + echo $OUTPUT->notification(get_string('pendingasyncerror', 'backup'), 'error'); + echo $OUTPUT->container(get_string('pendingasyncdetail', 'backup')); + echo $OUTPUT->continue_button($courseurl); } echo $OUTPUT->footer(); diff --git a/backup/controller/backup_controller.class.php b/backup/controller/backup_controller.class.php index 35edd9c318a..8ea8d517f91 100644 --- a/backup/controller/backup_controller.class.php +++ b/backup/controller/backup_controller.class.php @@ -58,7 +58,11 @@ class backup_controller extends base_controller { protected $plan; // Backup execution plan protected $includefiles; // Whether this backup includes files or not. - protected $execution; // inmediate/delayed + /** + * Immediate/delayed execution type. + * @var integer + */ + protected $execution; protected $executiontime; // epoch time when we want the backup to be executed (requires cron to run) protected $destination; // Destination chain object (fs_moodle, fs_os, db, email...) @@ -85,11 +89,17 @@ class backup_controller extends base_controller { $this->userid = $userid; // Apply some defaults - $this->execution = backup::EXECUTION_INMEDIATE; $this->operation = backup::OPERATION_BACKUP; $this->executiontime = 0; $this->checksum = ''; + // Set execution based on backup mode. + if ($mode == backup::MODE_ASYNC) { + $this->execution = backup::EXECUTION_DELAYED; + } else { + $this->execution = backup::EXECUTION_INMEDIATE; + } + // Apply current backup version and release if necessary backup_controller_dbops::apply_version_and_release(); @@ -112,7 +122,7 @@ class backup_controller extends base_controller { // display progress must set it. $this->progress = new \core\progress\none(); - // Instantiate the output_controller singleton and active it if interactive and inmediate + // Instantiate the output_controller singleton and active it if interactive and immediate. $oc = output_controller::get_instance(); if ($this->interactive == backup::INTERACTIVE_YES && $this->execution == backup::EXECUTION_INMEDIATE) { $oc->set_active(true); @@ -182,7 +192,8 @@ class backup_controller extends base_controller { // TODO: Check it's a correct status. $this->status = $status; // Ensure that, once set to backup::STATUS_AWAITING, controller is stored in DB. - if ($status == backup::STATUS_AWAITING) { + // Also save if executing so we can better track progress. + if ($status == backup::STATUS_AWAITING || $status == backup::STATUS_EXECUTING) { $this->save_controller(); $tbc = self::load_controller($this->backupid); $this->logger = $tbc->logger; // wakeup loggers @@ -192,14 +203,18 @@ class backup_controller extends base_controller { // If the operation has ended without error (backup::STATUS_FINISHED_OK) // proceed by cleaning the object from database. MDL-29262. $this->save_controller(false, true); + } else if ($status == backup::STATUS_FINISHED_ERR) { + // If the operation has ended with an error save the controller + // preserving the object in the database. We may want it for debugging. + $this->save_controller(); } } public function set_execution($execution, $executiontime = 0) { $this->log('setting controller execution', backup::LOG_DEBUG); - // TODO: Check valid execution mode - // TODO: Check time in future - // TODO: Check time = 0 if inmediate + // TODO: Check valid execution mode. + // TODO: Check time in future. + // TODO: Check time = 0 if immediate. $this->execution = $execution; $this->executiontime = $executiontime; @@ -333,8 +348,8 @@ class backup_controller extends base_controller { * @param bool $cleanobj to decide if the object itself must be cleaned (true) or no (false) */ public function save_controller($includeobj = true, $cleanobj = false) { - // Going to save controller to persistent storage, calculate checksum for later checks and save it - // TODO: flag the controller as NA. Any operation on it should be forbidden util loaded back + // Going to save controller to persistent storage, calculate checksum for later checks and save it. + // TODO: flag the controller as NA. Any operation on it should be forbidden until loaded back. $this->log('saving controller to db', backup::LOG_DEBUG); if ($includeobj ) { // Only calculate checksum if we are going to include the object. $this->checksum = $this->calculate_checksum(); @@ -399,6 +414,7 @@ class backup_controller extends base_controller { $this->log("setting file inclusion to {$this->includefiles}", backup::LOG_DEBUG); return $this->includefiles; } + } /* diff --git a/backup/controller/restore_controller.class.php b/backup/controller/restore_controller.class.php index 2e994f44def..cc5b86993d2 100644 --- a/backup/controller/restore_controller.class.php +++ b/backup/controller/restore_controller.class.php @@ -53,7 +53,11 @@ class restore_controller extends base_controller { /** @var restore_plan */ protected $plan; // Restore execution plan - protected $execution; // inmediate/delayed + /** + * Immediate/delayed execution type. + * @var integer + */ + protected $execution; protected $executiontime; // epoch time when we want the restore to be executed (requires cron to run) protected $checksum; // Cache @checksumable results for lighter @is_checksum_correct() uses @@ -88,7 +92,6 @@ class restore_controller extends base_controller { // Apply some defaults $this->type = ''; $this->format = backup::FORMAT_UNKNOWN; - $this->execution = backup::EXECUTION_INMEDIATE; $this->operation = backup::OPERATION_RESTORE; $this->executiontime = 0; $this->samesite = false; @@ -110,6 +113,13 @@ class restore_controller extends base_controller { // Default logger chain (based on interactive/execution) $this->logger = backup_factory::get_logger_chain($this->interactive, $this->execution, $this->restoreid); + // Set execution based on backup mode. + if ($mode == backup::MODE_ASYNC) { + $this->execution = backup::EXECUTION_DELAYED; + } else { + $this->execution = backup::EXECUTION_INMEDIATE; + } + // By default there is no progress reporter unless you specify one so it // can be used during loading of the plan. if ($progress) { @@ -119,7 +129,7 @@ class restore_controller extends base_controller { } $this->progress->start_progress('Constructing restore_controller'); - // Instantiate the output_controller singleton and active it if interactive and inmediate + // Instantiate the output_controller singleton and active it if interactive and immediate. $oc = output_controller::get_instance(); if ($this->interactive == backup::INTERACTIVE_YES && $this->execution == backup::EXECUTION_INMEDIATE) { $oc->set_active(true); @@ -198,7 +208,8 @@ class restore_controller extends base_controller { // TODO: Check it's a correct status. $this->status = $status; // Ensure that, once set to backup::STATUS_AWAITING | STATUS_NEED_PRECHECK, controller is stored in DB. - if ($status == backup::STATUS_AWAITING || $status == backup::STATUS_NEED_PRECHECK) { + // Also save if executing so we can better track progress. + if ($status == backup::STATUS_AWAITING || $status == backup::STATUS_NEED_PRECHECK || $status == backup::STATUS_EXECUTING) { $this->save_controller(); $tbc = self::load_controller($this->restoreid); $this->logger = $tbc->logger; // wakeup loggers @@ -208,14 +219,18 @@ class restore_controller extends base_controller { // If the operation has ended without error (backup::STATUS_FINISHED_OK) // proceed by cleaning the object from database. MDL-29262. $this->save_controller(false, true); + } else if ($status == backup::STATUS_FINISHED_ERR) { + // If the operation has ended with an error save the controller + // preserving the object in the database. We may want it for debugging. + $this->save_controller(); } } public function set_execution($execution, $executiontime = 0) { $this->log('setting controller execution', backup::LOG_DEBUG); - // TODO: Check valid execution mode - // TODO: Check time in future - // TODO: Check time = 0 if inmediate + // TODO: Check valid execution mode. + // TODO: Check time in future. + // TODO: Check time = 0 if immediate. $this->execution = $execution; $this->executiontime = $executiontime; diff --git a/backup/externallib.php b/backup/externallib.php new file mode 100644 index 00000000000..98119ecaa4e --- /dev/null +++ b/backup/externallib.php @@ -0,0 +1,240 @@ +. + + +/** + * External backup API. + * + * @package core_backup + * @category external + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die; + +require_once("$CFG->libdir/externallib.php"); +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); + +/** + * Backup external functions. + * + * @package core_backup + * @category external + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @since Moodle 3.7 + */ +class core_backup_external extends external_api { + + /** + * Returns description of method parameters + * + * @return external_function_parameters + * @since Moodle 3.7 + */ + public static function get_async_backup_progress_parameters() { + return new external_function_parameters( + array( + 'backupids' => new external_multiple_structure( + new external_value(PARAM_ALPHANUM, 'Backup id to get progress for', VALUE_REQUIRED, null, NULL_ALLOWED), + 'Backup id to get progress for', VALUE_REQUIRED + ), + 'contextid' => new external_value(PARAM_INT, 'Context id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED), + ) + ); + } + + /** + * Get asynchronous backup progress. + * + * @param string $backupids The ids of the backup to get progress for. + * @param int $contextid The context the backup relates to. + * @return array $results The array of results. + * @since Moodle 3.7 + */ + public static function get_async_backup_progress($backupids, $contextid) { + global $CFG; + require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); + require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php'); + + // Parameter validation. + self::validate_parameters( + self::get_async_backup_progress_parameters(), + array( + 'backupids' => $backupids, + 'contextid' => $contextid + ) + ); + + // Context validation. + list($context, $course, $cm) = get_context_info_array($contextid); + self::validate_context($context); + + if ($cm) { + require_capability('moodle/backup:backupactivity', $context); + } else { + require_capability('moodle/backup:backupcourse', $context); + $instanceid = $course->id; + } + + $results = array(); + foreach ($backupids as $backupid) { + $results[] = backup_controller_dbops::get_progress($backupid); + } + + return $results; + } + + /** + * Returns description of method result value + * + * @return external_description + * @since Moodle 3.7 + */ + public static function get_async_backup_progress_returns() { + return new external_multiple_structure( + new external_single_structure( + array( + 'status' => new external_value(PARAM_INT, 'Backup Status'), + 'progress' => new external_value(PARAM_FLOAT, 'Backup progress'), + 'backupid' => new external_value(PARAM_ALPHANUM, 'Backup id'), + 'operation' => new external_value(PARAM_ALPHANUM, 'operation type'), + ), 'Backup completion status' + ), 'Backup data' + ); + } + + /** + * Returns description of method parameters + * + * @return external_function_parameters + * @since Moodle 3.7 + */ + public static function get_async_backup_links_backup_parameters() { + return new external_function_parameters( + array( + 'filename' => new external_value(PARAM_FILE, 'Backup filename', VALUE_REQUIRED, null, NULL_NOT_ALLOWED), + 'contextid' => new external_value(PARAM_INT, 'Context id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED), + ) + ); + } + + /** + * Get the data to be used when generating the table row for an asynchronous backup, + * the table row updates via ajax when backup is complete. + * + * @param string $filename The file name of the backup file. + * @param int $contextid The context the backup relates to. + * @since Moodle 3.7 + */ + public static function get_async_backup_links_backup($filename, $contextid) { + // Parameter validation. + self::validate_parameters( + self::get_async_backup_links_backup_parameters(), + array( + 'filename' => $filename, + 'contextid' => $contextid + ) + ); + + // Context validation. + list($context, $course, $cm) = get_context_info_array($contextid); + self::validate_context($context); + require_capability('moodle/backup:backupcourse', $context); + + if ($cm) { + $filearea = 'activity'; + } else { + $filearea = 'course'; + } + + $results = \async_helper::get_backup_file_info($filename, $filearea, $contextid); + + return $results; + } + + /** + * Returns description of method result value. + * + * @return external_description + * @since Moodle 3.7 + */ + public static function get_async_backup_links_backup_returns() { + return new external_single_structure( + array( + 'filesize' => new external_value(PARAM_TEXT, 'Backup file size'), + 'fileurl' => new external_value(PARAM_URL, 'Backup file URL'), + 'restoreurl' => new external_value(PARAM_URL, 'Backup restore URL'), + ), 'Table row data.'); + } + /** + * Returns description of method parameters + * + * @return external_function_parameters + * @since Moodle 3.7 + */ + public static function get_async_backup_links_restore_parameters() { + return new external_function_parameters( + array( + 'backupid' => new external_value(PARAM_ALPHANUMEXT, 'Backup id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED), + 'contextid' => new external_value(PARAM_INT, 'Context id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED), + ) + ); + } + + /** + * Get the data to be used when generating the table row for an asynchronous restore, + * the table row updates via ajax when restore is complete. + * + * @param string $backupid The id of the backup record. + * @param int $contextid The context the restore relates to. + * @return array $results The array of results. + * @since Moodle 3.7 + */ + public static function get_async_backup_links_restore($backupid, $contextid) { + // Parameter validation. + self::validate_parameters( + self::get_async_backup_links_restore_parameters(), + array( + 'backupid' => $backupid, + 'contextid' => $contextid + ) + ); + + // Context validation. + $context = context::instance_by_id($contextid); + self::validate_context($context); + require_capability('moodle/restore:restorecourse', $context); + + $results = \async_helper::get_restore_url($backupid); + + return $results; + } + + /** + * Returns description of method result value. + * + * @return external_description + * @since Moodle 3.7 + */ + public static function get_async_backup_links_restore_returns() { + return new external_single_structure( + array( + 'restoreurl' => new external_value(PARAM_URL, 'Restore url'), + ), 'Table row data.'); + } +} diff --git a/backup/restore.php b/backup/restore.php index 06f58e65a34..4ee134e0093 100644 --- a/backup/restore.php +++ b/backup/restore.php @@ -28,10 +28,20 @@ define('NO_OUTPUT_BUFFERING', true); require_once('../config.php'); require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php'); +// Restore of large courses requires extra memory. Use the amount configured +// in admin settings. +raise_memory_limit(MEMORY_EXTRA); + $contextid = required_param('contextid', PARAM_INT); $stage = optional_param('stage', restore_ui::STAGE_CONFIRM, PARAM_INT); $cancel = optional_param('cancel', '', PARAM_ALPHA); +// Determine if we are performing realtime for asynchronous backups. +$backupmode = backup::MODE_GENERAL; +if (async_helper::is_async_enabled()) { + $backupmode = backup::MODE_ASYNC; +} + list($context, $course, $cm) = get_context_info_array($contextid); navigation_node::override_active_url(new moodle_url('/backup/restorefile.php', array('contextid'=>$contextid))); @@ -70,10 +80,6 @@ $slowprogress->start_progress('', 10); // This progress section counts for loading the restore controller. $slowprogress->start_progress('', 1, 1); -// Restore of large courses requires extra memory. Use the amount configured -// in admin settings. -raise_memory_limit(MEMORY_EXTRA); - if ($stage & restore_ui::STAGE_CONFIRM + restore_ui::STAGE_DESTINATION) { $restore = restore_ui::engage_independent_stage($stage, $contextid); } else { @@ -83,7 +89,7 @@ if ($stage & restore_ui::STAGE_CONFIRM + restore_ui::STAGE_DESTINATION) { $restore = restore_ui::engage_independent_stage($stage/2, $contextid); if ($restore->process()) { $rc = new restore_controller($restore->get_filepath(), $restore->get_course_id(), backup::INTERACTIVE_YES, - backup::MODE_GENERAL, $USER->id, $restore->get_target()); + $backupmode, $USER->id, $restore->get_target()); } } if ($rc) { @@ -120,7 +126,7 @@ if (!$restore->is_independent()) { // Use a temporary (disappearing) progress bar to show the precheck progress if any. $precheckprogress = new \core\progress\display_if_slow(get_string('preparingdata', 'backup')); $restore->get_controller()->set_progress($precheckprogress); - if ($restore->get_stage() == restore_ui::STAGE_PROCESS && !$restore->requires_substage()) { + if ($restore->get_stage() == restore_ui::STAGE_PROCESS && !$restore->requires_substage() && $backupmode != backup::MODE_ASYNC) { try { // Div used to hide the 'progress' step once the page gets onto 'finished'. echo html_writer::start_div('', array('id' => 'executionprogress')); @@ -150,7 +156,35 @@ if (!$restore->is_independent()) { } echo $renderer->progress_bar($restore->get_progress_bar()); -echo $restore->display($renderer); + +if ($restore->get_stage() != restore_ui::STAGE_PROCESS) { + echo $restore->display($renderer); +} else if ($restore->get_stage() == restore_ui::STAGE_PROCESS && $restore->requires_substage()) { + echo $restore->display($renderer); +} else if ($restore->get_stage() == restore_ui::STAGE_PROCESS + && !$restore->requires_substage() + && $backupmode == backup::MODE_ASYNC) { + // Asynchronous restore. + // Create adhoc task for restore. + $restoreid = $restore->get_restoreid(); + $asynctask = new \core\task\asynchronous_restore_task(); + $asynctask->set_blocking(false); + $asynctask->set_custom_data(array('backupid' => $restoreid)); + \core\task\manager::queue_adhoc_task($asynctask); + + // Add ajax progress bar and initiate ajax via a template. + $courseurl = new moodle_url('/course/view.php', array('id' => $course->id)); + $restoreurl = new moodle_url('/backup/restorefile.php', array('contextid' => $contextid)); + $progresssetup = array( + 'backupid' => $restoreid, + 'contextid' => $contextid, + 'courseurl' => $courseurl->out(), + 'restoreurl' => $restoreurl->out() + ); + echo $renderer->render_from_template('core/async_backup_status', $progresssetup); + +} + $restore->destroy(); unset($restore); diff --git a/backup/restorefile.php b/backup/restorefile.php index 2a745a829e1..eef560bf06c 100644 --- a/backup/restorefile.php +++ b/backup/restorefile.php @@ -114,6 +114,7 @@ $PAGE->set_context($context); $PAGE->set_title(get_string('course') . ': ' . $coursefullname); $PAGE->set_heading($heading); $PAGE->set_pagelayout('admin'); +$PAGE->requires->js_call_amd('core_backup/async_backup', 'asyncBackupAllStatus', array($context->id)); $form = new course_restore_form(null, array('contextid'=>$contextid)); $data = $form->get_data(); @@ -128,8 +129,6 @@ if ($data && has_capability('moodle/restore:uploadfile', $context)) { die; } - - echo $OUTPUT->header(); // require uploadfile cap to use file picker @@ -196,4 +195,13 @@ if (!empty($automatedbackups)) { echo $OUTPUT->container_end(); } +// In progress course restores. +if (async_helper::is_async_enabled()) { + echo $OUTPUT->heading_with_help(get_string('asyncrestoreinprogress', 'backup'), 'asyncrestoreinprogress', 'backup'); + echo $OUTPUT->container_start(); + $renderer = $PAGE->get_renderer('core', 'backup'); + echo $renderer->restore_progress_viewer($USER->id, $context); + echo $OUTPUT->container_end(); +} + echo $OUTPUT->footer(); diff --git a/backup/tests/async_backup_test.php b/backup/tests/async_backup_test.php new file mode 100644 index 00000000000..f982a6d497e --- /dev/null +++ b/backup/tests/async_backup_test.php @@ -0,0 +1,118 @@ +. + +/** + * Asyncronhous backup tests. + * + * @package core_backup + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); +require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php'); +require_once($CFG->libdir . '/completionlib.php'); + +/** + * Asyncronhous backup tests. + * + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_backup_async_backup_testcase extends \core_privacy\tests\provider_testcase { + + /** + * Tests the asynchronous backup. + */ + public function test_async_backup() { + global $DB, $CFG, $USER; + + $this->resetAfterTest(true); + $this->setAdminUser(); + $CFG->enableavailability = true; + $CFG->enablecompletion = true; + + // Create a course with some availability data set. + $generator = $this->getDataGenerator(); + $course = $generator->create_course( + array('format' => 'topics', 'numsections' => 3, + 'enablecompletion' => COMPLETION_ENABLED), + array('createsections' => true)); + $forum = $generator->create_module('forum', array( + 'course' => $course->id)); + $forum2 = $generator->create_module('forum', array( + 'course' => $course->id, 'completion' => COMPLETION_TRACKING_MANUAL)); + + // We need a grade, easiest is to add an assignment. + $assignrow = $generator->create_module('assign', array( + 'course' => $course->id)); + $assign = new assign(context_module::instance($assignrow->cmid), false, false); + $item = $assign->get_grade_item(); + + // Make a test grouping as well. + $grouping = $generator->create_grouping(array('courseid' => $course->id, + 'name' => 'Grouping!')); + + $availability = '{"op":"|","show":false,"c":[' . + '{"type":"completion","cm":' . $forum2->cmid .',"e":1},' . + '{"type":"grade","id":' . $item->id . ',"min":4,"max":94},' . + '{"type":"grouping","id":' . $grouping->id . '}' . + ']}'; + $DB->set_field('course_modules', 'availability', $availability, array( + 'id' => $forum->cmid)); + $DB->set_field('course_sections', 'availability', $availability, array( + 'course' => $course->id, 'section' => 1)); + + // Start backup process. + + // Make the backup controller for an async backup. + $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE, + backup::INTERACTIVE_YES, backup::MODE_ASYNC, $USER->id); + $bc->finish_ui(); + $backupid = $bc->get_backupid(); + + $prebackuprec = $DB->get_record('backup_controllers', array('backupid' => $backupid)); + + // Check the initial backup controller was created correctly. + $this->assertEquals(backup::STATUS_AWAITING, $prebackuprec->status); + $this->assertEquals(2, $prebackuprec->execution); + + // Create the adhoc task. + $asynctask = new \core\task\asynchronous_backup_task(); + $asynctask->set_blocking(false); + $asynctask->set_custom_data(array('backupid' => $backupid)); + \core\task\manager::queue_adhoc_task($asynctask); + + // We are expecting trace output during this test. + $this->expectOutputRegex("/$backupid/"); + + // Execute adhoc task. + $now = time(); + $task = \core\task\manager::get_next_adhoc_task($now); + $this->assertInstanceOf('\\core\\task\\asynchronous_backup_task', $task); + $task->execute(); + \core\task\manager::adhoc_task_complete($task); + + $postbackuprec = $DB->get_record('backup_controllers', array('backupid' => $backupid)); + + // Check backup was created successfully. + $this->assertEquals(backup::STATUS_FINISHED_OK, $postbackuprec->status); + $this->assertEquals(1.0, $postbackuprec->progress); + } +} diff --git a/backup/tests/async_restore_test.php b/backup/tests/async_restore_test.php new file mode 100644 index 00000000000..1121e371ad5 --- /dev/null +++ b/backup/tests/async_restore_test.php @@ -0,0 +1,140 @@ +. + +/** + * Asyncronhous restore tests. + * + * @package core_backup + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); +require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php'); +require_once($CFG->libdir . '/completionlib.php'); + +/** + * Asyncronhous restore tests. + * + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_backup_async_restore_testcase extends \core_privacy\tests\provider_testcase { + + /** + * Tests the asynchronous backup. + */ + public function test_async_restore() { + global $DB, $CFG, $USER; + + $this->resetAfterTest(true); + $this->setAdminUser(); + $CFG->enableavailability = true; + $CFG->enablecompletion = true; + + // Create a course with some availability data set. + $generator = $this->getDataGenerator(); + $course = $generator->create_course( + array('format' => 'topics', 'numsections' => 3, + 'enablecompletion' => COMPLETION_ENABLED), + array('createsections' => true)); + $forum = $generator->create_module('forum', array( + 'course' => $course->id)); + $forum2 = $generator->create_module('forum', array( + 'course' => $course->id, 'completion' => COMPLETION_TRACKING_MANUAL)); + + // We need a grade, easiest is to add an assignment. + $assignrow = $generator->create_module('assign', array( + 'course' => $course->id)); + $assign = new assign(context_module::instance($assignrow->cmid), false, false); + $item = $assign->get_grade_item(); + + // Make a test grouping as well. + $grouping = $generator->create_grouping(array('courseid' => $course->id, + 'name' => 'Grouping!')); + + $availability = '{"op":"|","show":false,"c":[' . + '{"type":"completion","cm":' . $forum2->cmid .',"e":1},' . + '{"type":"grade","id":' . $item->id . ',"min":4,"max":94},' . + '{"type":"grouping","id":' . $grouping->id . '}' . + ']}'; + $DB->set_field('course_modules', 'availability', $availability, array( + 'id' => $forum->cmid)); + $DB->set_field('course_sections', 'availability', $availability, array( + 'course' => $course->id, 'section' => 1)); + + // Backup the course. + $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE, + backup::INTERACTIVE_YES, backup::MODE_GENERAL, $USER->id); + $bc->finish_ui(); + $backupid = $bc->get_backupid(); + $bc->execute_plan(); + $bc->destroy(); + + // Get the backup file. + $coursecontext = context_course::instance($course->id); + $fs = get_file_storage(); + $files = $fs->get_area_files($coursecontext->id, 'backup', 'course', false, 'id ASC'); + $backupfile = reset($files); + + // Extract backup file. + $backupdir = "restore_" . uniqid(); + $path = $CFG->tempdir . DIRECTORY_SEPARATOR . "backup" . DIRECTORY_SEPARATOR . $backupdir; + + $fp = get_file_packer('application/vnd.moodle.backup'); + $fp->extract_to_pathname($backupfile, $path); + + // Create restore controller. + $newcourseid = restore_dbops::create_new_course( + $course->fullname, $course->shortname . '_2', $course->category); + $rc = new restore_controller($backupdir, $newcourseid, + backup::INTERACTIVE_NO, backup::MODE_ASYNC, $USER->id, + backup::TARGET_NEW_COURSE); + + $this->assertTrue($rc->execute_precheck()); + $restoreid = $rc->get_restoreid(); + + $prerestorerec = $DB->get_record('backup_controllers', array('backupid' => $restoreid)); + $prerestorerec->controller = ''; + + $rc->destroy(); + + // Create the adhoc task. + $asynctask = new \core\task\asynchronous_restore_task(); + $asynctask->set_blocking(false); + $asynctask->set_custom_data(array('backupid' => $restoreid)); + \core\task\manager::queue_adhoc_task($asynctask); + + // We are expecting trace output during this test. + $this->expectOutputRegex("/$restoreid/"); + + // Execute adhoc task. + $now = time(); + $task = \core\task\manager::get_next_adhoc_task($now); + $this->assertInstanceOf('\\core\\task\\asynchronous_restore_task', $task); + $task->execute(); + \core\task\manager::adhoc_task_complete($task); + + $postrestorerec = $DB->get_record('backup_controllers', array('backupid' => $restoreid)); + + // Check backup was created successfully. + $this->assertEquals(backup::STATUS_FINISHED_OK, $postrestorerec->status); + $this->assertEquals(1.0, $postrestorerec->progress); + } +} diff --git a/backup/util/dbops/backup_controller_dbops.class.php b/backup/util/dbops/backup_controller_dbops.class.php index d203db45d3b..3f94cd0eb15 100644 --- a/backup/util/dbops/backup_controller_dbops.class.php +++ b/backup/util/dbops/backup_controller_dbops.class.php @@ -641,4 +641,31 @@ abstract class backup_controller_dbops extends backup_dbops { } } } + + /** + * Get the progress details of a backup operation. + * Get backup records directly from database, if the backup has successfully completed + * there will be no controller object to load. + * + * @param string $backupid The backup id to query. + * @return array $progress The backup progress details. + */ + public static function get_progress($backupid) { + global $DB; + + $progress = array(); + $backuprecord = $DB->get_record( + 'backup_controllers', + array('backupid' => $backupid), + 'status, progress, operation', + MUST_EXIST); + + $status = $backuprecord->status; + $progress = $backuprecord->progress; + $operation = $backuprecord->operation; + + $progress = array('status' => $status, 'progress' => $progress, 'backupid' => $backupid, 'operation' => $operation); + + return $progress; + } } diff --git a/backup/util/helper/async_helper.class.php b/backup/util/helper/async_helper.class.php new file mode 100644 index 00000000000..3cf391fee2f --- /dev/null +++ b/backup/util/helper/async_helper.class.php @@ -0,0 +1,367 @@ +. + +/** + * Helper functions for asynchronous backups and restores. + * + * @package core + * @copyright 2019 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/user/lib.php'); + +/** + * Helper functions for asynchronous backups and restores. + * + * @package core + * @copyright 2019 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class async_helper { + + /** + * @var string $type The type of async operation. + */ + protected $type = 'backup'; + + /** + * @var string $backupid The id of the backup or restore. + */ + protected $backupid; + + /** + * @var object $user The user who created the backup record. + */ + protected $user; + + /** + * @var object $backuprec The backup controller record from the database. + */ + protected $backuprec; + + /** + * Class constructor. + * + * @param string $type The type of async operation. + * @param string $id The id of the backup or restore. + */ + public function __construct($type, $id) { + $this->type = $type; + $this->backupid = $id; + $this->backuprec = $this->get_backup_record($id); + $this->user = $this->get_user(); + } + + /** + * Given a backup id return a the record from the database. + * We use this method rather than 'load_controller' as the controller may + * not exist if this backup/restore has completed. + * + * @param int $id The backup id to get. + * @return object $backuprec The backup controller record. + */ + private function get_backup_record($id) { + global $DB; + + $backuprec = $DB->get_record('backup_controllers', array('backupid' => $id), '*', MUST_EXIST); + + return $backuprec; + } + + /** + * Given a user id return a user object. + * + * @return object $user The limited user record. + */ + private function get_user() { + $userid = $this->backuprec->userid; + $user = core_user::get_user($userid, '*', MUST_EXIST); + + return $user; + } + + /** + * Callback for preg_replace_callback. + * Replaces message placeholders with real values. + * + * @param array $matches The match array from from preg_replace_callback. + * @return string $match The replaced string. + */ + private function lookup_message_variables($matches) { + $options = array( + 'operation' => $this->type, + 'backupid' => $this->backupid, + 'user_username' => $this->user->username, + 'user_email' => $this->user->email, + 'user_firstname' => $this->user->firstname, + 'user_lastname' => $this->user->lastname, + 'link' => $this->get_resource_link(), + ); + + $match = $options[$matches[1]] ?? $matches[1]; + + return $match; + } + + /** + * Get the link to the resource that is being backuped or restored. + * + * @return moodle_url $url The link to the resource. + */ + private function get_resource_link() { + // Get activity context only for backups. + if ($this->backuprec->type == 'activity' && $this->type == 'backup') { + $context = context_module::instance($this->backuprec->itemid); + } else { // Course or Section which have the same context getter. + $context = context_course::instance($this->backuprec->itemid); + } + + // Generate link based on operation type. + if ($this->type == 'backup') { + // For backups simply generate link to restore file area UI. + $url = new moodle_url('/backup/restorefile.php', array('contextid' => $context->id)); + } else { + // For restore generate link to the item itself. + $url = $context->get_url(); + } + + return $url; + } + + /** + * Sends a confirmation message for an aynchronous process. + * + * @return int $messageid The id of the sent message. + */ + public function send_message() { + global $USER; + + $subjectraw = get_config('backup', 'backup_async_message_subject'); + $subjecttext = preg_replace_callback( + '/\{([-_A-Za-z0-9]+)\}/u', + array('async_helper', 'lookup_message_variables'), + $subjectraw); + + $messageraw = get_config('backup', 'backup_async_message'); + $messagehtml = preg_replace_callback( + '/\{([-_A-Za-z0-9]+)\}/u', + array('async_helper', 'lookup_message_variables'), + $messageraw); + $messagetext = html_to_text($messagehtml); + + $message = new \core\message\message(); + $message->component = 'moodle'; + $message->name = 'asyncbackupnotification'; + $message->userfrom = $USER; + $message->userto = $this->user; + $message->subject = $subjecttext; + $message->fullmessage = $messagetext; + $message->fullmessageformat = FORMAT_HTML; + $message->fullmessagehtml = $messagehtml; + $message->smallmessage = ''; + $message->notification = '1'; + + $messageid = message_send($message); + + return $messageid; + } + + /** + * Check if asynchronous backup and restore mode is + * enabled at system level. + * + * @return bool $async True if async mode enabled false otherwise. + */ + static public function is_async_enabled() { + global $CFG; + + $async = false; + if (!empty($CFG->enableasyncbackup)) { + $async = true; + } + + return $async; + } + + /** + * Check if there is a pending async operation for given details. + * + * @param int $id The item id to check in the backup record. + * @param string $type The type of operation: course, activity or section. + * @param string $operation Operation backup or restore. + * @return boolean $asyncpedning Is there a pending async operation. + */ + public static function is_async_pending($id, $type, $operation) { + global $DB, $USER; + $asyncpending = false; + + // Only check for pending async operations if async mode is enabled. + if (self::is_async_enabled()) { + $select = 'userid = ? AND itemid = ? AND type = ? AND operation = ? AND execution = ? AND status < ? AND status > ?'; + $params = array( + $USER->id, + $id, + $type, + $operation, + backup::EXECUTION_DELAYED, + backup::STATUS_FINISHED_ERR, + backup::STATUS_NEED_PRECHECK + ); + $asyncpending = $DB->record_exists_select('backup_controllers', $select, $params); + } + return $asyncpending; + } + + /** + * Get the size, url and restore url for a backup file. + * + * @param string $filename The name of the file to get info for. + * @param string $filearea The file area for the file. + * @param int $contextid The context ID of the file. + * @return array $results The result array containing the size, url and restore url of the file. + */ + public static function get_backup_file_info($filename, $filearea, $contextid) { + $fs = get_file_storage(); + $file = $fs->get_file($contextid, 'backup', $filearea, 0, '/', $filename); + $filesize = display_size ($file->get_filesize()); + $fileurl = moodle_url::make_pluginfile_url( + $file->get_contextid(), + $file->get_component(), + $file->get_filearea(), + null, + $file->get_filepath(), + $file->get_filename(), + true + ); + + $params = array(); + $params['action'] = 'choosebackupfile'; + $params['filename'] = $file->get_filename(); + $params['filepath'] = $file->get_filepath(); + $params['component'] = $file->get_component(); + $params['filearea'] = $file->get_filearea(); + $params['filecontextid'] = $file->get_contextid(); + $params['contextid'] = $contextid; + $params['itemid'] = $file->get_itemid(); + $restoreurl = new moodle_url('/backup/restorefile.php', $params); + $filesize = display_size ($file->get_filesize()); + + $results = array( + 'filesize' => $filesize, + 'fileurl' => $fileurl->out(false), + 'restoreurl' => $restoreurl->out(false)); + + return $results; + } + + /** + * Get the url of a restored backup item based on the backup ID. + * + * @param string $backupid The backup ID to get the restore location url. + * @return array $urlarray The restored item URL as an array. + */ + public static function get_restore_url($backupid) { + global $DB; + + $backupitemid = $DB->get_field('backup_controllers', 'itemid', array('backupid' => $backupid), MUST_EXIST); + $newcontext = context_course::instance($backupitemid); + + $restoreurl = $newcontext->get_url()->out(); + $urlarray = array('restoreurl' => $restoreurl); + + return $urlarray; + } + + /** + * Get markup for in progress async backups, + * to use in backup table UI. + * + * @param \core_backup_renderer $renderer The backup renderer object. + * @param integer $instanceid The context id to get backup data for. + * @return array $tabledata the rows of table data. + */ + public static function get_async_backups($renderer, $instanceid) { + global $DB; + + $tabledata = array(); + + // Get relevant backup ids based on context instance id. + $select = 'itemid = ? AND execution = ? AND status < ? AND status > ?'; + $params = array($instanceid, backup::EXECUTION_DELAYED, backup::STATUS_FINISHED_ERR, backup::STATUS_NEED_PRECHECK); + $backups = $DB->get_records_select('backup_controllers', $select, $params, 'timecreated DESC', 'id, backupid, timecreated'); + + foreach ($backups as $backup) { + $bc = \backup_controller::load_controller($backup->backupid); // Get the backup controller. + $filename = $bc->get_plan()->get_setting('filename')->get_value(); + $timecreated = $backup->timecreated; + $status = $renderer->get_status_display($bc->get_status(), $bc->get_backupid()); + + $tablerow = array($filename, userdate($timecreated), '-', '-', '-', $status); + $tabledata[] = $tablerow; + } + + return $tabledata; + } + + /** + * Get the course name of the resource being restored. + * + * @param \context $context The Moodle context for the restores. + * @return string $coursename The full name of the course. + */ + public static function get_restore_name(\context $context) { + global $DB; + $instanceid = $context->instanceid; + + if ($context->contextlevel == CONTEXT_MODULE) { + // For modules get the course name and module name. + $cm = get_coursemodule_from_id('', $context->instanceid, 0, false, MUST_EXIST); + $coursename = $DB->get_field('course', 'fullname', array('id' => $cm->course)); + $itemname = $coursename . ' - ' . $cm->name; + } else { + $itemname = $DB->get_field('course', 'fullname', array('id' => $context->instanceid)); + + } + + return $itemname; + } + + /** + * Get all the current in progress async restores for a user. + * + * @param int $userid Moodle user id. + * @return array $restores List of current restores in progress. + */ + public static function get_async_restores($userid) { + global $DB; + + $select = 'userid = ? AND execution = ? AND status < ? AND status > ? AND operation = ?'; + $params = array($userid, backup::EXECUTION_DELAYED, backup::STATUS_FINISHED_ERR, backup::STATUS_NEED_PRECHECK, 'restore'); + $restores = $DB->get_records_select( + 'backup_controllers', + $select, + $params, + 'timecreated DESC', + 'id, backupid, status, itemid, timecreated'); + + return $restores; + } + +} + diff --git a/backup/util/helper/tests/async_helper_test.php b/backup/util/helper/tests/async_helper_test.php new file mode 100644 index 00000000000..70b50a2078a --- /dev/null +++ b/backup/util/helper/tests/async_helper_test.php @@ -0,0 +1,148 @@ +. + +/** + * Asyncronhous helper tests. + * + * @package core_backup + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); +require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php'); + +/** + * Asyncronhous helper tests. + * + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_backup_async_helper_testcase extends \core_privacy\tests\provider_testcase { + + /** + * Tests sending message for asynchronous backup. + */ + public function test_send_message() { + global $DB, $USER; + $this->preventResetByRollback(); + $this->resetAfterTest(true); + $this->setAdminUser(); + + set_config('backup_async_message_users', '1', 'backup'); + set_config('backup_async_message_subject', 'Moodle {operation} completed sucessfully', 'backup'); + set_config('backup_async_message', + 'Dear {user_firstname} {user_lastname},
Your {operation} (ID: {backupid}) has completed successfully!', + 'backup'); + set_config('allowedemaildomains', 'example.com'); + + $generator = $this->getDataGenerator(); + $course = $generator->create_course(); // Create a course with some availability data set. + $user2 = $generator->create_user(array('firstname' => 'test', 'lastname' => 'human', 'maildisplay' => 1)); + $generator->enrol_user($user2->id, $course->id, 'editingteacher'); + + $DB->set_field_select('message_processors', 'enabled', 0, "name <> 'email'"); + set_user_preference('message_provider_moodle_asyncbackupnotification', 'email', $user2); + + // Make the backup controller for an async backup. + $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE, + backup::INTERACTIVE_YES, backup::MODE_ASYNC, $user2->id); + $bc->finish_ui(); + $backupid = $bc->get_backupid(); + $bc->destroy(); + + $sink = $this->redirectEmails(); + + // Send message. + $asynchelper = new async_helper('backup', $backupid); + $messageid = $asynchelper->send_message(); + + $emails = $sink->get_messages(); + $this->assertCount(1, $emails); + $email = reset($emails); + + $this->assertSame($USER->email, $email->from); + $this->assertSame($user2->email, $email->to); + $this->assertSame('Moodle backup completed sucessfully', $email->subject); + $this->assertNotEmpty($email->header); + $this->assertNotEmpty($email->body); + $this->assertRegExp("/$backupid/", $email->body); + $this->assertThat($email->body, $this->logicalNot($this->stringContains('{'))); + $this->assertGreaterThan(0, $messageid); + $sink->clear(); + } + + /** + * Tests getting the asynchronous backup table items. + */ + public function test_get_async_backups() { + global $DB, $CFG, $USER, $PAGE; + + $this->resetAfterTest(true); + $this->setAdminUser(); + $CFG->enableavailability = true; + $CFG->enablecompletion = true; + + // Create a course with some availability data set. + $generator = $this->getDataGenerator(); + $course = $generator->create_course( + array('format' => 'topics', 'numsections' => 3, + 'enablecompletion' => COMPLETION_ENABLED), + array('createsections' => true)); + $forum = $generator->create_module('forum', array( + 'course' => $course->id)); + $forum2 = $generator->create_module('forum', array( + 'course' => $course->id, 'completion' => COMPLETION_TRACKING_MANUAL)); + + // We need a grade, easiest is to add an assignment. + $assignrow = $generator->create_module('assign', array( + 'course' => $course->id)); + $assign = new assign(context_module::instance($assignrow->cmid), false, false); + $item = $assign->get_grade_item(); + + // Make a test grouping as well. + $grouping = $generator->create_grouping(array('courseid' => $course->id, + 'name' => 'Grouping!')); + + $availability = '{"op":"|","show":false,"c":[' . + '{"type":"completion","cm":' . $forum2->cmid .',"e":1},' . + '{"type":"grade","id":' . $item->id . ',"min":4,"max":94},' . + '{"type":"grouping","id":' . $grouping->id . '}' . + ']}'; + $DB->set_field('course_modules', 'availability', $availability, array( + 'id' => $forum->cmid)); + $DB->set_field('course_sections', 'availability', $availability, array( + 'course' => $course->id, 'section' => 1)); + + // Make the backup controller for an async backup. + $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE, + backup::INTERACTIVE_YES, backup::MODE_ASYNC, $USER->id); + $bc->finish_ui(); + $bc->destroy(); + unset($bc); + + $coursecontext = context_course::instance($course->id); + $renderer = $PAGE->get_renderer('core', 'backup'); + + $result = \async_helper::get_async_backups($renderer, $coursecontext->instanceid); + + $this->assertEquals(1, count($result)); + $this->assertEquals('backup.mbz', $result[0][0]); + } +} diff --git a/backup/util/includes/backup_includes.php b/backup/util/includes/backup_includes.php index 87d237d690b..0080a0e3b19 100644 --- a/backup/util/includes/backup_includes.php +++ b/backup/util/includes/backup_includes.php @@ -54,6 +54,7 @@ require_once($CFG->dirroot . '/backup/util/structure/backup_nested_element.class require_once($CFG->dirroot . '/backup/util/structure/backup_optigroup.class.php'); require_once($CFG->dirroot . '/backup/util/structure/backup_optigroup_element.class.php'); require_once($CFG->dirroot . '/backup/util/structure/backup_structure_processor.class.php'); +require_once($CFG->dirroot . '/backup/util/helper/async_helper.class.php'); require_once($CFG->dirroot . '/backup/util/helper/backup_helper.class.php'); require_once($CFG->dirroot . '/backup/util/helper/backup_general_helper.class.php'); require_once($CFG->dirroot . '/backup/util/helper/backup_null_iterator.class.php'); diff --git a/backup/util/includes/restore_includes.php b/backup/util/includes/restore_includes.php index ec0f8eb9801..2a8e020b638 100644 --- a/backup/util/includes/restore_includes.php +++ b/backup/util/includes/restore_includes.php @@ -34,6 +34,7 @@ require_once($CFG->dirroot . '/backup/util/interfaces/executable.class.php'); require_once($CFG->dirroot . '/backup/util/interfaces/processable.class.php'); require_once($CFG->dirroot . '/backup/backup.class.php'); require_once($CFG->dirroot . '/backup/util/structure/restore_path_element.class.php'); +require_once($CFG->dirroot . '/backup/util/helper/async_helper.class.php'); require_once($CFG->dirroot . '/backup/util/helper/backup_anonymizer_helper.class.php'); require_once($CFG->dirroot . '/backup/util/helper/backup_file_manager.class.php'); require_once($CFG->dirroot . '/backup/util/helper/restore_prechecks_helper.class.php'); diff --git a/backup/util/ui/amd/build/async_backup.min.js b/backup/util/ui/amd/build/async_backup.min.js new file mode 100644 index 00000000000..41384ff5135 --- /dev/null +++ b/backup/util/ui/amd/build/async_backup.min.js @@ -0,0 +1 @@ +define(["jquery","core/ajax","core/str","core/notification","core/templates"],function(a,b,c,d,e){function f(b,c){var d=Math.round(c)+"%",e=a("#"+b+"_bar"),f=c.toFixed(2)+"%";e.attr("aria-valuenow",d),e.css("width",d),e.text(f)}function g(c){var f=a("#"+c+"_bar").parent().parent(),g=f.parent(),h=f.siblings(),i=h[1],j=a(i).text(),k=h[0],l=a(k).text();b.call([{methodname:"core_backup_get_async_backup_links_backup",args:{filename:l,contextid:n}}])[0].done(function(a){var b={filename:l,time:j,size:a.filesize,fileurl:a.fileurl,restoreurl:a.restoreurl};e.render("core/async_backup_progress_row",b).then(function(a,b){e.replaceNodeContents(g,a,b)}).fail(function(){d.exception(new Error("Failed to load table row"))})})}function h(c){var f=a("#"+c+"_bar").parent().parent(),g=f.parent(),h=f.siblings(),i=h[0],j=h[1],k=a(j).text();b.call([{methodname:"core_backup_get_async_backup_links_restore",args:{backupid:c,contextid:n}}])[0].done(function(b){var c=a(i).text(),f={resourcename:c,restoreurl:b.restoreurl,time:k};e.render("core/async_restore_progress_row",f).then(function(a,b){e.replaceNodeContents(g,a,b)}).fail(function(){d.exception(new Error("Failed to load table row"))})})}function i(e){var g,h=100*e.progress,i=a("#"+m+"_bar"),j=a("#"+m+"_status"),k=a("#"+m+"_detail"),l=a("#"+m+"_button");if(e.status==s){i.addClass("bg-success"),f(m,h);var r="async"+p+"processing";c.get_string(r,"backup").then(function(a){return j.text(a),a})["catch"](function(){d.exception(new Error("Failed to load string: backup "+r))})}else if(e.status==t){i.addClass("bg-danger"),i.removeClass("bg-success"),f(m,100);var v="async"+p+"error",w="async"+p+"errordetail";g=[{key:v,component:"backup"},{key:w,component:"backup"}],c.get_strings(g).then(function(a){return j.text(a[0]),k.text(a[1]),a})["catch"](function(){d.exception(new Error("Failed to load string"))}),a(".backup_progress").children("span").removeClass("backup_stage_current"),a(".backup_progress").children("span").last().addClass("backup_stage_current"),clearInterval(q)}else if(e.status==u){i.addClass("bg-success"),f(m,100);var x="async"+p+"complete";if(c.get_string(x,"backup").then(function(a){return j.text(a),a})["catch"](function(){d.exception(new Error("Failed to load string: backup "+x))}),"restore"==p)b.call([{methodname:"core_backup_get_async_backup_links_restore",args:{backupid:m,contextid:n}}])[0].done(function(a){var b="async"+p+"completedetail",e="async"+p+"completebutton",f=[{key:b,component:"backup",param:a.restoreurl},{key:e,component:"backup"}];c.get_strings(f).then(function(b){return k.html(b[0]),l.text(b[1]),l.attr("href",a.restoreurl),b})["catch"](function(){d.exception(new Error("Failed to load string"))})});else{var y="async"+p+"completedetail",z="async"+p+"completebutton";g=[{key:y,component:"backup",param:o},{key:z,component:"backup"}],c.get_strings(g).then(function(a){return k.html(a[0]),l.text(a[1]),l.attr("href",o),a})["catch"](function(){d.exception(new Error("Failed to load string"))})}a(".backup_progress").children("span").removeClass("backup_stage_current"),a(".backup_progress").children("span").last().addClass("backup_stage_current"),clearInterval(q)}}function j(b){b.forEach(function(b){var c=100*b.progress,d=b.backupid,e=a("#"+d+"_bar"),i=b.operation;b.status==s?(e.addClass("bg-success"),f(d,c)):b.status==t?(e.addClass("bg-danger"),e.addClass("complete"),a("#"+d+"_bar").removeClass("bg-success"),f(d,100)):b.status==u&&(e.addClass("bg-success"),e.addClass("complete"),f(d,100),"backup"==i?g(d):h(d))})}function k(){b.call([{methodname:"core_backup_get_async_backup_progress",args:{backupids:[m],contextid:n}}])[0].done(function(a){i(a[0])})}function l(){var c=[],d=a(".progress").find(".progress-bar").not(".complete");d.each(function(){c.push(this.id.substring(0,32))}),c.length>0?b.call([{methodname:"core_backup_get_async_backup_progress",args:{backupids:c,contextid:n}}])[0].done(function(a){j(a)}):clearInterval(r)}var m,n,o,p,q,r,s=800,t=900,u=1e3,v={},w=5e3;return v.asyncBackupAllStatus=function(a){n=a,r=setInterval(l,w)},v.asyncBackupStatus=function(b,c,d,e){m=b,n=c,o=d,p="backup"==e?"backup":"restore",a(".backup_progress").children("a").removeAttr("href"),q=setInterval(k,w)},v}); \ No newline at end of file diff --git a/backup/util/ui/amd/src/async_backup.js b/backup/util/ui/amd/src/async_backup.js new file mode 100644 index 00000000000..0563fa8239d --- /dev/null +++ b/backup/util/ui/amd/src/async_backup.js @@ -0,0 +1,424 @@ +// This file is part of Moodle - http://moodle.org/ +// +// Moodle is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Moodle is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Moodle. If not, see . + +/** + * This module updates the UI during an asynchronous + * backup or restore process. + * + * @module backup/util/async_backup + * @package core + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @since 3.7 + */ +define(['jquery', 'core/ajax', 'core/str', 'core/notification', 'core/templates'], + function($, ajax, Str, notification, Templates) { + + /** + * Module level constants. + * + * Using var instead of const as ES6 isn't fully supported yet. + */ + var STATUS_EXECUTING = 800; + var STATUS_FINISHED_ERR = 900; + var STATUS_FINISHED_OK = 1000; + + /** + * Module level variables. + */ + var Asyncbackup = {}; + var checkdelay = 5000; // How often we check for progress updates. + var backupid; // The backup id to get the progress for. + var contextid; // The course this backup progress is for. + var restoreurl; // The URL to view course restores. + var typeid; // The type of operation backup or restore. + var backupintervalid; // The id of the setInterval function. + var allbackupintervalid; // The id of the setInterval function. + + /** + * Helper function to update UI components. + * + * @param {string} backupid The id to match elements on. + * @param {number} percentage The completion percentage to apply. + */ + function updateElement(backupid, percentage) { + var percentagewidth = Math.round(percentage) + '%'; + var elementbar = $('#' + backupid + '_bar'); + var percentagetext = percentage.toFixed(2) + '%'; + + // Set progress bar percentage indicators + elementbar.attr('aria-valuenow', percentagewidth); + elementbar.css('width', percentagewidth); + elementbar.text(percentagetext); + } + + /** + * Update backup table row when an async backup completes. + * + * @param {string} backupid The id to match elements on. + */ + function updateBackupTableRow(backupid) { + var statuscell = $('#' + backupid + '_bar').parent().parent(); + var tablerow = statuscell.parent(); + var cellsiblings = statuscell.siblings(); + var timecell = cellsiblings[1]; + var timevalue = $(timecell).text(); + var filenamecell = cellsiblings[0]; + var filename = $(filenamecell).text(); + + ajax.call([{ + // Get the table data via webservice. + methodname: 'core_backup_get_async_backup_links_backup', + args: { + 'filename': filename, + 'contextid': contextid + }, + }])[0].done(function(response) { + // We have the data now update the UI. + var context = { + filename: filename, + time: timevalue, + size: response.filesize, + fileurl: response.fileurl, + restoreurl: response.restoreurl + }; + + Templates.render('core/async_backup_progress_row', context).then(function(html, js) { + Templates.replaceNodeContents(tablerow, html, js); + return; + }).fail(function() { + notification.exception(new Error('Failed to load table row')); + return; + }); + }); + } + + /** + * Update restore table row when an async restore completes. + * + * @param {string} backupid The id to match elements on. + */ + function updateRestoreTableRow(backupid) { + var statuscell = $('#' + backupid + '_bar').parent().parent(); + var tablerow = statuscell.parent(); + var cellsiblings = statuscell.siblings(); + var coursecell = cellsiblings[0]; + var timecell = cellsiblings[1]; + var timevalue = $(timecell).text(); + + ajax.call([{ + // Get the table data via webservice. + methodname: 'core_backup_get_async_backup_links_restore', + args: { + 'backupid': backupid, + 'contextid': contextid + }, + }])[0].done(function(response) { + // We have the data now update the UI. + var resourcename = $(coursecell).text(); + var context = { + resourcename: resourcename, + restoreurl: response.restoreurl, + time: timevalue + }; + + Templates.render('core/async_restore_progress_row', context).then(function(html, js) { + Templates.replaceNodeContents(tablerow, html, js); + return; + }).fail(function() { + notification.exception(new Error('Failed to load table row')); + return; + }); + }); + } + + /** + * Update the Moodle user interface with the progress of + * the backup process. + * + * @param {object} progress The progress and status of the process. + */ + function updateProgress(progress) { + var percentage = progress.progress * 100; + var elementbar = $('#' + backupid + '_bar'); + var elementstatus = $('#' + backupid + '_status'); + var elementdetail = $('#' + backupid + '_detail'); + var elementbutton = $('#' + backupid + '_button'); + var stringRequests; + + if (progress.status == STATUS_EXECUTING) { + // Process is in progress. + // Add in progress class color to bar + elementbar.addClass('bg-success'); + + updateElement(backupid, percentage); + + // Change heading + var strProcessing = 'async' + typeid + 'processing'; + Str.get_string(strProcessing, 'backup').then(function(title) { + elementstatus.text(title); + return title; + }).catch(function() { + notification.exception(new Error('Failed to load string: backup ' + strProcessing)); + }); + + } else if (progress.status == STATUS_FINISHED_ERR) { + // Process completed with error. + + // Add in fail class color to bar + elementbar.addClass('bg-danger'); + + // Remove in progress class color to bar + elementbar.removeClass('bg-success'); + + updateElement(backupid, 100); + + // Change heading and text + var strStatus = 'async' + typeid + 'error'; + var strStatusDetail = 'async' + typeid + 'errordetail'; + stringRequests = [ + {key: strStatus, component: 'backup'}, + {key: strStatusDetail, component: 'backup'} + ]; + Str.get_strings(stringRequests).then(function(strings) { + elementstatus.text(strings[0]); + elementdetail.text(strings[1]); + + return strings; + }) + .catch(function() { + notification.exception(new Error('Failed to load string')); + return; + }); + + $('.backup_progress').children('span').removeClass('backup_stage_current'); + $('.backup_progress').children('span').last().addClass('backup_stage_current'); + + // Stop checking when we either have an error or a completion. + clearInterval(backupintervalid); + + } else if (progress.status == STATUS_FINISHED_OK) { + // Process completed successfully. + + // Add in progress class color to bar + elementbar.addClass('bg-success'); + + updateElement(backupid, 100); + + // Change heading and text + var strComplete = 'async' + typeid + 'complete'; + Str.get_string(strComplete, 'backup').then(function(title) { + elementstatus.text(title); + return title; + }).catch(function() { + notification.exception(new Error('Failed to load string: backup ' + strComplete)); + }); + + if (typeid == 'restore') { + ajax.call([{ + // Get the table data via webservice. + methodname: 'core_backup_get_async_backup_links_restore', + args: { + 'backupid': backupid, + 'contextid': contextid + }, + }])[0].done(function(response) { + var strDetail = 'async' + typeid + 'completedetail'; + var strButton = 'async' + typeid + 'completebutton'; + var stringRequests = [ + {key: strDetail, component: 'backup', param: response.restoreurl}, + {key: strButton, component: 'backup'} + ]; + Str.get_strings(stringRequests).then(function(strings) { + elementdetail.html(strings[0]); + elementbutton.text(strings[1]); + elementbutton.attr('href', response.restoreurl); + + return strings; + }) + .catch(function() { + notification.exception(new Error('Failed to load string')); + return; + }); + + }); + } else { + var strDetail = 'async' + typeid + 'completedetail'; + var strButton = 'async' + typeid + 'completebutton'; + stringRequests = [ + {key: strDetail, component: 'backup', param: restoreurl}, + {key: strButton, component: 'backup'} + ]; + Str.get_strings(stringRequests).then(function(strings) { + elementdetail.html(strings[0]); + elementbutton.text(strings[1]); + elementbutton.attr('href', restoreurl); + + return strings; + }) + .catch(function() { + notification.exception(new Error('Failed to load string')); + return; + }); + + } + + $('.backup_progress').children('span').removeClass('backup_stage_current'); + $('.backup_progress').children('span').last().addClass('backup_stage_current'); + + // Stop checking when we either have an error or a completion. + clearInterval(backupintervalid); + } + } + + /** + * Update the Moodle user interface with the progress of + * all the pending processes. + * + * @param {object} progress The progress and status of the process. + */ + function updateProgressAll(progress) { + progress.forEach(function(element) { + var percentage = element.progress * 100; + var backupid = element.backupid; + var elementbar = $('#' + backupid + '_bar'); + var type = element.operation; + + if (element.status == STATUS_EXECUTING) { + // Process is in element. + + // Add in element class color to bar + elementbar.addClass('bg-success'); + + updateElement(backupid, percentage); + + } else if (element.status == STATUS_FINISHED_ERR) { + // Process completed with error. + + // Add in fail class color to bar + elementbar.addClass('bg-danger'); + elementbar.addClass('complete'); + + // Remove in element class color to bar + $('#' + backupid + '_bar').removeClass('bg-success'); + + updateElement(backupid, 100); + + } else if (element.status == STATUS_FINISHED_OK) { + // Process completed successfully. + + // Add in element class color to bar + elementbar.addClass('bg-success'); + elementbar.addClass('complete'); + + updateElement(backupid, 100); + + // We have a successful backup. Update the UI with download and file details. + if (type == 'backup') { + updateBackupTableRow(backupid); + } else { + updateRestoreTableRow(backupid); + } + + } + + }); + } + + /** + * Get the progress of the backup process via ajax. + */ + function getBackupProgress() { + ajax.call([{ + // Get the backup progress via webservice. + methodname: 'core_backup_get_async_backup_progress', + args: { + 'backupids': [backupid], + 'contextid': contextid + }, + }])[0].done(function(response) { + // We have the progress now update the UI. + updateProgress(response[0]); + }); + } + + /** + * Get the progress of all backup processes via ajax. + */ + function getAllBackupProgress() { + var backupids = []; + var progressbars = $('.progress').find('.progress-bar').not('.complete'); + + progressbars.each(function() { + backupids.push((this.id).substring(0, 32)); + }); + + if (backupids.length > 0) { + ajax.call([{ + // Get the backup progress via webservice. + methodname: 'core_backup_get_async_backup_progress', + args: { + 'backupids': backupids, + 'contextid': contextid + }, + }])[0].done(function(response) { + updateProgressAll(response); + }); + } else { + clearInterval(allbackupintervalid); // No more progress bars to update, stop checking. + } + } + + /** + * Get status updates for all backups. + * + * @public + * @param {number} context The context id. + */ + Asyncbackup.asyncBackupAllStatus = function(context) { + contextid = context; + allbackupintervalid = setInterval(getAllBackupProgress, checkdelay); + }; + + /** + * Get status updates for backup. + * + * @public + * @param {string} backup The backup record id. + * @param {number} context The context id. + * @param {string} restore The restore link. + * @param {string} type The operation type (backup or restore). + */ + Asyncbackup.asyncBackupStatus = function(backup, context, restore, type) { + backupid = backup; + contextid = context; + restoreurl = restore; + + if (type == 'backup') { + typeid = 'backup'; + } else { + typeid = 'restore'; + } + + // Remove the links from the progress bar, no going back now. + $('.backup_progress').children('a').removeAttr('href'); + + // Periodically check for progress updates and update the UI as required. + backupintervalid = setInterval(getBackupProgress, checkdelay); + + }; + + return Asyncbackup; +}); diff --git a/backup/util/ui/renderer.php b/backup/util/ui/renderer.php index d307e54ebb9..4ee83e966f5 100644 --- a/backup/util/ui/renderer.php +++ b/backup/util/ui/renderer.php @@ -22,6 +22,13 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +defined('MOODLE_INTERNAL') || die; + +global $CFG; +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); +require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php'); +require_once($CFG->dirroot . '/backup/moodle2/backup_plan_builder.class.php'); + /** * The primary renderer for the backup. * @@ -64,9 +71,8 @@ class core_backup_renderer extends plugin_renderer_base { * @return string HTML content that shows the log */ public function log_display($loghtml) { - global $OUTPUT; $out = html_writer::start_div('backup_log'); - $out .= $OUTPUT->heading(get_string('backuplog', 'backup')); + $out .= $this->output->heading(get_string('backuplog', 'backup')); $out .= html_writer::start_div('backup_log_contents'); $out .= $loghtml; $out .= html_writer::end_div(); @@ -533,6 +539,32 @@ class core_backup_renderer extends plugin_renderer_base { return $this->render($files); } + /** + * Generate the status indicator markup for display in the + * backup restore file area UI. + * + * @param int $statuscode The status code of the backup. + * @param string $backupid The backup record id. + * @return string|boolean $status The status indicator for the operation. + */ + public function get_status_display($statuscode, $backupid) { + if ($statuscode == backup::STATUS_AWAITING || $statuscode == backup::STATUS_EXECUTING) { // Inprogress. + $progresssetup = array( + 'backupid' => $backupid, + 'width' => '100' + ); + $status = $this->render_from_template('core/async_backup_progress', $progresssetup); + } else if ($statuscode == backup::STATUS_FINISHED_ERR) { // Error. + $icon = $this->output->render(new \pix_icon('i/delete', get_string('failed', 'backup'))); + $status = \html_writer::span($icon, 'action-icon'); + } else if ($statuscode == backup::STATUS_FINISHED_OK) { // Complete. + $icon = $this->output->render(new \pix_icon('i/checked', get_string('successful', 'backup'))); + $status = \html_writer::span($icon, 'action-icon'); + } + + return $status; + } + /** * Displays a backup files viewer * @@ -544,12 +576,35 @@ class core_backup_renderer extends plugin_renderer_base { global $CFG; $files = $viewer->files; + $async = async_helper::is_async_enabled(); + + $tablehead = array( + get_string('filename', 'backup'), + get_string('time'), + get_string('size'), + get_string('download'), + get_string('restore')); + if ($async) { + $tablehead[] = get_string('status', 'backup'); + } + $table = new html_table(); $table->attributes['class'] = 'backup-files-table generaltable'; - $table->head = array(get_string('filename', 'backup'), get_string('time'), get_string('size'), get_string('download'), get_string('restore')); + $table->head = $tablehead; $table->width = '100%'; $table->data = array(); + // First add in progress asynchronous backups. + // Only if asynchronous backups are enabled. + // Also only render async status in correct area. Courese OR activity (not both). + if ($async + && (($viewer->filearea == 'course' && $viewer->currentcontext->contextlevel == CONTEXT_COURSE) + || ($viewer->filearea == 'activity' && $viewer->currentcontext->contextlevel == CONTEXT_MODULE)) + ) { + $table->data = \async_helper::get_async_backups($this, $viewer->currentcontext->instanceid); + } + + // Add completed backups. foreach ($files as $file) { if ($file->is_directory()) { continue; @@ -585,13 +640,18 @@ class core_backup_renderer extends plugin_renderer_base { $downloadlink = ''; } } - $table->data[] = array( + $tabledata = array( $file->get_filename(), - userdate($file->get_timemodified()), - display_size($file->get_filesize()), + userdate ($file->get_timemodified()), + display_size ($file->get_filesize()), $downloadlink, - $restorelink, - ); + $restorelink + ); + if ($async) { + $tabledata[] = $this->get_status_display(backup::STATUS_FINISHED_OK, null); + } + + $table->data[] = $tabledata; } $html = html_writer::table($table); @@ -853,6 +913,42 @@ class core_backup_renderer extends plugin_renderer_base { $output .= html_writer::end_tag('div'); return $output; } + + /** + * Get markup to render table for all of a users async + * in progress restores. + * + * @param int $userid The Moodle user id. + * @param \context $context The Moodle context for these restores. + * @return string $html The table HTML. + */ + public function restore_progress_viewer ($userid, $context) { + $tablehead = array(get_string('course'), get_string('time'), get_string('status', 'backup')); + + $table = new html_table(); + $table->attributes['class'] = 'backup-files-table generaltable'; + $table->head = $tablehead; + $tabledata = array(); + + // Get all in progress async restores for this user. + $restores = \async_helper::get_async_restores($userid); + + // For each backup get, new item name, time restore created and progress. + foreach ($restores as $restore) { + + $restorename = \async_helper::get_restore_name($context); + $timecreated = $restore->timecreated; + $status = $this->get_status_display($restore->status, $restore->backupid); + + $tablerow = array($restorename, userdate($timecreated), $status); + $tabledata[] = $tablerow; + } + + $table->data = $tabledata; + $html = html_writer::table($table); + + return $html; + } } /** diff --git a/badges/cron.php b/badges/cron.php deleted file mode 100644 index fc51a8dda3d..00000000000 --- a/badges/cron.php +++ /dev/null @@ -1,158 +0,0 @@ -. - -/** - * Cron job for reviewing and aggregating badge award criteria - * - * @package core - * @subpackage badges - * @copyright 2012 onwards Totara Learning Solutions Ltd {@link http://www.totaralms.com/} - * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later - * @author Yuliya Bozhko - */ - -defined('MOODLE_INTERNAL') || die(); -require_once($CFG->libdir . '/badgeslib.php'); - -function badge_cron() { - global $CFG; - - if (!empty($CFG->enablebadges)) { - badge_review_cron(); - badge_message_cron(); - } -} - -/** - * Reviews criteria and awards badges - * - * First find all badges that can be earned, then reviews each badge. - * (Not sure how efficient this is timewise). - */ -function badge_review_cron() { - global $DB, $CFG; - $total = 0; - - $courseparams = array(); - if (empty($CFG->badges_allowcoursebadges)) { - $coursesql = ''; - } else { - $coursesql = ' OR EXISTS (SELECT id FROM {course} WHERE visible = :visible AND startdate < :current) '; - $courseparams = array('visible' => true, 'current' => time()); - } - - $sql = 'SELECT id - FROM {badge} - WHERE (status = :active OR status = :activelocked) - AND (type = :site ' . $coursesql . ')'; - $badgeparams = array( - 'active' => BADGE_STATUS_ACTIVE, - 'activelocked' => BADGE_STATUS_ACTIVE_LOCKED, - 'site' => BADGE_TYPE_SITE - ); - $params = array_merge($badgeparams, $courseparams); - $badges = $DB->get_fieldset_sql($sql, $params); - - mtrace('Started reviewing available badges.'); - foreach ($badges as $bid) { - $badge = new badge($bid); - - if ($badge->has_criteria()) { - if (debugging()) { - mtrace('Processing badge "' . $badge->name . '"...'); - } - - $issued = $badge->review_all_criteria(); - - if (debugging()) { - mtrace('...badge was issued to ' . $issued . ' users.'); - } - $total += $issued; - } - } - - mtrace('Badges were issued ' . $total . ' time(s).'); -} - -/** - * Sends out scheduled messages to badge creators - * - */ -function badge_message_cron() { - global $DB; - - mtrace('Sending scheduled badge notifications.'); - - $scheduled = $DB->get_records_select('badge', 'notification > ? AND (status != ?) AND nextcron < ?', - array(BADGE_MESSAGE_ALWAYS, BADGE_STATUS_ARCHIVED, time()), - 'notification ASC', 'id, name, notification, usercreated as creator, timecreated'); - - foreach ($scheduled as $sch) { - // Send messages. - badge_assemble_notification($sch); - - // Update next cron value. - $nextcron = badges_calculate_message_schedule($sch->notification); - $DB->set_field('badge', 'nextcron', $nextcron, array('id' => $sch->id)); - } -} - -/** - * Creates single message for all notification and sends it out - * - * @param object $badge A badge which is notified about. - */ -function badge_assemble_notification(stdClass $badge) { - global $DB; - - $userfrom = core_user::get_noreply_user(); - $userfrom->maildisplay = true; - - if ($msgs = $DB->get_records_select('badge_issued', 'issuernotified IS NULL AND badgeid = ?', array($badge->id))) { - // Get badge creator. - $creator = $DB->get_record('user', array('id' => $badge->creator), '*', MUST_EXIST); - $creatorsubject = get_string('creatorsubject', 'badges', $badge->name); - $creatormessage = ''; - - // Put all messages in one digest. - foreach ($msgs as $msg) { - $issuedlink = html_writer::link(new moodle_url('/badges/badge.php', array('hash' => $msg->uniquehash)), $badge->name); - $recipient = $DB->get_record('user', array('id' => $msg->userid), '*', MUST_EXIST); - - $a = new stdClass(); - $a->user = fullname($recipient); - $a->link = $issuedlink; - $creatormessage .= get_string('creatorbody', 'badges', $a); - $DB->set_field('badge_issued', 'issuernotified', time(), array('badgeid' => $msg->badgeid, 'userid' => $msg->userid)); - } - - // Create a message object. - $eventdata = new \core\message\message(); - $eventdata->courseid = SITEID; - $eventdata->component = 'moodle'; - $eventdata->name = 'badgecreatornotice'; - $eventdata->userfrom = $userfrom; - $eventdata->userto = $creator; - $eventdata->notification = 1; - $eventdata->subject = $creatorsubject; - $eventdata->fullmessage = format_text_email($creatormessage, FORMAT_HTML); - $eventdata->fullmessageformat = FORMAT_PLAIN; - $eventdata->fullmessagehtml = $creatormessage; - $eventdata->smallmessage = $creatorsubject; - - message_send($eventdata); - } -} diff --git a/calendar/lib.php b/calendar/lib.php index fd6078a38be..84b5ab3bfee 100644 --- a/calendar/lib.php +++ b/calendar/lib.php @@ -3302,9 +3302,11 @@ function calendar_get_legacy_events($tstart, $tend, $users, $groups, $courses, * @param string $view The type of calendar to have displayed * @param bool $includenavigation Whether to include navigation * @param bool $skipevents Whether to load the events or not + * @param int $lookahead Overwrites site and users's lookahead setting. * @return array[array, string] */ -function calendar_get_view(\calendar_information $calendar, $view, $includenavigation = true, bool $skipevents = false) { +function calendar_get_view(\calendar_information $calendar, $view, $includenavigation = true, bool $skipevents = false, + ?int $lookahead = null) { global $PAGE, $CFG; $renderer = $PAGE->get_renderer('core_calendar'); @@ -3322,12 +3324,14 @@ function calendar_get_view(\calendar_information $calendar, $view, $includenavig $date->modify('+1 day'); } else if ($view === 'upcoming' || $view === 'upcoming_mini') { // Number of days in the future that will be used to fetch events. - if (isset($CFG->calendar_lookahead)) { - $defaultlookahead = intval($CFG->calendar_lookahead); - } else { - $defaultlookahead = CALENDAR_DEFAULT_UPCOMING_LOOKAHEAD; + if (!$lookahead) { + if (isset($CFG->calendar_lookahead)) { + $defaultlookahead = intval($CFG->calendar_lookahead); + } else { + $defaultlookahead = CALENDAR_DEFAULT_UPCOMING_LOOKAHEAD; + } + $lookahead = get_user_preferences('calendar_lookahead', $defaultlookahead); } - $lookahead = get_user_preferences('calendar_lookahead', $defaultlookahead); // Maximum number of events to be displayed on upcoming view. $defaultmaxevents = CALENDAR_DEFAULT_UPCOMING_MAXEVENTS; diff --git a/calendar/view.php b/calendar/view.php index df616e1f2b8..3fb15094218 100644 --- a/calendar/view.php +++ b/calendar/view.php @@ -53,6 +53,7 @@ $categoryid = optional_param('category', null, PARAM_INT); $courseid = optional_param('course', SITEID, PARAM_INT); $view = optional_param('view', 'upcoming', PARAM_ALPHA); $time = optional_param('time', 0, PARAM_INT); +$lookahead = optional_param('lookahead', null, PARAM_INT); $url = new moodle_url('/calendar/view.php'); @@ -124,7 +125,7 @@ echo html_writer::start_tag('div', array('class'=>'heightcontainer')); echo $OUTPUT->heading(get_string('calendar', 'calendar')); -list($data, $template) = calendar_get_view($calendar, $view); +list($data, $template) = calendar_get_view($calendar, $view, true, false, $lookahead); echo $renderer->render_from_template($template, $data); echo html_writer::end_tag('div'); diff --git a/course/amd/build/actions.min.js b/course/amd/build/actions.min.js index 067c0facace..aed4c3a9d2c 100644 --- a/course/amd/build/actions.min.js +++ b/course/amd/build/actions.min.js @@ -1 +1 @@ -define(["jquery","core/ajax","core/templates","core/notification","core/str","core/url","core/yui","core/modal_factory","core/modal_events","core/key_codes"],function(a,b,c,d,e,f,g,h,i,j){var k={EDITINPROGRESS:"editinprogress",SECTIONDRAGGABLE:"sectiondraggable",EDITINGMOVE:"editing_move"},l={ACTIVITYLI:"li.activity",ACTIONAREA:".actions",ACTIVITYACTION:"a.cm-edit-action",MENU:".moodle-actionmenu[data-enhance=moodle-core-actionmenu]",TOGGLE:".toggle-display,.dropdown-toggle",SECTIONLI:"li.section",SECTIONACTIONMENU:".section_action_menu",ADDSECTIONS:"#changenumsections [data-add-sections]"};g.use("moodle-course-coursebase",function(){var a=M.course.format.get_section_selector();a&&(l.SECTIONLI=a)});var m=function(a){var b;return g.use("moodle-course-util",function(c){b=c.Moodle.core_course.util.cm.getId(c.Node(a.get(0)))}),b},n=function(a){var b;return g.use("moodle-course-util",function(c){b=c.Moodle.core_course.util.cm.getName(c.Node(a.get(0)))}),b},o=function(a){a.addClass(k.EDITINPROGRESS);var b=a.find(l.ACTIONAREA).get(0);if(b){var c=M.util.add_spinner(g,g.Node(b));return c.show(),c}return null},p=function(a){a.addClass(k.EDITINPROGRESS);var b=a.find(l.SECTIONACTIONMENU).get(0);if(b){var c=M.util.add_spinner(g,g.Node(b));return c.show(),c}return null},q=function(a){var b=M.util.add_lightbox(g,g.Node(a.get(0)));return b.show(),b},r=function(a,b,c){window.setTimeout(function(){a.removeClass(k.EDITINPROGRESS),b&&b.hide()},c)},s=function(a,b){a&&window.setTimeout(function(){a.hide()},b)},t=function(a){g.use("moodle-course-coursebase",function(){M.course.coursebase.invoke_function("setup_for_resource","#"+a)}),M.core.actionmenu&&M.core.actionmenu.newDOMNode&&M.core.actionmenu.newDOMNode(g.one("#"+a))},u=function(b,c){var d=a("#"+b),e="[data-action="+c+"]";"groupsseparate"!==c&&"groupsvisible"!==c&&"groupsnone"!==c||(e="[data-action=groupsseparate],[data-action=groupsvisible],[data-action=groupsnone]"),d.find(e).is(":visible")?d.find(e).focus():d.find(l.MENU).find(l.TOGGLE).focus()},v=function(b){var c=a("a:visible"),d=!1,e=null;return c.each(function(){if(a.contains(b[0],this))d=!0;else if(d)return e=this,!1}),e},w=function(c,e,f){var g,h=f.attr("data-action"),i=o(c),j=b.call([{methodname:"core_course_edit_module",args:{id:e,action:h,sectionreturn:f.attr("data-sectionreturn")?f.attr("data-sectionreturn"):0}}],!0);"duplicate"===h&&(g=q(f.closest(l.SECTIONLI))),a.when.apply(a,j).done(function(b){var d=v(c);c.replaceWith(b),a("
"+b+"
").find(l.ACTIVITYLI).each(function(b){t(a(this).attr("id")),0===b&&(u(a(this).attr("id"),h),d=null)}),d&&d.focus(),r(c,i,400),s(g,400),c.trigger(a.Event("coursemoduleedited",{ajaxreturn:b,action:h}))}).fail(function(b){r(c,i),s(g);var e=a.Event("coursemoduleeditfailed",{exception:b,action:h});c.trigger(e),e.isDefaultPrevented()||d.exception(b)})},x=function(c,d,e){var f=o(c),g=b.call([{methodname:"core_course_get_module",args:{id:d,sectionreturn:e}}],!0);a.when.apply(a,g).done(function(a){r(c,f,400),C(a)}).fail(function(){r(c,f)})},y=function(a,b){var c=a.attr("class").match(/modtype_([^\s]*)/)[1],f=n(a);e.get_string("pluginname",c).done(function(a){var c={type:a,name:f};e.get_strings([{key:"confirm"},{key:null===f?"deletechecktype":"deletechecktypename",param:c},{key:"yes"},{key:"no"}]).done(function(a){d.confirm(a[0],a[1],a[2],a[3],b)})})},z=function(a,b){e.get_strings([{key:"confirm"},{key:"yes"},{key:"no"}]).done(function(c){d.confirm(c[0],a,c[1],c[2],b)})},A=function(a,b,f,g,h,i,j){var k=[{key:f,component:g}];return h&&k.push({key:h,component:i}),e.get_strings(k).then(function(d){a.find("span.menu-action-text").html(d[0]),a.attr("title",d[0]);var e="";return h&&(e=d[1],a.attr("title",e)),c.renderPix(b,"core",e)}).then(function(b){a.find(".icon").replaceWith(b),a.attr("data-action",j)})["catch"](d.exception)},B=function(b,c,d,e){var f=c.attr("data-action");if("hide"===f||"show"===f){if("hide"===f?(b.addClass("hidden"),A(c,"i/show","showfromothers","format_"+e,null,null,"show")):(b.removeClass("hidden"),A(c,"i/hide","hidefromothers","format_"+e,null,null,"hide")),void 0!==d.modules)for(var g in d.modules)C(d.modules[g]);void 0!==d.section_availability&&b.find(".section_availability").first().replaceWith(d.section_availability)}else if("setmarker"===f){var h=a(l.SECTIONLI+".current"),i=h.find(l.SECTIONACTIONMENU+" a[data-action=removemarker]");h.removeClass("current"),A(i,"i/marker","highlight","core","markthistopic","core","setmarker"),b.addClass("current"),A(c,"i/marked","highlightoff","core","markedthistopic","core","removemarker")}else"removemarker"===f&&(b.removeClass("current"),A(c,"i/marker","highlight","core","markthistopic","core","setmarker"))},C=function(b){a("
"+b+"
").find(l.ACTIVITYLI).each(function(){var c=a(this).attr("id");a(l.ACTIVITYLI+"#"+c).replaceWith(b),t(c)})},D=function(c,e,f,g){var h=f.attr("data-action"),i=f.attr("data-sectionreturn")?f.attr("data-sectionreturn"):0,j=p(c),k=b.call([{methodname:"core_course_edit_section",args:{id:e,action:h,sectionreturn:i}}],!0),m=q(c);a.when.apply(a,k).done(function(b){var d=a.parseJSON(b);r(c,j),s(m),c.find(l.SECTIONACTIONMENU).find(l.TOGGLE).focus();var e=a.Event("coursesectionedited",{ajaxreturn:d,action:h});c.trigger(e),e.isDefaultPrevented()||B(c,f,d,g)}).fail(function(b){r(c,j),s(m);var e=a.Event("coursesectioneditfailed",{exception:b,action:h});c.trigger(e),e.isDefaultPrevented()||d.exception(b)})};return g.use("moodle-course-coursebase",function(){M.course.coursebase.register_module({set_visibility_resource_ui:function(b){var c=a(b.element.getDOMNode()),d=m(c);if(d){var e=c.find("."+k.EDITINGMOVE).attr("data-sectionreturn");x(c,d,e)}}})}),{initCoursePage:function(b){a("body").on("click keypress",l.ACTIVITYLI+" "+l.ACTIVITYACTION+"[data-action]",function(b){if("keypress"!==b.type||13===b.keyCode){var c=a(this),d=c.closest(l.ACTIVITYLI),e=c.attr("data-action"),f=m(d);switch(e){case"moveleft":case"moveright":case"delete":case"duplicate":case"hide":case"stealth":case"show":case"groupsseparate":case"groupsvisible":case"groupsnone":break;default:return}f&&(b.preventDefault(),"delete"===e?y(d,function(){w(d,f,c)}):w(d,f,c))}}),a("body").on("click keypress",l.SECTIONLI+" "+l.SECTIONACTIONMENU+"[data-sectionid] a[data-action]",function(c){if("keypress"!==c.type||13===c.keyCode){var d=a(this),e=d.closest(l.SECTIONLI),f=d.closest(l.SECTIONACTIONMENU).attr("data-sectionid");c.preventDefault(),d.attr("data-confirm")?z(d.attr("data-confirm"),function(){D(e,f,d,b)}):D(e,f,d,b)}}),e.get_string("numberweeks").done(function(b){var c=a(l.ADDSECTIONS),d=c.attr("data-add-sections"),e=c.attr("new-sections"),f=a('
');f.find("label").html(b),h.create({title:d,type:h.types.SAVE_CANCEL,body:f.html()},c).done(function(b){var e=a(b.getBody()).find("#add_section_numsections"),f=function(){""+parseInt(e.val())===e.val()&&parseInt(e.val())>=1&&(document.location=c.attr("href")+"&numsections="+parseInt(e.val()))};b.setSaveButtonText(d),b.getRoot().on(i.shown,function(){e.focus().select().on("keydown",function(a){a.keyCode===j.enter&&f()})}),b.getRoot().on(i.save,function(a){a.preventDefault(),f()})})})},replaceSectionActionItem:function(a,b,c,d,e,f,g,h){var i=a.find(l.SECTIONACTIONMENU+" "+b);A(i,c,d,e,f,g,h)}}}); \ No newline at end of file +define(["jquery","core/ajax","core/templates","core/notification","core/str","core/url","core/yui","core/modal_factory","core/modal_events","core/key_codes"],function(a,b,c,d,e,f,g,h,i,j){var k={EDITINPROGRESS:"editinprogress",SECTIONDRAGGABLE:"sectiondraggable",EDITINGMOVE:"editing_move"},l={ACTIVITYLI:"li.activity",ACTIONAREA:".actions",ACTIVITYACTION:"a.cm-edit-action",MENU:".moodle-actionmenu[data-enhance=moodle-core-actionmenu]",TOGGLE:".toggle-display,.dropdown-toggle",SECTIONLI:"li.section",SECTIONACTIONMENU:".section_action_menu",ADDSECTIONS:"#changenumsections [data-add-sections]"};g.use("moodle-course-coursebase",function(){var a=M.course.format.get_section_selector();a&&(l.SECTIONLI=a)});var m=function(a){var b;return g.use("moodle-course-util",function(c){b=c.Moodle.core_course.util.cm.getId(c.Node(a.get(0)))}),b},n=function(a){var b;return g.use("moodle-course-util",function(c){b=c.Moodle.core_course.util.cm.getName(c.Node(a.get(0)))}),b},o=function(a){a.addClass(k.EDITINPROGRESS);var b=a.find(l.ACTIONAREA).get(0);if(b){var c=M.util.add_spinner(g,g.Node(b));return c.show(),c}return null},p=function(a){a.addClass(k.EDITINPROGRESS);var b=a.find(l.SECTIONACTIONMENU).get(0);if(b){var c=M.util.add_spinner(g,g.Node(b));return c.show(),c}return null},q=function(a){var b=M.util.add_lightbox(g,g.Node(a.get(0)));return b.show(),b},r=function(a,b,c){window.setTimeout(function(){a.removeClass(k.EDITINPROGRESS),b&&b.hide()},c)},s=function(a,b){a&&window.setTimeout(function(){a.hide()},b)},t=function(a){g.use("moodle-course-coursebase",function(){M.course.coursebase.invoke_function("setup_for_resource","#"+a)}),M.core.actionmenu&&M.core.actionmenu.newDOMNode&&M.core.actionmenu.newDOMNode(g.one("#"+a))},u=function(b,c){var d=a("#"+b),e="[data-action="+c+"]";"groupsseparate"!==c&&"groupsvisible"!==c&&"groupsnone"!==c||(e="[data-action=groupsseparate],[data-action=groupsvisible],[data-action=groupsnone]"),d.find(e).is(":visible")?d.find(e).focus():d.find(l.MENU).find(l.TOGGLE).focus()},v=function(b){var c=a("a:visible"),d=!1,e=null;return c.each(function(){if(a.contains(b[0],this))d=!0;else if(d)return e=this,!1}),e},w=function(c,e,f){var g,h=f.attr("data-action"),i=o(c),j=b.call([{methodname:"core_course_edit_module",args:{id:e,action:h,sectionreturn:f.attr("data-sectionreturn")?f.attr("data-sectionreturn"):0}}],!0);"duplicate"===h&&(g=q(f.closest(l.SECTIONLI))),a.when.apply(a,j).done(function(b){var d=v(c);c.replaceWith(b),a("
"+b+"
").find(l.ACTIVITYLI).each(function(b){t(a(this).attr("id")),0===b&&(u(a(this).attr("id"),h),d=null)}),d&&d.focus(),r(c,i,400),s(g,400),c.trigger(a.Event("coursemoduleedited",{ajaxreturn:b,action:h}))}).fail(function(b){r(c,i),s(g);var e=a.Event("coursemoduleeditfailed",{exception:b,action:h});c.trigger(e),e.isDefaultPrevented()||d.exception(b)})},x=function(c,d,e){var f=o(c),g=b.call([{methodname:"core_course_get_module",args:{id:d,sectionreturn:e}}],!0);a.when.apply(a,g).done(function(a){r(c,f,400),C(a)}).fail(function(){r(c,f)})},y=function(a,b){var c=a.attr("class").match(/modtype_([^\s]*)/)[1],f=n(a);e.get_string("pluginname",c).done(function(a){var c={type:a,name:f};e.get_strings([{key:"confirm"},{key:null===f?"deletechecktype":"deletechecktypename",param:c},{key:"yes"},{key:"no"}]).done(function(a){d.confirm(a[0],a[1],a[2],a[3],b)})})},z=function(a,b){e.get_strings([{key:"confirm"},{key:"yes"},{key:"no"}]).done(function(c){d.confirm(c[0],a,c[1],c[2],b)})},A=function(a,b,f,g,h,i,j){var k=[{key:f,component:g}];return e.get_strings(k).then(function(d){return a.find("span.menu-action-text").html(d[0]),c.renderPix(b,"core")}).then(function(b){a.find(".icon").replaceWith(b),a.attr("data-action",j)})["catch"](d.exception)},B=function(b,c,d,e){var f=c.attr("data-action");if("hide"===f||"show"===f){if("hide"===f?(b.addClass("hidden"),A(c,"i/show","showfromothers","format_"+e,null,null,"show")):(b.removeClass("hidden"),A(c,"i/hide","hidefromothers","format_"+e,null,null,"hide")),void 0!==d.modules)for(var g in d.modules)C(d.modules[g]);void 0!==d.section_availability&&b.find(".section_availability").first().replaceWith(d.section_availability)}else if("setmarker"===f){var h=a(l.SECTIONLI+".current"),i=h.find(l.SECTIONACTIONMENU+" a[data-action=removemarker]");h.removeClass("current"),A(i,"i/marker","highlight","core","markthistopic","core","setmarker"),b.addClass("current"),A(c,"i/marked","highlightoff","core","markedthistopic","core","removemarker")}else"removemarker"===f&&(b.removeClass("current"),A(c,"i/marker","highlight","core","markthistopic","core","setmarker"))},C=function(b){a("
"+b+"
").find(l.ACTIVITYLI).each(function(){var c=a(this).attr("id");a(l.ACTIVITYLI+"#"+c).replaceWith(b),t(c)})},D=function(c,e,f,g){var h=f.attr("data-action"),i=f.attr("data-sectionreturn")?f.attr("data-sectionreturn"):0,j=p(c),k=b.call([{methodname:"core_course_edit_section",args:{id:e,action:h,sectionreturn:i}}],!0),m=q(c);a.when.apply(a,k).done(function(b){var d=a.parseJSON(b);r(c,j),s(m),c.find(l.SECTIONACTIONMENU).find(l.TOGGLE).focus();var e=a.Event("coursesectionedited",{ajaxreturn:d,action:h});c.trigger(e),e.isDefaultPrevented()||B(c,f,d,g)}).fail(function(b){r(c,j),s(m);var e=a.Event("coursesectioneditfailed",{exception:b,action:h});c.trigger(e),e.isDefaultPrevented()||d.exception(b)})};return g.use("moodle-course-coursebase",function(){M.course.coursebase.register_module({set_visibility_resource_ui:function(b){var c=a(b.element.getDOMNode()),d=m(c);if(d){var e=c.find("."+k.EDITINGMOVE).attr("data-sectionreturn");x(c,d,e)}}})}),{initCoursePage:function(b){a("body").on("click keypress",l.ACTIVITYLI+" "+l.ACTIVITYACTION+"[data-action]",function(b){if("keypress"!==b.type||13===b.keyCode){var c=a(this),d=c.closest(l.ACTIVITYLI),e=c.attr("data-action"),f=m(d);switch(e){case"moveleft":case"moveright":case"delete":case"duplicate":case"hide":case"stealth":case"show":case"groupsseparate":case"groupsvisible":case"groupsnone":break;default:return}f&&(b.preventDefault(),"delete"===e?y(d,function(){w(d,f,c)}):w(d,f,c))}}),a("body").on("click keypress",l.SECTIONLI+" "+l.SECTIONACTIONMENU+"[data-sectionid] a[data-action]",function(c){if("keypress"!==c.type||13===c.keyCode){var d=a(this),e=d.closest(l.SECTIONLI),f=d.closest(l.SECTIONACTIONMENU).attr("data-sectionid");c.preventDefault(),d.attr("data-confirm")?z(d.attr("data-confirm"),function(){D(e,f,d,b)}):D(e,f,d,b)}}),e.get_string("numberweeks").done(function(b){var c=a(l.ADDSECTIONS),d=c.attr("data-add-sections"),e=c.attr("new-sections"),f=a('
');f.find("label").html(b),h.create({title:d,type:h.types.SAVE_CANCEL,body:f.html()},c).done(function(b){var e=a(b.getBody()).find("#add_section_numsections"),f=function(){""+parseInt(e.val())===e.val()&&parseInt(e.val())>=1&&(document.location=c.attr("href")+"&numsections="+parseInt(e.val()))};b.setSaveButtonText(d),b.getRoot().on(i.shown,function(){e.focus().select().on("keydown",function(a){a.keyCode===j.enter&&f()})}),b.getRoot().on(i.save,function(a){a.preventDefault(),f()})})})},replaceSectionActionItem:function(a,b,c,d,e,f,g,h){var i=a.find(l.SECTIONACTIONMENU+" "+b);A(i,c,d,e,f,g,h)}}}); \ No newline at end of file diff --git a/course/amd/src/actions.js b/course/amd/src/actions.js index 27a84b23527..1caedcb3335 100644 --- a/course/amd/src/actions.js +++ b/course/amd/src/actions.js @@ -341,30 +341,21 @@ define(['jquery', 'core/ajax', 'core/templates', 'core/notification', 'core/str' * @param {String} image new image name ("i/show", "i/hide", etc.) * @param {String} stringname new string for the action menu item * @param {String} stringcomponent - * @param {String} titlestr string for "title" attribute (if different from stringname) - * @param {String} titlecomponent + * @param {String} titlestr not used + * @param {String} titlecomponent not used * @param {String} newaction new value for data-action attribute of the link * @return {Promise} promise which is resolved when the replacement has completed */ var replaceActionItem = function(actionitem, image, stringname, stringcomponent, titlestr, titlecomponent, newaction) { - var stringRequests = [{key: stringname, component: stringcomponent}]; - if (titlestr) { - stringRequests.push({key: titlestr, component: titlecomponent}); - } + // Do not provide an icon with duplicate, different text to the menu item. return str.get_strings(stringRequests).then(function(strings) { actionitem.find('span.menu-action-text').html(strings[0]); - actionitem.attr('title', strings[0]); - var title = ''; - if (titlestr) { - title = strings[1]; - actionitem.attr('title', title); - } - return templates.renderPix(image, 'core', title); + return templates.renderPix(image, 'core'); }).then(function(pixhtml) { actionitem.find('.icon').replaceWith(pixhtml); actionitem.attr('data-action', newaction); diff --git a/course/classes/analytics/indicator/activities_due.php b/course/classes/analytics/indicator/activities_due.php new file mode 100644 index 00000000000..425785173bc --- /dev/null +++ b/course/classes/analytics/indicator/activities_due.php @@ -0,0 +1,100 @@ +. + +/** + * Activities due indicator. + * + * @package core + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_course\analytics\indicator; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/calendar/externallib.php'); + +/** + * Activities due indicator. + * + * @package core + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class activities_due extends \core_analytics\local\indicator\binary { + + /** + * Returns the name. + * + * If there is a corresponding '_help' string this will be shown as well. + * + * @return \lang_string + */ + public static function get_name() : \lang_string { + return new \lang_string('indicator:activitiesdue'); + } + + /** + * required_sample_data + * + * @return string[] + */ + public static function required_sample_data() { + return array('user'); + } + + /** + * calculate_sample + * + * @param int $sampleid + * @param string $sampleorigin + * @param int $starttime + * @param int $endtime + * @return float + */ + protected function calculate_sample($sampleid, $sampleorigin, $starttime = false, $endtime = false) { + + $actionevents = \core_calendar_external::get_calendar_action_events_by_timesort($starttime, $endtime, 0, 1, + true, $sampleid); + + if ($actionevents->events) { + + // We first need to check that at least one of the core_calendar_provide_event_action + // callbacks has the $userid param. + foreach ($actionevents->events as $event) { + $nparams = $this->get_provide_event_action_num_params($event->modulename); + if ($nparams > 2) { + return self::get_max_value(); + } + } + } + + return self::get_min_value(); + } + + /** + * Returns the number of params declared in core_calendar_provide_event_action's implementation. + * + * @param string $modulename The module name + * @return int + */ + private function get_provide_event_action_num_params(string $modulename) { + $functionname = 'mod_' . $modulename . '_core_calendar_provide_event_action'; + $reflection = new \ReflectionFunction($functionname); + return $reflection->getNumberOfParameters(); + } +} diff --git a/course/delete.php b/course/delete.php index def41a6b7cc..66bb257276f 100644 --- a/course/delete.php +++ b/course/delete.php @@ -24,6 +24,7 @@ require_once(__DIR__ . '/../config.php'); require_once($CFG->dirroot . '/course/lib.php'); +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); $id = required_param('id', PARAM_INT); // Course ID. $delete = optional_param('delete', '', PARAM_ALPHANUM); // Confirmation hash. @@ -74,16 +75,26 @@ if ($delete === md5($course->timemodified)) { } $strdeletecheck = get_string("deletecheck", "", $courseshortname); -$strdeletecoursecheck = get_string("deletecoursecheck"); -$message = "{$strdeletecoursecheck}

{$coursefullname} ({$courseshortname})"; - -$continueurl = new moodle_url('/course/delete.php', array('id' => $course->id, 'delete' => md5($course->timemodified))); -$continuebutton = new single_button($continueurl, get_string('delete'), 'post'); $PAGE->navbar->add($strdeletecheck); $PAGE->set_title("$SITE->shortname: $strdeletecheck"); $PAGE->set_heading($SITE->fullname); echo $OUTPUT->header(); -echo $OUTPUT->confirm($message, $continuebutton, $categoryurl); + +// Only let user delete this course if there is not an async backup in progress. +if (!async_helper::is_async_pending($id, 'course', 'backup')) { + $strdeletecoursecheck = get_string("deletecoursecheck"); + $message = "{$strdeletecoursecheck}

{$coursefullname} ({$courseshortname})"; + + $continueurl = new moodle_url('/course/delete.php', array('id' => $course->id, 'delete' => md5($course->timemodified))); + $continuebutton = new single_button($continueurl, get_string('delete'), 'post'); + echo $OUTPUT->confirm($message, $continuebutton, $categoryurl); +} else { + // Async backup is pending, don't let user delete course. + echo $OUTPUT->notification(get_string('pendingasyncerror', 'backup'), 'error'); + echo $OUTPUT->container(get_string('pendingasyncdeletedetail', 'backup')); + echo $OUTPUT->continue_button($categoryurl); +} + echo $OUTPUT->footer(); exit; diff --git a/course/format/renderer.php b/course/format/renderer.php index 45ff162e6b4..2f94041f734 100644 --- a/course/format/renderer.php +++ b/course/format/renderer.php @@ -117,10 +117,9 @@ abstract class format_section_renderer_base extends plugin_renderer_base { $name = empty($value['name']) ? '' : $value['name']; $attr = empty($value['attr']) ? array() : $value['attr']; $class = empty($value['pixattr']['class']) ? '' : $value['pixattr']['class']; - $alt = empty($value['pixattr']['alt']) ? '' : $value['pixattr']['alt']; $al = new action_menu_link_secondary( new moodle_url($url), - new pix_icon($icon, $alt, null, array('class' => "smallicon " . $class)), + new pix_icon($icon, '', null, array('class' => "smallicon " . $class)), $name, $attr ); @@ -328,8 +327,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base { 'url' => new moodle_url('/course/editsection.php', array('id' => $section->id, 'sr' => $sectionreturn)), 'icon' => 'i/settings', 'name' => $streditsection, - 'pixattr' => array('class' => '', 'alt' => $streditsection), - 'attr' => array('class' => 'icon edit', 'title' => $streditsection)); + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'icon edit')); } if ($section->section) { @@ -343,8 +342,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base { 'url' => $url, 'icon' => 'i/hide', 'name' => $strhidefromothers, - 'pixattr' => array('class' => '', 'alt' => $strhidefromothers), - 'attr' => array('class' => 'icon editing_showhide', 'title' => $strhidefromothers, + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'icon editing_showhide', 'data-sectionreturn' => $sectionreturn, 'data-action' => 'hide')); } else { $strshowfromothers = get_string('showfromothers', 'format_'.$course->format); @@ -353,8 +352,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base { 'url' => $url, 'icon' => 'i/show', 'name' => $strshowfromothers, - 'pixattr' => array('class' => '', 'alt' => $strshowfromothers), - 'attr' => array('class' => 'icon editing_showhide', 'title' => $strshowfromothers, + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'icon editing_showhide', 'data-sectionreturn' => $sectionreturn, 'data-action' => 'show')); } } @@ -370,8 +369,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base { 'url' => $url, 'icon' => 'i/up', 'name' => $strmoveup, - 'pixattr' => array('class' => '', 'alt' => $strmoveup), - 'attr' => array('class' => 'icon moveup', 'title' => $strmoveup)); + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'icon moveup')); } $url = clone($baseurl); @@ -383,8 +382,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base { 'url' => $url, 'icon' => 'i/down', 'name' => $strmovedown, - 'pixattr' => array('class' => '', 'alt' => $strmovedown), - 'attr' => array('class' => 'icon movedown', 'title' => $strmovedown)); + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'icon movedown')); } } } @@ -405,8 +404,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base { 'url' => $url, 'icon' => 'i/delete', 'name' => $strdelete, - 'pixattr' => array('class' => '', 'alt' => $strdelete), - 'attr' => array('class' => 'icon editing_delete', 'title' => $strdelete)); + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'icon editing_delete')); } } @@ -1009,7 +1008,7 @@ abstract class format_section_renderer_base extends plugin_renderer_base { if ($sectionreturn !== null) { $url->param('sectionreturn', $sectionreturn); } - $icon = $this->output->pix_icon('t/add', $straddsections); + $icon = $this->output->pix_icon('t/add', ''); $newsections = $maxsections - $lastsection; echo html_writer::link($url, $icon . $straddsections, array('class' => 'add-sections', 'data-add-sections' => $straddsections, 'new-sections' => $newsections)); diff --git a/course/format/topics/renderer.php b/course/format/topics/renderer.php index 248c73a5cdb..1efad308bd5 100644 --- a/course/format/topics/renderer.php +++ b/course/format/topics/renderer.php @@ -123,21 +123,19 @@ class format_topics_renderer extends format_section_renderer_base { if ($section->section && has_capability('moodle/course:setcurrentsection', $coursecontext)) { if ($course->marker == $section->section) { // Show the "light globe" on/off. $url->param('marker', 0); - $markedthistopic = get_string('markedthistopic'); $highlightoff = get_string('highlightoff'); $controls['highlight'] = array('url' => $url, "icon" => 'i/marked', 'name' => $highlightoff, - 'pixattr' => array('class' => '', 'alt' => $markedthistopic), - 'attr' => array('class' => 'editing_highlight', 'title' => $markedthistopic, + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'editing_highlight', 'data-action' => 'removemarker')); } else { $url->param('marker', $section->section); - $markthistopic = get_string('markthistopic'); $highlight = get_string('highlight'); $controls['highlight'] = array('url' => $url, "icon" => 'i/marker', 'name' => $highlight, - 'pixattr' => array('class' => '', 'alt' => $markthistopic), - 'attr' => array('class' => 'editing_highlight', 'title' => $markthistopic, + 'pixattr' => array('class' => ''), + 'attr' => array('class' => 'editing_highlight', 'data-action' => 'setmarker')); } } diff --git a/course/lib.php b/course/lib.php index 6903e0d4b70..380a7625fd0 100644 --- a/course/lib.php +++ b/course/lib.php @@ -1891,7 +1891,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { if ($hasmanageactivities) { $actions['update'] = new action_menu_link_secondary( new moodle_url($baseurl, array('update' => $mod->id)), - new pix_icon('t/edit', $str->editsettings, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/edit', '', 'moodle', array('class' => 'iconsmall')), $str->editsettings, array('class' => 'editing_update', 'data-action' => 'update') ); @@ -1917,7 +1917,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { } $actions['moveright'] = new action_menu_link_secondary( new moodle_url($baseurl, array('id' => $mod->id, 'indent' => '1')), - new pix_icon($rightarrow, $str->moveright, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon($rightarrow, '', 'moodle', array('class' => 'iconsmall')), $str->moveright, array('class' => 'editing_moveright ' . $enabledclass, 'data-action' => 'moveright', 'data-keepopen' => true, 'data-sectionreturn' => $sr) @@ -1930,7 +1930,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { } $actions['moveleft'] = new action_menu_link_secondary( new moodle_url($baseurl, array('id' => $mod->id, 'indent' => '-1')), - new pix_icon($leftarrow, $str->moveleft, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon($leftarrow, '', 'moodle', array('class' => 'iconsmall')), $str->moveleft, array('class' => 'editing_moveleft ' . $enabledclass, 'data-action' => 'moveleft', 'data-keepopen' => true, 'data-sectionreturn' => $sr) @@ -1953,7 +1953,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { if ($displayedoncoursepage) { $actions['hide'] = new action_menu_link_secondary( new moodle_url($baseurl, array('hide' => $mod->id)), - new pix_icon('t/hide', $str->modhide, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/hide', '', 'moodle', array('class' => 'iconsmall')), $str->modhide, array('class' => 'editing_hide', 'data-action' => 'hide') ); @@ -1961,7 +1961,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { // Offer to "show" only if the section is visible. $actions['show'] = new action_menu_link_secondary( new moodle_url($baseurl, array('show' => $mod->id)), - new pix_icon('t/show', $str->modshow, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/show', '', 'moodle', array('class' => 'iconsmall')), $str->modshow, array('class' => 'editing_show', 'data-action' => 'show') ); @@ -1971,7 +1971,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { // When making the "stealth" module unavailable we perform the same action as hiding the visible module. $actions['hide'] = new action_menu_link_secondary( new moodle_url($baseurl, array('hide' => $mod->id)), - new pix_icon('t/unblock', $str->makeunavailable, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/unblock', '', 'moodle', array('class' => 'iconsmall')), $str->makeunavailable, array('class' => 'editing_makeunavailable', 'data-action' => 'hide', 'data-sectionreturn' => $sr) ); @@ -1982,7 +1982,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { $action = $sectionvisible ? 'stealth' : 'show'; $actions[$action] = new action_menu_link_secondary( new moodle_url($baseurl, array($action => $mod->id)), - new pix_icon('t/block', $str->makeavailable, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/block', '', 'moodle', array('class' => 'iconsmall')), $str->makeavailable, array('class' => 'editing_makeavailable', 'data-action' => $action, 'data-sectionreturn' => $sr) ); @@ -1995,7 +1995,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { course_allowed_module($mod->get_course(), $mod->modname)) { $actions['duplicate'] = new action_menu_link_secondary( new moodle_url($baseurl, array('duplicate' => $mod->id)), - new pix_icon('t/copy', $str->duplicate, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/copy', '', 'moodle', array('class' => 'iconsmall')), $str->duplicate, array('class' => 'editing_duplicate', 'data-action' => 'duplicate', 'data-sectionreturn' => $sr) ); @@ -2026,7 +2026,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { $actions[$actionname] = new action_menu_link_primary( new moodle_url($baseurl, array('id' => $mod->id, 'groupmode' => $nextgroupmode)), - new pix_icon($groupimage, $grouptitle, 'moodle', array('class' => 'iconsmall')), + new pix_icon($groupimage, '', 'moodle', array('class' => 'iconsmall')), $grouptitle, array('class' => 'editing_'. $actionname, 'data-action' => $nextactionname, 'aria-live' => 'assertive', 'data-sectionreturn' => $sr) @@ -2040,7 +2040,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { if (has_capability('moodle/role:assign', $modcontext)){ $actions['assign'] = new action_menu_link_secondary( new moodle_url('/admin/roles/assign.php', array('contextid' => $modcontext->id)), - new pix_icon('t/assignroles', $str->assign, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/assignroles', '', 'moodle', array('class' => 'iconsmall')), $str->assign, array('class' => 'editing_assign', 'data-action' => 'assignroles', 'data-sectionreturn' => $sr) ); @@ -2050,7 +2050,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) { if ($hasmanageactivities) { $actions['delete'] = new action_menu_link_secondary( new moodle_url($baseurl, array('delete' => $mod->id)), - new pix_icon('t/delete', $str->delete, 'moodle', array('class' => 'iconsmall', 'title' => '')), + new pix_icon('t/delete', '', 'moodle', array('class' => 'iconsmall')), $str->delete, array('class' => 'editing_delete', 'data-action' => 'delete', 'data-sectionreturn' => $sr) ); diff --git a/course/tests/behat/behat_course.php b/course/tests/behat/behat_course.php index 1039cfa83c1..7f2b057194c 100644 --- a/course/tests/behat/behat_course.php +++ b/course/tests/behat/behat_course.php @@ -306,7 +306,7 @@ class behat_course extends behat_base { // Click on highlight topic link. $this->execute('behat_general::i_click_on_in_the', - array(get_string('markthistopic'), "link", $this->escape($xpath), "xpath_element") + array(get_string('highlight'), "link", $this->escape($xpath), "xpath_element") ); } @@ -328,7 +328,7 @@ class behat_course extends behat_base { // Click on un-highlight topic link. $this->execute('behat_general::i_click_on_in_the', - array(get_string('markedthistopic'), "link", $this->escape($xpath), "xpath_element") + array(get_string('highlightoff'), "link", $this->escape($xpath), "xpath_element") ); } @@ -446,7 +446,7 @@ class behat_course extends behat_base { $xpath = $this->section_exists($sectionnumber); // The important checking, we can not check the img. - $this->execute('behat_general::should_exist_in_the', ['This topic is highlighted as the current topic', 'icon', $xpath, 'xpath_element']); + $this->execute('behat_general::should_exist_in_the', ['Remove highlight', 'link', $xpath, 'xpath_element']); } /** @@ -777,11 +777,10 @@ class behat_course extends behat_base { // Ensure the destination is valid. $sectionxpath = $this->section_exists($sectionnumber); - $activitynode = $this->get_activity_element('Move', 'icon', $activityname); - // JS enabled. if ($this->running_javascript()) { + $activitynode = $this->get_activity_element('Move', 'icon', $activityname); $destinationxpath = $sectionxpath . "/descendant::ul[contains(concat(' ', normalize-space(@class), ' '), ' yui3-dd-drop ')]"; $this->execute("behat_general::i_drag_and_i_drop_it_in", @@ -1186,9 +1185,9 @@ class behat_course extends behat_base { // Checking the show button alt text and show icon. $showtext = get_string('showfromothers', $courseformat); - $linkxpath = $xpath . "/descendant::a[@title=" . behat_context_helper::escape($showtext) . "]"; + $linkxpath = $xpath . "//a[*[contains(text(), " . behat_context_helper::escape($showtext) . ")]]"; - $exception = new ElementNotFoundException($this->getSession(), 'Show section link '); + $exception = new ElementNotFoundException($this->getSession(), 'Show section link'); // Returing the link so both Non-JS and JS browsers can interact with it. return $this->find('xpath', $linkxpath, $exception); diff --git a/course/view.php b/course/view.php index 81db3ce4280..e1a3e3c0b9b 100644 --- a/course/view.php +++ b/course/view.php @@ -5,6 +5,7 @@ require_once('../config.php'); require_once('lib.php'); require_once($CFG->libdir.'/completionlib.php'); + require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); $id = optional_param('id', 0, PARAM_INT); $name = optional_param('name', '', PARAM_TEXT); @@ -243,6 +244,10 @@ $PAGE->set_heading($course->fullname); echo $OUTPUT->header(); +if ($USER->editing == 1 && async_helper::is_async_pending($id, 'course', 'backup')) { + echo $OUTPUT->notification(get_string('pendingasyncedit', 'backup'), 'warning'); +} + if ($completion->is_enabled()) { // This value tracks whether there has been a dynamic change to the page. // It is used so that if a user does this - (a) set some tickmarks, (b) diff --git a/dataformat/pdf/classes/privacy/provider.php b/dataformat/pdf/classes/privacy/provider.php new file mode 100644 index 00000000000..770142b1dcf --- /dev/null +++ b/dataformat/pdf/classes/privacy/provider.php @@ -0,0 +1,46 @@ +. + +/** + * Privacy provider implementation for dataformat_pdf. + * + * @package dataformat_pdf + * @copyright 2019 Shamim Rezaie + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace dataformat_pdf\privacy; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Privacy provider implementation for dataformat_pdf. + * + * @copyright 2019 Shamim Rezaie + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class provider implements \core_privacy\local\metadata\null_provider { + + /** + * Get the language string identifier with the component's language + * file to explain why this plugin stores no data. + * + * @return string + */ + public static function get_reason() : string { + return 'privacy:metadata'; + } +} diff --git a/dataformat/pdf/classes/writer.php b/dataformat/pdf/classes/writer.php new file mode 100644 index 00000000000..0559c93aebc --- /dev/null +++ b/dataformat/pdf/classes/writer.php @@ -0,0 +1,139 @@ +. + +/** + * pdf data format writer + * + * @package dataformat_pdf + * @copyright 2019 Shamim Rezaie + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace dataformat_pdf; + +defined('MOODLE_INTERNAL') || die(); + +/** + * pdf data format writer + * + * @package dataformat_pdf + * @copyright 2019 Shamim Rezaie + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class writer extends \core\dataformat\base { + + public $mimetype = "application/pdf"; + + public $extension = ".pdf"; + + /** + * @var \pdf The pdf object that is used to generate the pdf file. + */ + protected $pdf; + + /** + * @var float Each column's width in the current sheet. + */ + protected $colwidth; + + /** + * @var string[] Title of columns in the current sheet. + */ + protected $columns; + + /** + * writer constructor. + */ + public function __construct() { + global $CFG; + require_once($CFG->libdir . '/pdflib.php'); + + $this->pdf = new \pdf(); + $this->pdf->setPrintHeader(false); + $this->pdf->SetFooterMargin(PDF_MARGIN_FOOTER); + + // Set background color for headings. + $this->pdf->SetFillColor(238, 238, 238); + } + + public function send_http_headers() { + } + + public function start_output() { + $this->pdf->AddPage('L'); + } + + public function start_sheet($columns) { + $margins = $this->pdf->getMargins(); + $pagewidth = $this->pdf->getPageWidth() - $margins['left'] - $margins['right']; + + $this->colwidth = $pagewidth / count($columns); + $this->columns = $columns; + + $this->print_heading(); + } + + public function write_record($record, $rownum) { + $rowheight = 0; + + foreach ($record as $cell) { + $rowheight = max($rowheight, $this->pdf->getStringHeight($this->colwidth, $cell, false, true, '', 1)); + } + + $margins = $this->pdf->getMargins(); + if ($this->pdf->GetY() + $rowheight + $margins['bottom'] > $this->pdf->getPageHeight()) { + $this->pdf->AddPage('L'); + $this->print_heading(); + } + + $total = count($record); + $counter = 1; + foreach ($record as $cell) { + $nextposition = ($counter == $total) ? 1 : 0; + $this->pdf->Multicell($this->colwidth, $rowheight, $cell, 1, 'L', false, $nextposition); + $counter++; + } + } + + public function close_output() { + $filename = $this->filename . $this->get_extension(); + + $this->pdf->Output($filename, 'D'); + } + + /** + * Prints the heading row. + */ + private function print_heading() { + $fontfamily = $this->pdf->getFontFamily(); + $fontstyle = $this->pdf->getFontStyle(); + $this->pdf->SetFont($fontfamily, 'B'); + $rowheight = 0; + foreach ($this->columns as $columns) { + $rowheight = max($rowheight, $this->pdf->getStringHeight($this->colwidth, $columns, false, true, '', 1)); + } + + $total = count($this->columns); + $counter = 1; + foreach ($this->columns as $columns) { + $nextposition = ($counter == $total) ? 1 : 0; + $this->pdf->Multicell($this->colwidth, $rowheight, $columns, 1, 'C', true, $nextposition); + $counter++; + } + + $this->pdf->SetFont($fontfamily, $fontstyle); + } +} diff --git a/dataformat/pdf/lang/en/dataformat_pdf.php b/dataformat/pdf/lang/en/dataformat_pdf.php new file mode 100644 index 00000000000..e4ce79c2474 --- /dev/null +++ b/dataformat/pdf/lang/en/dataformat_pdf.php @@ -0,0 +1,28 @@ +. + +/** + * pdf dataformat lang strings. + * + * @package dataformat_pdf + * @copyright 2019 Shamim Rezaie + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +$string['dataformat'] = 'Portable Document Format (.pdf)'; +$string['privacy:metadata'] = 'The PDF data format plugin does not store any personal data.'; +$string['shortname'] = 'PDF'; + diff --git a/dataformat/pdf/version.php b/dataformat/pdf/version.php new file mode 100644 index 00000000000..99c62d8f956 --- /dev/null +++ b/dataformat/pdf/version.php @@ -0,0 +1,29 @@ +. + +/** + * Data activity filter version information + * + * @package dataformat_pdf + * @copyright 2019 Shamim Rezaie + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +$plugin->version = 2019040100; +$plugin->requires = 2018112800; // Requires this Moodle version. +$plugin->component = 'dataformat_pdf'; diff --git a/lang/en/admin.php b/lang/en/admin.php index b5964e95fdd..0e8f39bb577 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -1216,6 +1216,7 @@ $string['taskanalyticscleanup'] = 'Analytics cleanup'; $string['taskautomatedbackup'] = 'Automated backups'; $string['taskbackupcleanup'] = 'Clean backup tables and logs'; $string['taskbadgescron'] = 'Award badges'; +$string['taskbadgesmessagecron'] = 'Background processing for sending badges notifications'; $string['taskblogcron'] = 'Sync external blogs'; $string['taskcachecleanup'] = 'Remove expired cache entries'; $string['taskcachecron'] = 'Background processing for caches'; diff --git a/lang/en/analytics.php b/lang/en/analytics.php index a25a3cd69d2..2849ab6243a 100644 --- a/lang/en/analytics.php +++ b/lang/en/analytics.php @@ -67,9 +67,10 @@ $string['errorunknownaction'] = 'Unknown action'; $string['eventpredictionactionstarted'] = 'Prediction process started'; $string['eventinsightsviewed'] = 'Insights viewed'; $string['fixedack'] = 'Acknowledged'; -$string['insightmessagesubject'] = 'New insight for "{$a->contextname}": {$a->insightname}'; -$string['insightinfomessage'] = 'The system generated some insights for you: {$a}'; -$string['insightinfomessagehtml'] = 'The system generated some insights for you: {$a}.'; +$string['insightmessagesubject'] = 'New insight for "{$a}"'; +$string['insightinfomessage'] = 'The system generated an insight for you: {$a}'; +$string['insightinfomessagehtml'] = 'The system generated an insight for you.'; +$string['insightinfomessageaction'] = '{$a->text}: {$a->url}'; $string['invalidtimesplitting'] = 'Model with ID {$a} needs a time-splitting method before it can be used to train.'; $string['invalidanalysablefortimesplitting'] = 'It cannot be analysed using {$a} time-splitting method.'; $string['nocourses'] = 'No courses to analyse'; @@ -122,5 +123,9 @@ $string['privacy:metadata:analytics:predictionactions:timecreated'] = 'When the $string['processingsitecontents'] = 'Processing site contents'; $string['successfullyanalysed'] = 'Successfully analysed'; $string['timesplittingmethod'] = 'Time-splitting method'; -$string['timesplittingmethod_help'] = 'The time-splitting method is what defines when the system will calculate predictions and the portion of activity logs that will be considered for those predictions. E.g. They can divide the course duration in parts and generate a prediction at the end of these parts.'; +$string['timesplittingmethod_help'] = 'The time-splitting method is what defines when the system will calculate predictions and the portion of activity logs that will be considered for those predictions. E.g. they can divide the course duration in parts and generate a prediction at the end of these parts.'; +$string['timesplittingmethod_link'] = 'Time_splitting_methods'; +$string['viewdetails'] = 'View details'; +$string['viewinsight'] = 'View insight'; +$string['viewinsightdetails'] = 'View insight details'; $string['viewprediction'] = 'View prediction details'; diff --git a/lang/en/backup.php b/lang/en/backup.php index 2fdc6b33a1f..c518dab3d2c 100644 --- a/lang/en/backup.php +++ b/lang/en/backup.php @@ -22,6 +22,36 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +$string['asyncbackupcomplete'] = 'The backup process has completed'; +$string['asyncbackupcompletebutton'] = 'Continue'; +$string['asyncbackupcompletedetail'] = 'The backup process has completed successfully completed.
You can access the backup in the restore page.'; +$string['asyncbackuperror'] = 'The backup process has failed'; +$string['asyncbackuperrordetail'] = 'The backup process has failed. Please contact your system administrator.'; +$string['asyncbackuppending'] = 'The backup process is pending'; +$string['asyncbackupprocessing'] = 'The backup is in progress'; +$string['asyncbadexecution'] = 'Bad backup controller execution, is {$a} should be 2'; +$string['asynccheckprogress'] = ' You can check the progress at anytime at the restore page.'; +$string['asyncgeneralsettings'] = 'Asynchronous backup/restore general settings'; +$string['asyncemailenable'] = 'Enable message notifications'; +$string['asyncemailenabledetail'] = 'When enabled users will receive a message when an asynchronous restore/backup completes'; +$string['asyncmessagebody'] = 'Message'; +$string['asyncmessagebodydetail'] = 'Message to send when an asynchronous restore/backup completes'; +$string['asyncmessagebodydefault'] = 'Dear {user_firstname} {user_lastname},
Your {operation} (ID: {backupid}) has completed successfully!

You can view it here {link}.
Kind Regards,
Your Moodle Administrator.'; +$string['asyncmessagesubject'] = 'Subject'; +$string['asyncmessagesubjectdetail'] = 'Message subject'; +$string['asyncmessagesubjectdefault'] = 'Moodle {operation} completed successfully'; +$string['asyncnowait'] = 'You don\'t need to wait here, the process will continue in the background.'; +$string['asyncprocesspending'] = 'Process pending'; +$string['asyncrestorecomplete'] = 'The restore process has completed'; +$string['asyncrestorecompletebutton'] = 'Continue'; +$string['asyncrestorecompletedetail'] = 'The restore process has completed successfully completed. Clicking continue will take you to the course for the restored item.'; +$string['asyncrestoreerror'] = 'The restore process has failed'; +$string['asyncrestoreerrordetail'] = 'The restore process has failed. Please contact your system administrator.'; +$string['asyncrestorepending'] = 'The restore process is pending'; +$string['asyncrestoreprocessing'] = 'The restore is in progress'; +$string['asyncreturn'] = 'Return to course'; +$string['asyncrestoreinprogress'] = 'Restores in progress'; +$string['asyncrestoreinprogress_help'] = 'Asynchronous course restores that are in progress are shown here.'; $string['autoactivedisabled'] = 'Disabled'; $string['autoactiveenabled'] = 'Enabled'; $string['autoactivemanual'] = 'Manual'; @@ -61,6 +91,7 @@ $string['backupmode30'] = 'Hub'; $string['backupmode40'] = 'Same site'; $string['backupmode50'] = 'Automated'; $string['backupmode60'] = 'Converted'; +$string['backupmode70'] = 'Asynchronous'; $string['backupsection'] = 'Backup course section: {$a}'; $string['backupsettings'] = 'Backup settings'; $string['backupsitedetails'] = 'Site details'; @@ -137,6 +168,8 @@ $string['currentstage2'] = 'Schema settings'; $string['currentstage4'] = 'Confirmation and review'; $string['currentstage8'] = 'Perform backup'; $string['currentstage16'] = 'Complete'; +$string['enableasyncbackup'] = 'Enable asynchronous backups'; +$string['enableasyncbackup_help'] = 'If enabled, all backup and restore operations will be done asynchronously. This does not effect imports and exports. Asynchronous backups and restores allow users to do other operations while a backup or restore is in progress.'; $string['enterasearch'] = 'Enter a search'; $string['error_block_for_module_not_found'] = 'Orphan block instance (id: {$a->bid}) for course module (id: {$a->mid}) found. This block will not be backed up'; $string['error_course_module_not_found'] = 'Orphan course module (id: {$a}) found. This module will not be backed up.'; @@ -149,6 +182,7 @@ $string['errorinvalidformat'] = 'Unknown backup format'; $string['errorinvalidformatinfo'] = 'The selected file is not a valid Moodle backup file and can\'t be restored.'; $string['errorrestorefrontpagebackup'] = 'You can only restore front page backups on the front page'; $string['executionsuccess'] = 'The backup file was successfully created.'; +$string['failed'] = 'Backup failed'; $string['filename'] = 'Filename'; $string['filealiasesrestorefailures'] = 'Aliases restore failures'; $string['filealiasesrestorefailuresinfo'] = 'Some aliases included in the backup file could not be restored. The following list contains their expected location and the source file they were referring to at the original site.'; @@ -203,6 +237,7 @@ $string['importcurrentstage16'] = 'Complete'; $string['importrootsettings'] = 'Import settings'; $string['importsettings'] = 'General import settings'; $string['importsuccess'] = 'Import complete. Click continue to return to the course.'; +$string['inprogress'] = 'Backup in progress'; $string['includeactivities'] = 'Include:'; $string['includeditems'] = 'Included items:'; $string['includesection'] = 'Section {$a}'; @@ -223,6 +258,10 @@ $string['nomatchingcourses'] = 'There are no courses to display'; $string['norestoreoptions'] = 'There are no categories or existing courses you can restore to.'; $string['originalwwwroot'] = 'URL of backup'; $string['overwrite'] = 'Overwrite'; +$string['pendingasyncdetail'] = 'Asynchronous backups only allow a user to have one pending backup for a resource at a time.
Muliple asynchronous backups of the same resource can\'t be queued, as this would likely result in multiple backups with the same content.'; +$string['pendingasyncdeletedetail'] = 'This course has an asynchronous backup pending.
Courses can\'t be deleted until this backup finishes.'; +$string['pendingasyncedit'] = 'There is a pending asynchronous backup for this course. Please do not edit this course until backup is complete.'; +$string['pendingasyncerror'] = 'Backup pending for this resource'; $string['previousstage'] = 'Previous'; $string['preparingui'] = 'Preparing to display page'; $string['preparingdata'] = 'Preparing data'; @@ -323,6 +362,9 @@ $string['skipmodifdays'] = 'Skip courses not modified since'; $string['skipmodifdayshelp'] = 'Choose to skip courses that have not been modified since a number of days'; $string['skipmodifprev'] = 'Skip courses not modified since previous backup'; $string['skipmodifprevhelp'] = 'Choose whether to skip courses that have not been modified since the last automatic backup. This requires logging to be enabled.'; +$string['status'] = 'Status'; +$string['successful'] = 'Backup successful'; +$string['successfulrestore'] = 'Restore successful'; $string['timetaken'] = 'Time taken'; $string['title'] = 'Title'; $string['totalcategorysearchresults'] = 'Total categories: {$a}'; diff --git a/lang/en/course.php b/lang/en/course.php index aba28b38c76..16e50800c97 100644 --- a/lang/en/course.php +++ b/lang/en/course.php @@ -36,6 +36,7 @@ $string['customfield_visibletoall'] = 'Everyone'; $string['customfield_visibletoteachers'] = 'Teachers'; $string['customfieldsettings'] = 'Settings for course custom fields'; $string['favourite'] = 'Starred course'; +$string['gradetopassnotset'] = 'This course does not have a grade to pass set. It may be set in the grade item of the course (Gradebook setup).'; $string['privacy:perpage'] = 'The number of courses to show per page.'; $string['privacy:completionpath'] = 'Course completion'; $string['privacy:favouritespath'] = 'Course starred information'; diff --git a/lang/en/moodle.php b/lang/en/moodle.php index 5fb1e314289..f531b32d335 100644 --- a/lang/en/moodle.php +++ b/lang/en/moodle.php @@ -1053,6 +1053,8 @@ $string['indicator:accessesafterend'] = 'Course accessed after end date'; $string['indicator:accessesafterend_help'] = 'This indicator reflects if the student accessed the course after the course end date.'; $string['indicator:accessesbeforestart'] = 'Course accessed before start date'; $string['indicator:accessesbeforestart_help'] = 'This indicator reflects if the student accessed the course before the course start date.'; +$string['indicator:activitiesdue'] = 'Activities due'; +$string['indicator:activitiesdue_help'] = 'The user has activities due.'; $string['indicator:anywrite'] = 'Any write action'; $string['indicator:anywrite_help'] = 'This indicator represents any write (submit) action taken by the student.'; $string['indicator:anywriteincourse'] = 'Any write action in the course'; @@ -1188,6 +1190,7 @@ $string['memberincourse'] = 'People in the course'; $string['messagebody'] = 'Message body'; $string['messagedselectedusers'] = 'Selected users have been messaged and the recipient list has been reset.'; $string['messagedselectedusersfailed'] = 'Something went wrong while messaging selected users. Some may have received the email.'; +$string['messageprovider:asyncbackupnotification'] = 'Asynchronous backup/restore notifications'; $string['messageprovider:availableupdate'] = 'Available update notifications'; $string['messageprovider:backup'] = 'Backup notifications'; $string['messageprovider:badgecreatornotice'] = 'Badge creator notifications'; @@ -1428,6 +1431,7 @@ $string['nosuchemail'] = 'No such email address'; $string['notavailable'] = 'Not available'; $string['notavailablecourse'] = '{$a} is not available'; $string['noteachersyet'] = 'No teachers in this course yet'; +$string['noteachingupcomingcourses'] = 'Upcoming courses have no teachers or students'; $string['notenrolled'] = '{$a} is not enrolled in this course.'; $string['notenrolledprofile'] = 'This profile is not available because this user is not enrolled in this course.'; $string['noteusercannotrolldatesoncontext'] = 'Note: The ability to roll dates when restoring this backup has been disabled because you lack the required permissions.'; @@ -1947,6 +1951,7 @@ $string['stringsnotset'] = 'The following strings are not defined in {$a}'; $string['studentnotallowed'] = 'Sorry, but you can not enter this course as \'{$a}\''; $string['students'] = 'Students'; $string['studentsandteachers'] = 'Students and teachers'; +$string['studentsatriskincourse'] = 'Students at risk in {$a} course'; $string['subcategories'] = 'Subcategories'; $string['subcategory'] = 'Subcategory'; $string['subcategoryof'] = 'Subcategory of {$a}'; @@ -1976,14 +1981,20 @@ $string['target:coursecompetencies'] = 'Students at risk of not achieving the co $string['target:coursecompetencies_help'] = 'This target describes whether a student is at risk of not achieving the competencies assigned to a course. This target considers that all competencies assigned to the course must be achieved by the end of the course.'; $string['target:coursedropout'] = 'Students at risk of dropping out'; $string['target:coursedropout_help'] = 'This target describes whether the student is considered at risk of dropping out.'; +$string['target:coursegradetopass'] = 'Students at risk of not getting the minimum grade to pass the course.'; +$string['target:coursegradetopass_help'] = 'This target describes whether the student is at risk of not getting the minimum grade to pass the course.'; $string['target:noteachingactivity'] = 'No teaching'; $string['target:noteachingactivity_help'] = 'This target describes whether courses due to start in the coming week will have teaching activity.'; +$string['target:upcomingactivitiesdue'] = 'Upcoming activities due'; +$string['target:upcomingactivitiesdue_help'] = 'This target generates reminders for upcoming activities due.'; $string['targetlabelstudentcompletionno'] = 'Student who is likely to meet the course completion conditions'; $string['targetlabelstudentcompletionyes'] = 'Student at risk of not meeting the course completion conditions'; $string['targetlabelstudentcompetenciesno'] = 'Student who is likely to achieve the competencies assigned to a course'; $string['targetlabelstudentcompetenciesyes'] = 'Student at risk of not achieving the competencies assigned to a course'; $string['targetlabelstudentdropoutyes'] = 'Student at risk of dropping out'; $string['targetlabelstudentdropoutno'] = 'Not at risk'; +$string['targetlabelstudentgradetopassno'] = 'Student who is likely to meet the minimum grade to pass the course.'; +$string['targetlabelstudentgradetopassyes'] = 'Student at risk of not meeting the minimum grade to pass the course.'; $string['targetlabelteachingyes'] = 'Users with teaching capabilities have access to the course'; $string['targetlabelteachingno'] = 'No teaching'; $string['targetrole'] = 'Target role'; @@ -2010,10 +2021,8 @@ $string['timesplitting:quartersaccum'] = 'Quarters accumulative'; $string['timesplitting:quartersaccum_help'] = 'This time-splitting method divides the course into quarters (4 equal parts), with each prediction being based on the data of all previous quarters.'; $string['timesplitting:singlerange'] = 'Single range'; $string['timesplitting:singlerange_help'] = 'This time-splitting method considers the entire course as a single span.'; -$string['timesplitting:weekly'] = 'Weekly'; -$string['timesplitting:weekly_help'] = 'This time-splitting method divides the course into weeks (7 days), with each prediction being based on the data of only the most recent previous week.'; -$string['timesplitting:weeklyaccum'] = 'Weekly accumulative'; -$string['timesplitting:weeklyaccum_help'] = 'This time-splitting method divides the course into weeks (7 days), with each prediction being based on the data of all previous weeks.'; +$string['timesplitting:upcomingweek'] = 'Upcoming week'; +$string['timesplitting:upcomingweek_help'] = 'This time-splitting method generates predictions every week. The indicators calculations will be based on the upcoming week.'; $string['thanks'] = 'Thanks'; $string['theme'] = 'Theme'; $string['themes'] = 'Themes'; @@ -2207,6 +2216,7 @@ $string['years'] = 'years'; $string['yes'] = 'Yes'; $string['youareabouttocreatezip'] = 'You are about to create a zip file containing'; $string['youaregoingtorestorefrom'] = 'You are about to start the restore process for'; +$string['youhaveupcomingactivitiesdue'] = 'You have upcoming activities due'; $string['youneedtoenrol'] = 'To perform that action you need to enrol in this course.'; $string['yourlastlogin'] = 'Your last login was'; $string['yourself'] = 'yourself'; diff --git a/lang/en/role.php b/lang/en/role.php index 79fea6d4144..ef271e8b7ce 100644 --- a/lang/en/role.php +++ b/lang/en/role.php @@ -37,6 +37,7 @@ $string['allowswitch'] = 'Allow role switches'; $string['allowview'] = 'Allow role to view'; $string['allsiteusers'] = 'All site users'; $string['analytics:listinsights'] = 'List insights'; +$string['analytics:listowninsights'] = 'List own insights'; $string['analytics:managemodels'] = 'Manage models'; $string['archetype'] = 'Role archetype'; $string['archetype_help'] = 'The role archetype determines the permissions when a role is reset to default. It also determines any new permissions for the role when the site is upgraded.'; diff --git a/lib/amd/build/icon_system_fontawesome.min.js b/lib/amd/build/icon_system_fontawesome.min.js index 2a6fd4019da..9b9bcf3f158 100644 --- a/lib/amd/build/icon_system_fontawesome.min.js +++ b/lib/amd/build/icon_system_fontawesome.min.js @@ -1 +1 @@ -define(["core/icon_system","jquery","core/ajax","core/mustache","core/localstorage","core/url"],function(a,b,c,d,e,f){var g=null,h=null,i=function(){a.apply(this,arguments)};return i.prototype=Object.create(a.prototype),i.prototype.init=function(){if(g)return b.when(this);var a=e.get("core/iconmap-fontawesome");return a&&(a=JSON.parse(a)),a?(g=a,b.when(this)):(null===h&&(h=c.call([{methodname:"core_output_load_fontawesome_icon_map",args:[]}],!0,!1)[0]),h.then(function(a){return g={},b.each(a,function(a,b){g[b.component+"/"+b.pix]=b.to}),e.set("core/iconmap-fontawesome",JSON.stringify(g)),this}.bind(this)))},i.prototype.renderIcon=function(a,b,c,e){var h=g[b+"/"+a],i=!1;if("undefined"==typeof h){var j=f.imageUrl(a,b);i={attributes:[{name:"src",value:j},{name:"alt",value:c},{name:"title",value:c}]}}var k={key:h,title:c,alt:c,unmappedIcon:i},l=d.render(e,k);return l.trim()},i.prototype.getTemplateName=function(){return"core/pix_icon_fontawesome"},i}); \ No newline at end of file +define(["core/icon_system","jquery","core/ajax","core/mustache","core/localstorage","core/url"],function(a,b,c,d,e,f){var g=null,h=null,i=function(){a.apply(this,arguments)};return i.prototype=Object.create(a.prototype),i.prototype.init=function(){if(g)return b.when(this);var a=e.get("core/iconmap-fontawesome");return a&&(a=JSON.parse(a)),a?(g=a,b.when(this)):(null===h&&(h=c.call([{methodname:"core_output_load_fontawesome_icon_map",args:[]}],!0,!1)[0]),h.then(function(a){return g={},b.each(a,function(a,b){g[b.component+"/"+b.pix]=b.to}),e.set("core/iconmap-fontawesome",JSON.stringify(g)),this}.bind(this)))},i.prototype.renderIcon=function(a,b,c,e){var h=g[b+"/"+a],i=!1;if("undefined"==typeof h){var j=f.imageUrl(a,b);i={attributes:[{name:"src",value:j},{name:"alt",value:c},{name:"title",value:c}]}}var k={key:h,title:c,alt:c,unmappedIcon:i};"undefined"!=typeof c&&""!==c||(k["aria-hidden"]=!0);var l=d.render(e,k);return l.trim()},i.prototype.getTemplateName=function(){return"core/pix_icon_fontawesome"},i}); \ No newline at end of file diff --git a/lib/amd/build/icon_system_standard.min.js b/lib/amd/build/icon_system_standard.min.js index d165a79845c..4591141c2a9 100644 --- a/lib/amd/build/icon_system_standard.min.js +++ b/lib/amd/build/icon_system_standard.min.js @@ -1 +1 @@ -define(["core/icon_system","core/url","core/mustache"],function(a,b,c){var d=function(){a.apply(this,arguments)};return d.prototype=Object.create(a.prototype),d.prototype.renderIcon=function(a,d,e,f){var g=b.imageUrl(a,d),h={attributes:[{name:"src",value:g},{name:"alt",value:e},{name:"title",value:e}]},i=c.render(f,h);return i.trim()},d.prototype.getTemplateName=function(){return"core/pix_icon"},d}); \ No newline at end of file +define(["core/icon_system","core/url","core/mustache"],function(a,b,c){var d=function(){a.apply(this,arguments)};return d.prototype=Object.create(a.prototype),d.prototype.renderIcon=function(a,d,e,f){var g=b.imageUrl(a,d),h={attributes:[{name:"src",value:g},{name:"alt",value:e},{name:"title",value:e}]};"undefined"!=typeof e&&""!=e||h.attributes.push({name:"aria-hidden",value:"true"});var i=c.render(f,h);return i.trim()},d.prototype.getTemplateName=function(){return"core/pix_icon"},d}); \ No newline at end of file diff --git a/lib/amd/src/icon_system_fontawesome.js b/lib/amd/src/icon_system_fontawesome.js index 970b0b95911..5ef205ac2fc 100644 --- a/lib/amd/src/icon_system_fontawesome.js +++ b/lib/amd/src/icon_system_fontawesome.js @@ -105,6 +105,10 @@ define(['core/icon_system', 'jquery', 'core/ajax', 'core/mustache', 'core/locals unmappedIcon: unmappedIcon }; + if (typeof title === "undefined" || title === '') { + context['aria-hidden'] = true; + } + var result = Mustache.render(template, context); return result.trim(); }; diff --git a/lib/amd/src/icon_system_standard.js b/lib/amd/src/icon_system_standard.js index 516245474d9..a1bce77a6ab 100644 --- a/lib/amd/src/icon_system_standard.js +++ b/lib/amd/src/icon_system_standard.js @@ -52,6 +52,10 @@ define(['core/icon_system', 'core/url', 'core/mustache'], {name: 'title', value: title} ] }; + if (typeof title === "undefined" || title == "") { + templatecontext.attributes.push({name: 'aria-hidden', value: 'true'}); + } + var result = Mustache.render(template, templatecontext); return result.trim(); }; diff --git a/lib/badgeslib.php b/lib/badgeslib.php index 066927c58dd..2f156d92dbd 100644 --- a/lib/badgeslib.php +++ b/lib/badgeslib.php @@ -1611,3 +1611,50 @@ function badge_award_criteria_competency_has_records_for_competencies($competenc return $DB->record_exists_sql($sql, $params); } + +/** + * Creates single message for all notification and sends it out + * + * @param object $badge A badge which is notified about. + */ +function badge_assemble_notification(stdClass $badge) { + global $DB; + + $userfrom = core_user::get_noreply_user(); + $userfrom->maildisplay = true; + + if ($msgs = $DB->get_records_select('badge_issued', 'issuernotified IS NULL AND badgeid = ?', array($badge->id))) { + // Get badge creator. + $creator = $DB->get_record('user', array('id' => $badge->creator), '*', MUST_EXIST); + $creatorsubject = get_string('creatorsubject', 'badges', $badge->name); + $creatormessage = ''; + + // Put all messages in one digest. + foreach ($msgs as $msg) { + $issuedlink = html_writer::link(new moodle_url('/badges/badge.php', array('hash' => $msg->uniquehash)), $badge->name); + $recipient = $DB->get_record('user', array('id' => $msg->userid), '*', MUST_EXIST); + + $a = new stdClass(); + $a->user = fullname($recipient); + $a->link = $issuedlink; + $creatormessage .= get_string('creatorbody', 'badges', $a); + $DB->set_field('badge_issued', 'issuernotified', time(), array('badgeid' => $msg->badgeid, 'userid' => $msg->userid)); + } + + // Create a message object. + $eventdata = new \core\message\message(); + $eventdata->courseid = SITEID; + $eventdata->component = 'moodle'; + $eventdata->name = 'badgecreatornotice'; + $eventdata->userfrom = $userfrom; + $eventdata->userto = $creator; + $eventdata->notification = 1; + $eventdata->subject = $creatorsubject; + $eventdata->fullmessage = format_text_email($creatormessage, FORMAT_HTML); + $eventdata->fullmessageformat = FORMAT_PLAIN; + $eventdata->fullmessagehtml = $creatormessage; + $eventdata->smallmessage = $creatorsubject; + + message_send($eventdata); + } +} diff --git a/lib/behat/classes/partial_named_selector.php b/lib/behat/classes/partial_named_selector.php index 91757840ec5..26fa673fe89 100644 --- a/lib/behat/classes/partial_named_selector.php +++ b/lib/behat/classes/partial_named_selector.php @@ -89,6 +89,7 @@ class behat_partial_named_selector extends \Behat\Mink\Selector\PartialNamedSele 'file' => 'file', 'filemanager' => 'filemanager', 'group_message' => 'group_message', + 'group_message_conversation' => 'group_message_conversation', 'group_message_header' => 'group_message_header', 'group_message_member' => 'group_message_member', 'group_message_tab' => 'group_message_tab', @@ -154,7 +155,10 @@ XPATH , 'group_message' => << << << << <<get_id()); diff --git a/lib/classes/analytics/analyser/site_courses.php b/lib/classes/analytics/analyser/site_courses.php index aa3b50e8d60..e2f817e060c 100644 --- a/lib/classes/analytics/analyser/site_courses.php +++ b/lib/classes/analytics/analyser/site_courses.php @@ -79,7 +79,7 @@ class site_courses extends \core_analytics\local\analyser\sitewide { * @param \core_analytics\analysable $site * @return array */ - protected function get_all_samples(\core_analytics\analysable $site) { + public function get_all_samples(\core_analytics\analysable $site) { global $DB; // Getting courses from DB instead of from the site as these samples diff --git a/lib/classes/analytics/analyser/student_enrolments.php b/lib/classes/analytics/analyser/student_enrolments.php index 758f67bf739..d35c0a33769 100644 --- a/lib/classes/analytics/analyser/student_enrolments.php +++ b/lib/classes/analytics/analyser/student_enrolments.php @@ -111,7 +111,7 @@ class student_enrolments extends \core_analytics\local\analyser\by_course { * @param \core_analytics\analysable $course * @return array */ - protected function get_all_samples(\core_analytics\analysable $course) { + public function get_all_samples(\core_analytics\analysable $course) { $enrolments = enrol_get_course_users($course->get_id()); diff --git a/lib/classes/analytics/analyser/users.php b/lib/classes/analytics/analyser/users.php new file mode 100644 index 00000000000..40ba586e762 --- /dev/null +++ b/lib/classes/analytics/analyser/users.php @@ -0,0 +1,191 @@ +. + +/** + * Users analyser (insights for users). + * + * @package core + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\analytics\analyser; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Users analyser (insights for users). + * + * @package core + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class users extends \core_analytics\local\analyser\base { + + /** + * The site users are the analysable elements returned by this analyser. + * + * @param string|null $action 'prediction', 'training' or null if no specific action needed. + * @return \Iterator + */ + public function get_analysables_iterator(?string $action = null) { + global $DB, $CFG; + + $siteadmins = explode(',', $CFG->siteadmins); + + list($sql, $params) = $this->get_iterator_sql('user', CONTEXT_USER, $action, 'u'); + + $sql .= " AND u.deleted = :deleted AND u.confirmed = :confirmed AND u.suspended = :suspended"; + $params = $params + ['deleted' => 0, 'confirmed' => 1, 'suspended' => 0]; + + $ordersql = $this->order_sql('timecreated', 'ASC', 'u'); + + $recordset = $DB->get_recordset_sql($sql, $params); + if (!$recordset->valid()) { + $this->add_log(get_string('nousersfound')); + return new \ArrayIterator([]); + } + + return new \core\dml\recordset_walk($recordset, function($record) use ($siteadmins) { + + if (in_array($record->id, $siteadmins) || isguestuser($record->id)) { + // Skip admins and the guest user. + return false; + } + $context = \context_helper::preload_from_record($record); + return \core_analytics\user::instance($record, $context); + }); + } + + /** + * Just one sample per analysable. + * + * @return bool + */ + public static function one_sample_per_analysable() { + return true; + } + + /** + * Samples origin is user table. + * + * @return string + */ + public function get_samples_origin() { + return 'user'; + } + + /** + * Returns the analysable of a sample + * + * @param int $sampleid + * @return \core_analytics\analysable + */ + public function get_sample_analysable($sampleid) { + return \core_analytics\user::instance($sampleid); + } + + /** + * This provides samples' user and context. + * + * @return string[] + */ + protected function provided_sample_data() { + return ['user', 'context']; + } + + /** + * Returns the context of a sample. + * + * @param int $sampleid + * @return \context + */ + public function sample_access_context($sampleid) { + return \context_user::instance($sampleid); + } + + /** + * This will return just one user as we analyse users separately. + * + * @param \core_analytics\analysable $user + * @return array + */ + public function get_all_samples(\core_analytics\analysable $user) { + + $context = \context_user::instance($user->get_id()); + + // Just 1 sample per analysable. + return [ + [$user->get_id() => $user->get_id()], + [$user->get_id() => ['user' => $user->get_user_data(), 'context' => $context]] + ]; + } + + /** + * Returns samples data from sample ids. + * + * @param int[] $sampleids + * @return array + */ + public function get_samples($sampleids) { + global $DB; + + list($sql, $params) = $DB->get_in_or_equal($sampleids, SQL_PARAMS_NAMED); + $users = $DB->get_records_select('user', "id $sql", $params); + + $userids = array_keys($users); + $sampleids = array_combine($userids, $userids); + + $users = array_map(function($user) { + return ['user' => $user, 'context' => \context_user::instance($user->id)]; + }, $users); + + // No related data attached. + return [$sampleids, $users]; + } + + /** + * Returns the description of a sample. + * + * @param int $sampleid + * @param int $contextid + * @param array $sampledata + * @return array array(string, \renderable) + */ + public function sample_description($sampleid, $contextid, $sampledata) { + $description = fullname($sampledata['user']); + return [$description, new \user_picture($sampledata['user'])]; + } + + /** + * We need to delete associated data if a user requests his data to be deleted. + * + * @return bool + */ + public function processes_user_data() { + return true; + } + + /** + * Join the samples origin table with the user id table. + * + * @param string $sampletablealias + * @return string + */ + public function join_sample_user($sampletablealias) { + return "JOIN {user} u ON u.id = {$sampletablealias}.sampleid"; + } +} diff --git a/lib/classes/analytics/target/course_dropout.php b/lib/classes/analytics/target/course_dropout.php index 9a7cfd867eb..170602d4ca5 100644 --- a/lib/classes/analytics/target/course_dropout.php +++ b/lib/classes/analytics/target/course_dropout.php @@ -63,17 +63,6 @@ class course_dropout extends \core\analytics\target\course_enrolments { ); } - /** - * Returns the predicted classes that will be ignored. - * - * Overwriten because we are also interested in knowing when the student is far from the risk of dropping out. - * - * @return array - */ - protected function ignored_predicted_classes() { - return array(); - } - /** * Discards courses that are not yet ready to be used for training or prediction. * diff --git a/lib/classes/analytics/target/course_enrolments.php b/lib/classes/analytics/target/course_enrolments.php index edc33e4bcb4..8193626cc2d 100644 --- a/lib/classes/analytics/target/course_enrolments.php +++ b/lib/classes/analytics/target/course_enrolments.php @@ -50,6 +50,17 @@ abstract class course_enrolments extends \core_analytics\local\target\binary { return '\core\analytics\analyser\student_enrolments'; } + /** + * Overwritten to show a simpler language string. + * + * @param int $modelid + * @param \context $context + * @return string + */ + public function get_insight_subject(int $modelid, \context $context) { + return get_string('studentsatriskincourse', 'moodle', $context->get_context_name(false)); + } + /** * Discards courses that are not yet ready to be used for training or prediction. * @@ -142,9 +153,11 @@ abstract class course_enrolments extends \core_analytics\local\target\binary { * * @param \core_analytics\prediction $prediction * @param bool $includedetailsaction + * @param bool $isinsightuser * @return \core_analytics\prediction_action[] */ - public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false) { + public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false, + $isinsightuser = false) { global $USER; $actions = array(); diff --git a/lib/classes/analytics/target/course_gradetopass.php b/lib/classes/analytics/target/course_gradetopass.php new file mode 100644 index 00000000000..a530cd565a9 --- /dev/null +++ b/lib/classes/analytics/target/course_gradetopass.php @@ -0,0 +1,181 @@ +. + +/** + * Getting the minimum grade to pass target. + * + * @package core + * @copyright 2019 Victor Deniz + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\analytics\target; + +defined('MOODLE_INTERNAL') || die(); + + +/** + * Getting the minimum grade to pass target. + * + * @package core + * @copyright 2019 Victor Deniz + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class course_gradetopass extends \core\analytics\target\course_enrolments { + + /** + * Courses grades to pass. + * @var mixed[] + */ + protected $coursesgradetopass = array(); + + /** + * Courses grades. + * @var mixed[] + */ + protected $coursesgrades = array(); + + /** + * Returns the grade to pass a course. + * + * Save the value in $coursesgradetopass array to prevent new accesses to the database. + * + * @param int $courseid The course id. + * @return array The courseitem id and the required grade to pass the course. + */ + protected function get_course_gradetopass($courseid) { + if (!isset($this->coursesgradetopass[$courseid])) { + // Get course grade_item. + $courseitem = \grade_item::fetch_course_item($courseid); + + $ci = array(); + $ci['courseitemid'] = $courseitem->id; + + if ($courseitem->gradetype == GRADE_TYPE_VALUE && grade_floats_different($courseitem->gradepass, 0.0)) { + $ci['gradetopass'] = $courseitem->gradepass; + } else { + $ci['gradetopass'] = null; + } + $this->coursesgradetopass[$courseid] = $ci; + } + + return $this->coursesgradetopass[$courseid]; + } + + /** + * Returns the grade of a user in a course. + * + * Saves the grades of all course users in $coursesgrades array to prevent new accesses to the database. + * + * @param int $courseitemid The course item id. + * @param int $userid the user whose grade is requested. + * @return array The courseitem id and the required grade to pass the course. + */ + protected function get_user_grade($courseitemid, $userid) { + // If the user grade for this course is not available, get all the grades for the course. + if (!isset($this->coursesgrades[$courseitemid])) { + // Ony a course is cached to avoid high memory usage. + unset($this->coursesgrades); + $gg = new \grade_grade(null, false); + $usersgrades = $gg->fetch_all(array('itemid' => $courseitemid)); + + if ($usersgrades) { + foreach ($usersgrades as $ug) { + $this->coursesgrades[$courseitemid][$ug->userid] = $ug->finalgrade; + } + } + } + + if (!isset($this->coursesgrades[$courseitemid][$userid])) { + $this->coursesgrades[$courseitemid][$userid] = null; + } + + return $this->coursesgrades[$courseitemid][$userid]; + } + + /** + * Returns the name. + * + * If there is a corresponding '_help' string this will be shown as well. + * + * @return \lang_string + */ + public static function get_name() : \lang_string { + return new \lang_string('target:coursegradetopass'); + } + + /** + * Returns descriptions for each of the values the target calculation can return. + * + * @return string[] + */ + protected static function classes_description() { + return array( + get_string('targetlabelstudentgradetopassno'), + get_string('targetlabelstudentgradetopassyes') + ); + } + + /** + * Discards courses that are not yet ready to be used for training or prediction. + * + * Only courses with "value" grade type and grade to pass set are valid. + * + * @param \core_analytics\analysable $course + * @param bool $fortraining + * @return true|string + */ + public function is_valid_analysable(\core_analytics\analysable $course, $fortraining = true) { + $isvalid = parent::is_valid_analysable($course, $fortraining); + + if (is_string($isvalid)) { + return $isvalid; + } + + $courseitem = $this->get_course_gradetopass ($course->get_id()); + if (is_null($courseitem['gradetopass'])) { + return get_string('gradetopassnotset', 'course'); + } + + return true; + } + + /** + * The user's grade in the course sets the target value. + * + * @param int $sampleid + * @param \core_analytics\analysable $course + * @param int $starttime + * @param int $endtime + * @return float 0 -> course grade to pass achieved, 1 -> course grade to pass not achieved + */ + protected function calculate_sample($sampleid, \core_analytics\analysable $course, $starttime = false, $endtime = false) { + + $userenrol = $this->retrieve('user_enrolments', $sampleid); + + // Get course grade to pass. + $courseitem = $this->get_course_gradetopass($course->get_id()); + + // Get the user grade. + $usergrade = $this->get_user_grade($courseitem['courseitemid'], $userenrol->userid); + + if ($usergrade >= $courseitem['gradetopass']) { + return 0; + } + + return 1; + } +} diff --git a/lib/classes/analytics/target/no_teaching.php b/lib/classes/analytics/target/no_teaching.php index f4b1352712c..a7769b81751 100644 --- a/lib/classes/analytics/target/no_teaching.php +++ b/lib/classes/analytics/target/no_teaching.php @@ -55,14 +55,27 @@ class no_teaching extends \core_analytics\local\target\binary { return new \lang_string('target:noteachingactivity'); } + /** + * Overwritten to show a simpler language string. + * + * @param int $modelid + * @param \context $context + * @return string + */ + public function get_insight_subject(int $modelid, \context $context) { + return get_string('noteachingupcomingcourses'); + } + /** * prediction_actions * * @param \core_analytics\prediction $prediction * @param mixed $includedetailsaction + * @param bool $isinsightuser * @return \core_analytics\prediction_action[] */ - public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false) { + public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false, + $isinsightuser = false) { global $CFG; require_once($CFG->dirroot . '/course/lib.php'); @@ -103,16 +116,6 @@ class no_teaching extends \core_analytics\local\target\binary { ); } - /** - * Returns the predicted classes that will be ignored. - * - * @return array - */ - protected function ignored_predicted_classes() { - // No need to list the course if there is teaching activity. - return array(0); - } - /** * get_analyser_class * @@ -148,9 +151,10 @@ class no_teaching extends \core_analytics\local\target\binary { $now = time(); - // No courses without start date, no finished courses nor predictions before start - 1 week. + // No courses without start date, no finished courses, no predictions before start - 1 week nor + // predictions for courses that started more than 1 week ago. if (!$course->startdate || (!empty($course->enddate) && $course->enddate < $now) || - $course->startdate - WEEKSECS > $now) { + $course->startdate - WEEKSECS > $now || $course->startdate + WEEKSECS < $now) { return false; } return true; diff --git a/lib/classes/analytics/time_splitting/upcoming_week.php b/lib/classes/analytics/time_splitting/upcoming_week.php new file mode 100644 index 00000000000..e8fbd4ea506 --- /dev/null +++ b/lib/classes/analytics/time_splitting/upcoming_week.php @@ -0,0 +1,53 @@ +. + +/** + * Time splitting method that generates weekly predictions. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\analytics\time_splitting; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Time splitting method that generates weekly predictions. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class upcoming_week extends \core_analytics\local\time_splitting\upcoming_periodic { + + /** + * The time splitting method name. + * @return \lang_string + */ + public static function get_name() : \lang_string { + return new \lang_string('timesplitting:upcomingweek'); + } + + /** + * Once per week. + * @return \DateInterval + */ + public function periodicity() { + return new \DateInterval('P1W'); + } +} diff --git a/lib/classes/plugin_manager.php b/lib/classes/plugin_manager.php index 6438fa2550e..96c4537d97a 100644 --- a/lib/classes/plugin_manager.php +++ b/lib/classes/plugin_manager.php @@ -1755,7 +1755,7 @@ class core_plugin_manager { ), 'dataformat' => array( - 'html', 'csv', 'json', 'excel', 'ods', + 'html', 'csv', 'json', 'excel', 'ods', 'pdf', ), 'datapreset' => array( diff --git a/lib/classes/progress/db_updater.php b/lib/classes/progress/db_updater.php new file mode 100644 index 00000000000..532ab6a3fa0 --- /dev/null +++ b/lib/classes/progress/db_updater.php @@ -0,0 +1,123 @@ +. + +/** + * Progress handler that updates a database table with the progress. + * + * @package core + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\progress; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Progress handler that updates a database table with the progress. + * + * @package core + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class db_updater extends base { + + /** + * The primary key of the database record to update. + * + * @var integer + */ + protected $recordid = 0; + + /** + * The database table to insert the progress updates into. + * + * @var string + */ + protected $table = ''; + + /** + * The table field to update with the progress. + * + * @var string + */ + protected $field = ''; + + /** + * The maximum frequency in seconds to update the database (default 5 seconds). + * Lower values will increase database calls. + * + * @var integer + */ + protected $interval = 5; + + + /** + * The timestamp of when the next progress update to the database will be. + * + * @var integer + */ + protected $nextupdate = 0; + + /** + * Constructs the progress reporter. + * + * @param int $recordid The primary key of the database record to update. + * @param string $table The databse table to insert the progress updates into. + * @param string $field The table field to update with the progress. + * @param int $interval The maximum frequency in seconds to update the database (default 5 seconds). + */ + public function __construct($recordid, $table, $field, $interval=5) { + $this->recordid = $recordid; + $this->table = $table; + $this->field = $field; + $this->interval = $interval; + } + + /** + * Updates the progress in the database. + * Database update frequency is set by $interval. + * + * @see \core\progress\base::update_progress() + */ + public function update_progress() { + global $DB; + $now = $this->get_time(); + $lastprogress = $this->lastprogresstime != 0 ? $this->lastprogresstime : $now; + + $progressrecord = new \stdClass(); + $progressrecord->id = $this->recordid; + $progressrecord->{$this->field} = ''; + + // Update database with progress. + if ($now > $this->nextupdate) { // Limit database updates based on time. + list ($min, $max) = $this->get_progress_proportion_range(); + + $progressrecord->{$this->field} = $min; + $DB->update_record($this->table, $progressrecord); + $this->nextupdate = $lastprogress + $this->interval; + } + + // Set progress to 1 (100%) when there are no more progress updates. + // Their is no guarantee that the final update from the get progress method + // will be 1 even for a successful process. So we explicitly set the final DB + // value to 1 when we are no longer in progress. + if (!$this->is_in_progress_section()) { + $progressrecord->{$this->field} = 1; + $DB->update_record($this->table, $progressrecord); + } + } +} diff --git a/lib/classes/task/asynchronous_backup_task.php b/lib/classes/task/asynchronous_backup_task.php new file mode 100644 index 00000000000..73554a3ae40 --- /dev/null +++ b/lib/classes/task/asynchronous_backup_task.php @@ -0,0 +1,90 @@ +. + +/** + * Adhoc task that performs asynchronous backups. + * + * @package core + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\task; + +use async_helper; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php'); +require_once($CFG->dirroot . '/backup/moodle2/backup_plan_builder.class.php'); + +/** + * Adhoc task that performs asynchronous backups. + * + * @package core + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class asynchronous_backup_task extends adhoc_task { + + /** + * Run the adhoc task and preform the backup. + */ + public function execute() { + global $DB; + $started = time(); + + $backupid = $this->get_custom_data()->backupid; + $backuprecordid = $DB->get_field('backup_controllers', 'id', array('backupid' => $backupid), MUST_EXIST); + mtrace('Processing asynchronous backup for backup: ' . $backupid); + + // Get the backup controller by backup id. + $bc = \backup_controller::load_controller($backupid); + $bc->set_progress(new \core\progress\db_updater($backuprecordid, 'backup_controllers', 'progress')); + + // Do some preflight checks on the backup. + $status = $bc->get_status(); + $execution = $bc->get_execution(); + + // Check that the backup is in the correct status and + // that is set for asynchronous execution. + if ($status == \backup::STATUS_AWAITING && $execution == \backup::EXECUTION_DELAYED) { + // Execute the backup. + $bc->execute_plan(); + + // Send message to user if enabled. + $messageenabled = (bool)get_config('backup', 'backup_async_message_users'); + if ($messageenabled && $bc->get_status() == \backup::STATUS_FINISHED_OK) { + $asynchelper = new async_helper('backup', $backupid); + $asynchelper->send_message(); + } + + } else { + // If status isn't 700, it means the process has failed. + // Retrying isn't going to fix it, so marked operation as failed. + $bc->set_status(\backup::STATUS_FINISHED_ERR); + mtrace('Bad backup controller status, is: ' . $status . ' should be 700, marking job as failed.'); + + } + + // Cleanup. + $bc->destroy(); + + $duration = time() - $started; + mtrace('Backup completed in: ' . $duration . ' seconds'); + } +} + diff --git a/lib/classes/task/asynchronous_restore_task.php b/lib/classes/task/asynchronous_restore_task.php new file mode 100644 index 00000000000..9572ba05ed3 --- /dev/null +++ b/lib/classes/task/asynchronous_restore_task.php @@ -0,0 +1,89 @@ +. + +/** + * Adhoc task that performs asynchronous restores. + * + * @package core + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\task; + +use async_helper; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php'); + +/** + * Adhoc task that performs asynchronous restores. + * + * @package core + * @copyright 2018 Matt Porritt + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class asynchronous_restore_task extends adhoc_task { + + /** + * Run the adhoc task and preform the restore. + */ + public function execute() { + global $DB; + $started = time(); + + $restoreid = $this->get_custom_data()->backupid; + $restorerecordid = $DB->get_field('backup_controllers', 'id', array('backupid' => $restoreid), MUST_EXIST); + mtrace('Processing asynchronous restore for id: ' . $restoreid); + + // Get the restore controller by backup id. + $rc = \restore_controller::load_controller($restoreid); + $rc->set_progress(new \core\progress\db_updater($restorerecordid, 'backup_controllers', 'progress')); + + // Do some preflight checks on the restore. + $status = $rc->get_status(); + $execution = $rc->get_execution(); + + // Check that the restore is in the correct status and + // that is set for asynchronous execution. + if ($status == \backup::STATUS_AWAITING && $execution == \backup::EXECUTION_DELAYED) { + // Execute the restore. + $rc->execute_plan(); + + // Send message to user if enabled. + $messageenabled = (bool)get_config('backup', 'backup_async_message_users'); + if ($messageenabled && $rc->get_status() == \backup::STATUS_FINISHED_OK) { + $asynchelper = new async_helper('restore', $restoreid); + $asynchelper->send_message(); + } + + } else { + // If status isn't 700, it means the process has failed. + // Retrying isn't going to fix it, so marked operation as failed. + $rc->set_status(\backup::STATUS_FINISHED_ERR); + mtrace('Bad backup controller status, is: ' . $status . ' should be 700, marking job as failed.'); + + } + + // Cleanup. + $rc->destroy(); + + $duration = time() - $started; + mtrace('Restore completed in: ' . $duration . ' seconds'); + } +} + diff --git a/lib/classes/task/badges_cron_task.php b/lib/classes/task/badges_cron_task.php index d5aaee0bc4d..54e71c83860 100644 --- a/lib/classes/task/badges_cron_task.php +++ b/lib/classes/task/badges_cron_task.php @@ -38,14 +38,56 @@ class badges_cron_task extends scheduled_task { } /** - * Do the job. - * Throw exceptions on errors (the job will be retried). + * Reviews criteria and awards badges + * + * First find all badges that can be earned, then reviews each badge. + * (Not sure how efficient this is timewise). */ public function execute() { - global $CFG; - // Run badges review cron. - require_once($CFG->dirroot . '/badges/cron.php'); - badge_cron(); - } + global $DB, $CFG; + if (!empty($CFG->enablebadges)) { + require_once($CFG->libdir . '/badgeslib.php'); + $total = 0; + $courseparams = array(); + if (empty($CFG->badges_allowcoursebadges)) { + $coursesql = ''; + } else { + $coursesql = ' OR EXISTS (SELECT id FROM {course} WHERE visible = :visible AND startdate < :current) '; + $courseparams = array('visible' => true, 'current' => time()); + } + + $sql = 'SELECT id + FROM {badge} + WHERE (status = :active OR status = :activelocked) + AND (type = :site ' . $coursesql . ')'; + $badgeparams = [ + 'active' => BADGE_STATUS_ACTIVE, + 'activelocked' => BADGE_STATUS_ACTIVE_LOCKED, + 'site' => BADGE_TYPE_SITE + ]; + $params = array_merge($badgeparams, $courseparams); + $badges = $DB->get_fieldset_sql($sql, $params); + + mtrace('Started reviewing available badges.'); + foreach ($badges as $bid) { + $badge = new \badge($bid); + + if ($badge->has_criteria()) { + if (debugging()) { + mtrace('Processing badge "' . $badge->name . '"...'); + } + + $issued = $badge->review_all_criteria(); + + if (debugging()) { + mtrace('...badge was issued to ' . $issued . ' users.'); + } + $total += $issued; + } + } + + mtrace('Badges were issued ' . $total . ' time(s).'); + } + } } diff --git a/lib/classes/task/badges_message_task.php b/lib/classes/task/badges_message_task.php new file mode 100644 index 00000000000..5c7f83878cb --- /dev/null +++ b/lib/classes/task/badges_message_task.php @@ -0,0 +1,67 @@ +. + +/** + * A scheduled task. + * + * @package core + * @copyright 2019 Simey Lameze + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +namespace core\task; + +/** + * Simple task to run the badges cron. + */ +class badges_message_task extends scheduled_task { + + /** + * Get a descriptive name for this task (shown to admins). + * + * @return string + */ + public function get_name() { + return get_string('taskbadgesmessagecron', 'admin'); + } + + /** + * Reviews criteria and awards badges + * + * First find all badges that can be earned, then reviews each badge. + */ + public function execute() { + global $CFG, $DB; + + if (!empty($CFG->enablebadges)) { + require_once($CFG->libdir . '/badgeslib.php'); + mtrace('Sending scheduled badge notifications.'); + + $scheduled = $DB->get_records_select('badge', 'notification > ? AND (status != ?) AND nextcron < ?', + array(BADGE_MESSAGE_ALWAYS, BADGE_STATUS_ARCHIVED, time()), + 'notification ASC', 'id, name, notification, usercreated as creator, timecreated'); + + foreach ($scheduled as $sch) { + // Send messages. + badge_assemble_notification($sch); + + // Update next cron value. + $nextcron = badges_calculate_message_schedule($sch->notification); + $DB->set_field('badge', 'nextcron', $nextcron, array('id' => $sch->id)); + } + } + } + +} diff --git a/lib/db/access.php b/lib/db/access.php index d47b7f0988b..c7e7d7326ca 100644 --- a/lib/db/access.php +++ b/lib/db/access.php @@ -2469,4 +2469,12 @@ $capabilities = array( 'user' => CAP_ALLOW, ], ], + + 'moodle/analytics:listowninsights' => array( + 'captype' => 'read', + 'contextlevel' => CONTEXT_SYSTEM, + 'archetypes' => array( + 'user' => CAP_ALLOW + ) + ), ); diff --git a/lib/db/analytics.php b/lib/db/analytics.php index 821ad953343..e5dd6197c9c 100644 --- a/lib/db/analytics.php +++ b/lib/db/analytics.php @@ -89,4 +89,11 @@ $models = [ 'timesplitting' => '\core\analytics\time_splitting\single_range', 'enabled' => true, ], + [ + 'target' => '\core_user\analytics\target\upcoming_activities_due', + 'indicators' => [ + '\core_course\analytics\indicator\activities_due', + ], + 'timesplitting' => '\core\analytics\time_splitting\upcoming_week', + ], ]; diff --git a/lib/db/install.xml b/lib/db/install.xml index 9c127a4edce..e187dea973a 100644 --- a/lib/db/install.xml +++ b/lib/db/install.xml @@ -1,5 +1,5 @@ - @@ -2898,6 +2898,7 @@ + @@ -2907,6 +2908,7 @@ + @@ -3883,14 +3885,12 @@ - - @@ -4112,4 +4112,4 @@
-
+ \ No newline at end of file diff --git a/lib/db/messages.php b/lib/db/messages.php index 9c0593c10a3..e2b63f125e8 100644 --- a/lib/db/messages.php +++ b/lib/db/messages.php @@ -117,4 +117,12 @@ $messageproviders = array ( 'email' => MESSAGE_PERMITTED + MESSAGE_DEFAULT_LOGGEDOFF, ] ], + + // Asyncronhous backup/restore notifications. + 'asyncbackupnotification' => array( + 'defaults' => array( + 'popup' => MESSAGE_PERMITTED + MESSAGE_DEFAULT_LOGGEDIN + MESSAGE_DEFAULT_LOGGEDOFF, + 'email' => MESSAGE_PERMITTED + MESSAGE_DEFAULT_LOGGEDOFF, + ) + ), ); diff --git a/lib/db/services.php b/lib/db/services.php index ea5f502b64e..ec4dd782222 100644 --- a/lib/db/services.php +++ b/lib/db/services.php @@ -74,6 +74,33 @@ $functions = array( 'ajax' => true, 'loginrequired' => false, ), + 'core_backup_get_async_backup_progress' => array( + 'classname' => 'core_backup_external', + 'classpath' => 'backup/externallib.php', + 'methodname' => 'get_async_backup_progress', + 'description' => 'Get the progress of an Asyncronhous backup.', + 'type' => 'read', + 'ajax' => true, + 'loginrequired' => true, + ), + 'core_backup_get_async_backup_links_backup' => array( + 'classname' => 'core_backup_external', + 'classpath' => 'backup/externallib.php', + 'methodname' => 'get_async_backup_links_backup', + 'description' => 'Gets the data to use when updating the status table row in the UI for when an async backup completes.', + 'type' => 'read', + 'ajax' => true, + 'loginrequired' => true, + ), + 'core_backup_get_async_backup_links_restore' => array( + 'classname' => 'core_backup_external', + 'classpath' => 'backup/externallib.php', + 'methodname' => 'get_async_backup_links_restore', + 'description' => 'Gets the data to use when updating the status table row in the UI for when an async restore completes.', + 'type' => 'read', + 'ajax' => true, + 'loginrequired' => true, + ), 'core_badges_get_user_badges' => array( 'classname' => 'core_badges_external', 'methodname' => 'get_user_badges', diff --git a/lib/db/tasks.php b/lib/db/tasks.php index 85381d442e0..14c1b600713 100644 --- a/lib/db/tasks.php +++ b/lib/db/tasks.php @@ -275,6 +275,15 @@ $tasks = array( 'dayofweek' => '*', 'month' => '*' ), + array( + 'classname' => 'core\task\badges_message_task', + 'blocking' => 0, + 'minute' => '*/5', + 'hour' => '*', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*' + ), array( 'classname' => 'core\task\file_temp_cleanup_task', 'blocking' => 0, diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php index 203e02d03a2..36ec2a9c1b0 100644 --- a/lib/db/upgrade.php +++ b/lib/db/upgrade.php @@ -2640,7 +2640,6 @@ function xmldb_main_upgrade($oldversion) { $key = new xmldb_key('useridgroupid', XMLDB_KEY_UNIQUE, array('userid', 'groupid')); // Launch add key useridgroupid. $dbman->add_key($table, $key); - // Main savepoint reached. upgrade_main_savepoint(true, 2019011801.03); } @@ -2934,5 +2933,45 @@ function xmldb_main_upgrade($oldversion) { upgrade_main_savepoint(true, 2019040200.01); } + if ($oldversion < 2019040600.02) { + + // Define key fileid (foreign) to be dropped form analytics_train_samples. + $table = new xmldb_table('analytics_train_samples'); + $key = new xmldb_key('fileid', XMLDB_KEY_FOREIGN, ['fileid'], 'files', ['id']); + + // Launch drop key fileid. + $dbman->drop_key($table, $key); + + // Define field fileid to be dropped from analytics_train_samples. + $table = new xmldb_table('analytics_train_samples'); + $field = new xmldb_field('fileid'); + + // Conditionally launch drop field fileid. + if ($dbman->field_exists($table, $field)) { + $dbman->drop_field($table, $field); + } + + // Main savepoint reached. + upgrade_main_savepoint(true, 2019040600.02); + } + + if ($oldversion < 2019040600.04) { + // Define field and index to be added to backup_controllers. + $table = new xmldb_table('backup_controllers'); + $field = new xmldb_field('progress', XMLDB_TYPE_NUMBER, '15, 14', null, XMLDB_NOTNULL, null, '0', 'timemodified'); + $index = new xmldb_index('useritem_ix', XMLDB_INDEX_NOTUNIQUE, ['userid', 'itemid']); + // Conditionally launch add field progress. + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + // Conditionally launch add index useritem_ix. + if (!$dbman->index_exists($table, $index)) { + $dbman->add_index($table, $index); + } + + // Main savepoint reached. + upgrade_main_savepoint(true, 2019040600.04); + } + return true; } diff --git a/lib/templates/async_backup_progress.mustache b/lib/templates/async_backup_progress.mustache new file mode 100644 index 00000000000..ba96e5311cc --- /dev/null +++ b/lib/templates/async_backup_progress.mustache @@ -0,0 +1,45 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core/async_backup_progress. + + Moodle Asynchronous backup status template. + + The purpose of this template is to render status + updates during an asynchronous backup or restore + process.. + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Context variables required for this template: + * + + Example context (json): + { + "backupid": "f04abf8cba0319e486a3dfa7e9cb4476", + "width": "500" + } +}} +
+
+ {{# str }} asyncprocesspending, backup {{/ str }} +
+
diff --git a/lib/templates/async_backup_progress_row.mustache b/lib/templates/async_backup_progress_row.mustache new file mode 100644 index 00000000000..012415932c9 --- /dev/null +++ b/lib/templates/async_backup_progress_row.mustache @@ -0,0 +1,52 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core/async_backup_progress_row. + + Moodle Asynchronous backup status table row template. + + The purpose of this template is to render status + table row updates during an asynchronous backup process. + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Context variables required for this template: + * + + Example context (json): + { + "filename": "backup-moodle2-course-49390-test_course-20190326-1546-nu-nf.mbz", + "time": "Tuesday, 26 March 2019, 3:47 PM", + "size": "7.2KB", + "fileurl": "https://moodle.local/pluginfile.php/7945628/backup/course/backup-moodle2-course-49390-test_course-20190326-1546-nu-nf.mbz?forcedownload=1", + "restoreurl": "https://moodle.local/backup/restorefile.php?action=choosebackupfile&filename=backup-moodle2-course-49390-test_course-20190326-1546-nu-nf.mbz&filepath=%2F&component=backup&filearea=course&filecontextid=7945628&contextid=7945628&itemid=0" + } +}} +{{filename}} +{{time}} +{{size}} +{{# str }} download, core {{/ str }} +{{# str }} restore, core {{/ str }} + + + + + diff --git a/lib/templates/async_backup_status.mustache b/lib/templates/async_backup_status.mustache new file mode 100644 index 00000000000..49cb0425e5e --- /dev/null +++ b/lib/templates/async_backup_status.mustache @@ -0,0 +1,57 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core/async_backup_status. + + Moodle Asynchronous backup status template. + + The purpose of this template is to render status + updates during an asynchronous backup or restore + process.. + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Context variables required for this template: + * + + Example context (json): + { + "backupid": "f04abf8cba0319e486a3dfa7e9cb4476", + "contextid": "4", + "courseurl": "/course/view.php?id=6", + "restoreurl": "/backup/restorefile.php?contextid=287", + "headingident": "backup", + "width": "500" + } +}} +
+ {{#headingident}}

{{# str }} asyncbackuppending, backup {{/ str }}

{{/headingident}} + {{^headingident}}

{{# str }} asyncrestorepending, backup {{/ str }}

{{/headingident}} + {{> core/async_backup_progress }} +

{{# str }} asyncnowait, backup {{/ str }}
{{# str }} asynccheckprogress, backup, {{restoreurl}} {{/ str }}

+ {{# str }} asyncreturn, backup {{/ str }} +
+ +{{#js}} +require(['core_backup/async_backup'], function(Async) { + Async.asyncBackupStatus("{{backupid}}", "{{contextid}}", "{{restoreurl}}", "{{headingident}}"); +}); +{{/js}} diff --git a/lib/templates/async_restore_progress_row.mustache b/lib/templates/async_restore_progress_row.mustache new file mode 100644 index 00000000000..c67e528b3fa --- /dev/null +++ b/lib/templates/async_restore_progress_row.mustache @@ -0,0 +1,51 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core/async_restore_progress_row. + + Moodle Asynchronous restore status table row template. + + The purpose of this template is to render status + table row updates during an asynchronous restore process. + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Context variables required for this template: + * + + Example context (json): + { + "resourcename" : "Mathematics 101", + "time": "Tuesday, 26 March 2019, 3:47 PM", + "restoreurl": "https://moodle.local/backup/restorefile.php?action=choosebackupfile&filename=backup-moodle2-course-49390-test_course-20190326-1546-nu-nf.mbz&filepath=%2F&component=backup&filearea=course&filecontextid=7945628&contextid=7945628&itemid=0" + } +}} + + {{resourcename}} + +{{time}} + + + + + + + diff --git a/lib/tests/analysers_test.php b/lib/tests/analysers_test.php index 4c28a0685e2..f52ab0f9636 100644 --- a/lib/tests/analysers_test.php +++ b/lib/tests/analysers_test.php @@ -25,7 +25,9 @@ defined('MOODLE_INTERNAL') || die(); +require_once(__DIR__ . '/../../analytics/tests/fixtures/test_target_course_level_shortname.php'); require_once(__DIR__ . '/../../analytics/tests/fixtures/test_target_shortname.php'); +require_once(__DIR__ . '/fixtures/deprecated_analyser.php'); require_once(__DIR__ . '/../../lib/enrollib.php'); /** @@ -119,7 +121,7 @@ class core_analytics_analysers_testcase extends advanced_testcase { } /** - * test_site_courses_analyser + * test_student_enrolments_analyser * * @return void */ @@ -175,4 +177,96 @@ class core_analytics_analysers_testcase extends advanced_testcase { $this->assertEquals($prevsampledata['course']->shortname, $samplesdata[$sampleid]['course']->shortname); $this->assertEquals($prevsampledata['user']->firstname, $samplesdata[$sampleid]['user']->firstname); } + + /** + * test_deprecated_analyser + * + * @return void + */ + public function test_deprecated_analyser() { + + $target = new test_target_shortname(); + $analyser = new deprecated_analyser(1, $target, [], [], []); + + $analysables = $analyser->get_analysables_iterator(); + $this->assertDebuggingCalled(); + } + + /** + * test_get_analysables_iterator description + * + * @return null + */ + public function test_get_analysables_iterator() { + global $DB; + + $this->resetAfterTest(true); + + $courses = array(); + for ($i = 0; $i < 2; $i++) { + $course = $this->getDataGenerator()->create_course(); + $analysable = new \core_analytics\course($course); + $courses[$analysable->get_id()] = $course; + } + + // Check that the analysis performs as expected. + $modelid = 1; + $includetarget = false; + + $target = new test_target_course_level_shortname(); + $analyser = new \core\analytics\analyser\courses($modelid, $target, [], [], []); + + $result = new \core_analytics\local\analysis\result_array($modelid, $includetarget, []); + $analysis = new \core_analytics\analysis($analyser, $includetarget, $result); + $analysis->run(); + $params = array('modelid' => $modelid, 'action' => 'prediction'); + $this->assertEquals(2, $DB->count_records('analytics_used_analysables', $params)); + + // Check that the previous records do not conflict with the includetarget == false ones. + $includetarget = true; + + $target = new test_target_course_level_shortname(); + $analyser = new \core\analytics\analyser\courses($modelid, $target, [], [], []); + + $result = new \core_analytics\local\analysis\result_array($modelid, $includetarget, []); + $analysis = new \core_analytics\analysis($analyser, $includetarget, $result); + $analysis->run(); + $params = array('modelid' => $modelid, 'action' => 'prediction'); + $this->assertEquals(2, $DB->count_records('analytics_used_analysables', $params)); + $params = array('modelid' => $modelid, 'action' => 'training'); + $this->assertEquals(2, $DB->count_records('analytics_used_analysables', $params)); + $params = array('modelid' => $modelid); + $this->assertEquals(4, $DB->count_records('analytics_used_analysables', $params)); + + // Check that other models' records do not conflict with previous records. + $prevmodelid = 1; + $modelid = 2; + $includetarget = false; + + $target = new test_target_course_level_shortname(); + $analyser = new \core\analytics\analyser\courses($modelid, $target, [], [], []); + + $result = new \core_analytics\local\analysis\result_array($modelid, $includetarget, []); + $analysis = new \core_analytics\analysis($analyser, $includetarget, $result); + $analysis->run(); + $params = array('modelid' => $prevmodelid); + $this->assertEquals(4, $DB->count_records('analytics_used_analysables', $params)); + $params = array('modelid' => $modelid, 'action' => 'prediction'); + $this->assertEquals(2, $DB->count_records('analytics_used_analysables', $params)); + $this->assertEquals(6, $DB->count_records('analytics_used_analysables')); + + $includetarget = true; + + $target = new test_target_course_level_shortname(); + $analyser = new \core\analytics\analyser\courses($modelid, $target, [], [], []); + + $result = new \core_analytics\local\analysis\result_array($modelid, $includetarget, []); + $analysis = new \core_analytics\analysis($analyser, $includetarget, $result); + $analysis->run(); + $params = array('modelid' => $prevmodelid); + $this->assertEquals(4, $DB->count_records('analytics_used_analysables', $params)); + $params = array('modelid' => $modelid, 'action' => 'training'); + $this->assertEquals(2, $DB->count_records('analytics_used_analysables', $params)); + $this->assertEquals(8, $DB->count_records('analytics_used_analysables')); + } } diff --git a/lib/tests/fixtures/deprecated_analyser.php b/lib/tests/fixtures/deprecated_analyser.php new file mode 100644 index 00000000000..a4a48efeab8 --- /dev/null +++ b/lib/tests/fixtures/deprecated_analyser.php @@ -0,0 +1,158 @@ +. + +/** + * Deprecated analyser for testing purposes. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Deprecated analyser for testing purposes. + * + * @package core_analytics + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class deprecated_analyser extends \core_analytics\local\analyser\base { + + /** + * Implementation of a deprecated method. + * + * It should be called by get_analysables_iterator, which triggers a debugging message. + * @return \core_analytics\analysable[] + */ + public function get_analysables() { + $analysable = new \core_analytics\site(); + return [SYSCONTEXTID => $analysable]; + } + + /** + * Samples origin is course table. + * + * @return string + */ + public function get_samples_origin() { + return 'user'; + } + + /** + * Returns the sample analysable + * + * @param int $sampleid + * @return \core_analytics\analysable + */ + public function get_sample_analysable($sampleid) { + return new \core_analytics\site(); + } + + /** + * Data this analyer samples provide. + * + * @return string[] + */ + protected function provided_sample_data() { + return array('user'); + } + + /** + * Returns the sample context. + * + * @param int $sampleid + * @return \context + */ + public function sample_access_context($sampleid) { + return \context_system::instance(); + } + + /** + * Returns all site courses. + * + * @param \core_analytics\analysable $site + * @return array + */ + public function get_all_samples(\core_analytics\analysable $site) { + global $DB; + + $users = $DB->get_records('user'); + $userids = array_keys($users); + $sampleids = array_combine($userids, $userids); + + $users = array_map(function($user) { + return array('user' => $user); + }, $users); + + return array($sampleids, $users); + } + + /** + * Return all complete samples data from sample ids. + * + * @param int[] $sampleids + * @return array + */ + public function get_samples($sampleids) { + global $DB; + + list($userssql, $params) = $DB->get_in_or_equal($sampleids, SQL_PARAMS_NAMED); + $users = $DB->get_records_select('user', "id {$userssql}", $params); + $userids = array_keys($users); + $sampleids = array_combine($userids, $userids); + + $users = array_map(function($user) { + return array('user' => $user); + }, $users); + + return array($sampleids, $users); + } + + /** + * Returns the description of a sample. + * + * @param int $sampleid + * @param int $contextid + * @param array $sampledata + * @return array array(string, \renderable) + */ + public function sample_description($sampleid, $contextid, $sampledata) { + $description = fullname($sampledata['user']); + $userimage = new \pix_icon('i/user', get_string('user')); + return array($description, $userimage); + } + + /** + * We need to delete associated data if a user requests his data to be deleted. + * + * @return bool + */ + public function processes_user_data() { + return true; + } + + /** + * Join the samples origin table with the user id table. + * + * @param string $sampletablealias + * @return string + */ + public function join_sample_user($sampletablealias) { + return "JOIN {user} u ON u.id = {$sampletablealias}.sampleid"; + } +} \ No newline at end of file diff --git a/lib/tests/targets_test.php b/lib/tests/targets_test.php index e40e486aee8..025073126ba 100644 --- a/lib/tests/targets_test.php +++ b/lib/tests/targets_test.php @@ -28,6 +28,10 @@ global $CFG; require_once($CFG->dirroot . '/completion/criteria/completion_criteria.php'); require_once($CFG->dirroot . '/completion/criteria/completion_criteria_activity.php'); +require_once($CFG->dirroot . '/lib/grade/grade_item.php'); +require_once($CFG->dirroot . '/lib/grade/grade_grade.php'); +require_once($CFG->dirroot . '/lib/grade/grade_category.php'); +require_once($CFG->dirroot . '/lib/grade/constants.php'); /** * Unit tests for core targets. @@ -341,4 +345,98 @@ class core_analytics_targets_testcase extends advanced_testcase { // Method calculate_sample() returns 0 when the user has achieved all the competencies assigned to the course. $this->assertEquals(0, $method->invoke($target, $sampleid, $analysable)); } + + /** + * Test the specific conditions of a valid analysable for the course_gradetopass target. + */ + public function test_core_target_course_gradetopass_analysable() { + global $DB; + + $this->resetAfterTest(true); + $now = time(); + + $dg = $this->getDataGenerator(); + + // Course without grade to pass set. + $course1 = $dg->create_course(array('startdate' => $now - WEEKSECS, 'enddate' => $now - DAYSECS)); + $student1 = $dg->create_user(); + $studentrole = $DB->get_record('role', array('shortname' => 'student')); + $dg->enrol_user($student1->id, $course1->id, $studentrole->id); + + $analysable = new \core_analytics\course($course1); + $target = new \core\analytics\target\course_gradetopass(); + $this->assertEquals(get_string('gradetopassnotset', 'course'), $target->is_valid_analysable($analysable)); + + // Set grade to pass. + $courseitem = grade_item::fetch_course_item($course1->id); + $courseitem->gradepass = 50; + $DB->update_record('grade_items', $courseitem); + // Since the grade to pass value is cached in the target, a new one it is instanciated. + $target = new \core\analytics\target\course_gradetopass(); + $this->assertTrue($target->is_valid_analysable($analysable)); + + } + + /** + * Test the target value calculation of the course_gradetopass target. + */ + public function test_core_target_course_gradetopass_calculate() { + global $DB; + + $this->resetAfterTest(true); + + $dg = $this->getDataGenerator(); + $course1 = $dg->create_course(); + // Set grade to pass. + $student1 = $dg->create_user(); + $student2 = $dg->create_user(); + $student3 = $dg->create_user(); + $studentrole = $DB->get_record('role', array('shortname' => 'student')); + $dg->enrol_user($student1->id, $course1->id, $studentrole->id); + $dg->enrol_user($student2->id, $course1->id, $studentrole->id); + $dg->enrol_user($student3->id, $course1->id, $studentrole->id); + + $courseitem = grade_item::fetch_course_item($course1->id); + // Student1 fails. + $courseitem->update_final_grade($student1->id, 30); + // Student2 pass. + $courseitem->update_final_grade($student2->id, 60); + // Student 3 has no grade. + + $courseitem->gradepass = 50; + $DB->update_record('grade_items', $courseitem); + + $target = new \core\analytics\target\course_gradetopass(); + $analyser = new \core\analytics\analyser\student_enrolments(1, $target, [], [], []); + $analysable = new \core_analytics\course($course1); + + $class = new ReflectionClass('\core\analytics\analyser\student_enrolments'); + $method = $class->getMethod('get_all_samples'); + $method->setAccessible(true); + + list($sampleids, $samplesdata) = $method->invoke($analyser, $analysable); + $target->add_sample_data($samplesdata); + + // Users in array $sampleids are sorted by user id, so student1 is the first sample. + $sampleid = reset($sampleids); + + $class = new ReflectionClass('\core\analytics\target\course_gradetopass'); + $method = $class->getMethod('calculate_sample'); + $method->setAccessible(true); + + // Method calculate_sample() returns 1 when the user has not successfully graded to pass the course. + $this->assertEquals(1, $method->invoke($target, $sampleid, $analysable)); + + // Student2. + $sampleid = next($sampleids); + + // Method calculate_sample() returns 0 when the user has successfully graded to pass the course. + $this->assertEquals(0, $method->invoke($target, $sampleid, $analysable)); + + // Student3. + $sampleid = next($sampleids); + + // Method calculate_sample() returns 1 when the user has not been graded. + $this->assertEquals(1, $method->invoke($target, $sampleid, $analysable)); + } } diff --git a/lib/tests/time_splittings_test.php b/lib/tests/time_splittings_test.php index fe99b830b0e..86a9efd3b98 100644 --- a/lib/tests/time_splittings_test.php +++ b/lib/tests/time_splittings_test.php @@ -25,7 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -require_once(__DIR__ . '/../../analytics/tests/fixtures/test_target_shortname.php'); +require_once(__DIR__ . '/../../analytics/tests/fixtures/test_timesplitting_seconds.php'); +require_once(__DIR__ . '/../../analytics/tests/fixtures/test_timesplitting_weekly.php'); require_once(__DIR__ . '/../../lib/enrollib.php'); /** @@ -52,8 +53,8 @@ class core_analytics_time_splittings_testcase extends advanced_testcase { 'startdate' => mktime(8, 15, 32, 10, 24, 2015), 'enddate' => mktime(12, 12, 31, 10, 24, 2016), ); - $course = $this->getDataGenerator()->create_course($params); - $this->analysable = new \core_analytics\course($course); + $this->course = $this->getDataGenerator()->create_course($params); + $this->analysable = new \core_analytics\course($this->course); } /** @@ -70,7 +71,8 @@ class core_analytics_time_splittings_testcase extends advanced_testcase { '\core\analytics\time_splitting\no_splitting', '\core\analytics\time_splitting\quarters', '\core\analytics\time_splitting\quarters_accum', - '\core\analytics\time_splitting\single_range' + '\core\analytics\time_splitting\single_range', + '\core\analytics\time_splitting\upcoming_week', ); // Check that defined ranges are valid (tested through validate_ranges). @@ -95,6 +97,8 @@ class core_analytics_time_splittings_testcase extends advanced_testcase { $quarters->set_analysable($this->analysable); $ranges = $quarters->get_all_ranges(); $this->assertCount(4, $ranges); + $this->assertCount(4, $quarters->get_training_ranges()); + $this->assertCount(4, $quarters->get_distinct_ranges()); $this->assertGreaterThan($ranges[0]['start'], $ranges[1]['start']); $this->assertGreaterThan($ranges[0]['end'], $ranges[1]['start']); @@ -121,6 +125,8 @@ class core_analytics_time_splittings_testcase extends advanced_testcase { $accum->set_analysable($this->analysable); $ranges = $accum->get_all_ranges(); $this->assertCount(4, $ranges); + $this->assertCount(4, $accum->get_training_ranges()); + $this->assertCount(4, $accum->get_distinct_ranges()); $this->assertEquals($ranges[0]['start'], $ranges[1]['start']); $this->assertEquals($ranges[1]['start'], $ranges[2]['start']); @@ -180,4 +186,115 @@ class core_analytics_time_splittings_testcase extends advanced_testcase { $range['time'] = 0; $this->assertTrue($singlerange->ready_to_predict($range)); } + + /** + * test_periodic + * + * @return void + */ + public function test_periodic() { + + // Using a finished course. + + $weekly = new test_timesplitting_weekly(); + $weekly->set_analysable($this->analysable); + $this->assertCount(1, $weekly->get_distinct_ranges()); + + $ranges = $weekly->get_all_ranges(); + $this->assertEquals(52, count($ranges)); + $this->assertEquals($this->course->startdate, $ranges[0]['start']); + $this->assertNotEquals($this->course->startdate, $ranges[0]['time']); + + // The analysable is finished so all ranges are available for training. + $this->assertCount(count($ranges), $weekly->get_training_ranges()); + + $ranges = $weekly->get_most_recent_prediction_range(); + $range = reset($ranges); + $this->assertEquals(51, key($ranges)); + + $upcomingweek = new \core\analytics\time_splitting\upcoming_week(); + $upcomingweek->set_analysable($this->analysable); + $this->assertCount(1, $upcomingweek->get_distinct_ranges()); + + $ranges = $upcomingweek->get_all_ranges(); + $this->assertEquals(53, count($ranges)); + $this->assertEquals($this->course->startdate, $ranges[0]['start']); + $this->assertEquals($this->course->startdate, $ranges[0]['time']); + + $this->assertCount(count($ranges), $upcomingweek->get_training_ranges()); + + $ranges = $upcomingweek->get_most_recent_prediction_range(); + $range = reset($ranges); + $this->assertEquals(52, key($ranges)); + + // We now use an ongoing course. + + $onemonthago = new DateTime('-30 days'); + $params = array( + 'startdate' => $onemonthago->getTimestamp(), + ); + $ongoingcourse = $this->getDataGenerator()->create_course($params); + $ongoinganalysable = new \core_analytics\course($ongoingcourse); + + $weekly = new test_timesplitting_weekly(); + $weekly->set_analysable($ongoinganalysable); + $this->assertCount(1, $weekly->get_distinct_ranges()); + + $ranges = $weekly->get_all_ranges(); + $this->assertEquals(4, count($ranges)); + $this->assertCount(4, $weekly->get_training_ranges()); + + $ranges = $weekly->get_most_recent_prediction_range(); + $range = reset($ranges); + $this->assertEquals(3, key($ranges)); + $this->assertLessThan(time(), $range['time']); + $this->assertLessThan(time(), $range['start']); + $this->assertLessThan(time(), $range['end']); + + $upcomingweek = new \core\analytics\time_splitting\upcoming_week(); + $upcomingweek->set_analysable($ongoinganalysable); + $this->assertCount(1, $upcomingweek->get_distinct_ranges()); + + $ranges = $upcomingweek->get_all_ranges(); + $this->assertEquals(5, count($ranges)); + $this->assertCount(4, $upcomingweek->get_training_ranges()); + + $ranges = $upcomingweek->get_most_recent_prediction_range(); + $range = reset($ranges); + $this->assertEquals(4, key($ranges)); + $this->assertLessThan(time(), $range['time']); + $this->assertLessThan(time(), $range['start']); + $this->assertGreaterThan(time(), $range['end']); + + // We now check how new ranges get added as time passes. + + $fewsecsago = new DateTime('-5 seconds'); + $params = array( + 'startdate' => $fewsecsago->getTimestamp(), + 'enddate' => (new DateTimeImmutable('+1 year'))->getTimestamp(), + ); + $course = $this->getDataGenerator()->create_course($params); + $analysable = new \core_analytics\course($course); + + $seconds = new test_timesplitting_seconds(); + $seconds->set_analysable($analysable); + + // Store the ranges we just obtained. + $nranges = count($seconds->get_all_ranges()); + $ntrainingranges = count($seconds->get_training_ranges()); + $mostrecentrange = $seconds->get_most_recent_prediction_range(); + $mostrecentrange = reset($mostrecentrange); + + // We wait for the next range to be added. + usleep(1000000); + + $seconds->set_analysable($analysable); + $nnewranges = $seconds->get_all_ranges(); + $nnewtrainingranges = $seconds->get_training_ranges(); + $newmostrecentrange = $seconds->get_most_recent_prediction_range(); + $newmostrecentrange = reset($newmostrecentrange); + $this->assertGreaterThan($nranges, $nnewranges); + $this->assertGreaterThan($ntrainingranges, $nnewtrainingranges); + $this->assertGreaterThan($mostrecentrange['time'], $newmostrecentrange['time']); + } } diff --git a/message/tests/behat/behat_message.php b/message/tests/behat/behat_message.php index 40c39f5f54d..90f325dfb60 100644 --- a/message/tests/behat/behat_message.php +++ b/message/tests/behat/behat_message.php @@ -48,6 +48,19 @@ class behat_message extends behat_base { $this->execute("behat_general::i_click_on", [get_string('togglemessagemenu', 'core_message'), 'link']); } + /** + * Open the messaging conversation list. + * + * @Given /^I open the "(?P(?:[^"]|\\")*)" conversations list/ + * @param string $tab + */ + public function i_open_the_conversations_list(string $tab) { + $this->execute('behat_general::i_click_on', [ + $this->escape($tab), + 'group_message_tab' + ]); + } + /** * Open the messaging UI. * @@ -153,7 +166,7 @@ class behat_message extends behat_base { array("//textarea[@data-region='send-message-txt']", $this->escape($messagecontent)) ); - $this->execute("behat_forms::press_button", get_string('send', 'message')); + $this->execute("behat_forms::press_button", get_string('sendmessage', 'message')); } /** @@ -204,4 +217,19 @@ class behat_message extends behat_base { ) ); } + + /** + * Select a user in a specific messaging UI conversations list. + * + * @Given /^I select "(?P(?:[^"]|\\")*)" conversation in the "(?P(?:[^"]|\\")*)" conversations list$/ + * @param string $convname + * @param string $listname + */ + public function i_select_conversation_in_the_conversations_list(string $convname, string $listname) { + $xpath = '//*[@data-region="message-drawer"]//div[@data-region="view-overview-'. + $this->escape($listname). + '"]//*[@data-conversation-id]//img[contains(@alt,"'. + $this->escape($convname).'")]'; + $this->execute('behat_general::i_click_on', array($xpath, 'xpath_element')); + } } diff --git a/message/tests/behat/message_drawer_manage_contacts.feature b/message/tests/behat/message_drawer_manage_contacts.feature index 07190be7228..b129e24dbf3 100644 --- a/message/tests/behat/message_drawer_manage_contacts.feature +++ b/message/tests/behat/message_drawer_manage_contacts.feature @@ -31,6 +31,7 @@ Feature: Manage contacts And I open contact menu And I click on "Add to contacts" "link" And I click on "Add" "button" + And I should see "Contact request sent" And I log out And I log in as "student4" Then I should see "2" in the "//*[@data-region='count-container']" "xpath_element" @@ -41,6 +42,7 @@ Feature: Manage contacts And I click on "Student 1 Would like to contact you" "link" Then I should see "Accept and add to contacts" And I click on "Accept and add to contacts" "link_or_button" + And I should not see "Accept and add to contacts" And I log out And I log in as "student1" And I open messaging @@ -54,6 +56,7 @@ Feature: Manage contacts And I open contact menu And I click on "Add to contacts" "link" And I click on "Add" "button" + And I should see "Contact request sent" And I log out And I log in as "student3" Then I should see "1" in the "//*[@data-region='count-container']" "xpath_element" @@ -64,6 +67,7 @@ Feature: Manage contacts And I click on "Student 1 Would like to contact you" "link" Then I should see "Accept and add to contacts" And I click on "Decline" "link_or_button" + And I should not see "Accept and add to contacts" And I open contact menu Then I should see "Add to contacts" in the "//div[@data-region='header-container']" "xpath_element" diff --git a/message/tests/behat/message_send_messages.feature b/message/tests/behat/message_send_messages.feature new file mode 100644 index 00000000000..4a566b38981 --- /dev/null +++ b/message/tests/behat/message_send_messages.feature @@ -0,0 +1,84 @@ +@core @core_message @javascript +Feature: Message send messages + In order to communicate with fellow users + As a user + I need to be able to send a message + + Background: + Given the following "courses" exist: + | fullname | shortname | category | groupmode | + | Course 1 | C1 | 0 | 1 | + And the following "users" exist: + | username | firstname | lastname | email | + | student1 | Student | 1 | student1@example.com | + | student2 | Student | 2 | student2@example.com | + And the following "course enrolments" exist: + | user | course | role | + | student1 | C1 | student | + | student2 | C1 | student | + And the following "groups" exist: + | name | course | idnumber | enablemessaging | + | Group 1 | C1 | G1 | 1 | + And the following "group members" exist: + | user | group | + | student1 | G1 | + | student2 | G1 | + And the following config values are set as admin: + | messaging | 1 | + + Scenario: Send a message to a group conversation + Given I log in as "student1" + And I open messaging + And "Group 1" "group_message" should exist + And I select "Group 1" conversation in messaging + When I send "Hi!" message in the message area + Then I should see "Hi!" in the "Group 1" "group_message_conversation" + And I should see "##today##j F##" in the "Group 1" "group_message_conversation" + And I log out + And I log in as "student2" + And I open messaging + And "Group 1" "group_message" should exist + And I select "Group 1" conversation in messaging + And I should see "Hi!" in the "Group 1" "group_message_conversation" + + Scenario: Send a message to a starred conversation + Given I log in as "student1" + When I open messaging + Then "Group 1" "group_message" should exist + And I select "Group 1" conversation in the "group-messages" conversations list + And I open contact menu + And I click on "Star" "link" in the "//div[@data-region='header-container']" "xpath_element" + And I go back in "view-conversation" message drawer + And I open the "Starred" conversations list + And I should see "Group 1" + And I select "Group 1" conversation in the "favourites" conversations list + And I send "Hi!" message in the message area + And I should see "Hi!" in the "Group 1" "group_message_conversation" + And I should see "##today##j F##" in the "Group 1" "group_message_conversation" + And I go back in "view-conversation" message drawer + And I open the "Group" conversations list + And I should not see "Group 1" in the "Group" "group_message_tab" + + Scenario: Send a message to a private conversation via contact tab + Given the following "message contacts" exist: + | user | contact | + | student1 | student2 | + And I log in as "student1" + And I open messaging + And I click on "Contacts" "link" + And I click on "Student 2" "link" in the "//*[@data-section='contacts']" "xpath_element" + When I send "Hi!" message in the message area + Then I should see "Hi!" in the "Student 2" "group_message_conversation" + And I should see "##today##j F##" in the "Student 2" "group_message_conversation" + + Scenario: Try to send a message to a private conversation is not contact but you are allowed to send a message + Given I log in as "student1" + And I open messaging + When I send "Hi!" message to "Student 2" user + Then I should see "Hi!" in the "Student 2" "group_message_conversation" + And I should see "##today##j F##" in the "Student 2" "group_message_conversation" + And I log out + And I log in as "student2" + And I open messaging + And I select "Student 1" conversation in messaging + And I should see "Hi!" in the "Student 1" "group_message_conversation" \ No newline at end of file diff --git a/report/insights/classes/output/insight.php b/report/insights/classes/output/insight.php index d0ffa6e2114..c424e940faa 100644 --- a/report/insights/classes/output/insight.php +++ b/report/insights/classes/output/insight.php @@ -74,8 +74,20 @@ class insight implements \renderable, \templatable { // Get the prediction data. $predictiondata = $this->prediction->get_prediction_data(); + $target = $this->model->get_target(); + $data = new \stdClass(); - $data->insightname = format_string($this->model->get_target()->get_name()); + $data->insightname = format_string($target->get_name()); + + $data->showpredictionheading = true; + if (!$target->is_linear()) { + $nclasses = count($target::get_classes()); + $nignoredclasses = count($target->ignored_predicted_classes()); + if ($nclasses - $nignoredclasses <= 1) { + // Hide the prediction heading if there is only 1 class displayed. Otherwise it is redundant with the insight name. + $data->showpredictionheading = false; + } + } // Get the details. $data->timecreated = userdate($predictiondata->timecreated); @@ -99,11 +111,11 @@ class insight implements \renderable, \templatable { // Prediction info. $predictedvalue = $predictiondata->prediction; $predictionid = $predictiondata->id; - $data->predictiondisplayvalue = $this->model->get_target()->get_display_value($predictedvalue); - list($data->style, $data->outcomeicon) = self::get_calculation_display($this->model->get_target(), + $data->predictiondisplayvalue = $target->get_display_value($predictedvalue); + list($data->style, $data->outcomeicon) = self::get_calculation_display($target, floatval($predictedvalue), $output); - $actions = $this->model->get_target()->prediction_actions($this->prediction, $this->includedetailsaction); + $actions = $target->prediction_actions($this->prediction, $this->includedetailsaction); if ($actions) { $actionsmenu = new \action_menu(); $actionsmenu->set_menu_trigger(get_string('actions')); diff --git a/report/insights/classes/output/insights_list.php b/report/insights/classes/output/insights_list.php index 1ef320e9963..821c867d85c 100644 --- a/report/insights/classes/output/insights_list.php +++ b/report/insights/classes/output/insights_list.php @@ -87,8 +87,20 @@ class insights_list implements \renderable, \templatable { public function export_for_template(\renderer_base $output) { global $PAGE; + $target = $this->model->get_target(); + $data = new \stdClass(); - $data->insightname = format_string($this->model->get_target()->get_name()); + $data->insightname = format_string($target->get_name()); + + $data->showpredictionheading = true; + if (!$target->is_linear()) { + $nclasses = count($target::get_classes()); + $nignoredclasses = count($target->ignored_predicted_classes()); + if ($nclasses - $nignoredclasses <= 1) { + // Hide the prediction heading if there is only 1 class displayed. Otherwise it is redundant with the insight name. + $data->showpredictionheading = false; + } + } $total = 0; @@ -112,9 +124,9 @@ class insights_list implements \renderable, \templatable { // Only need to fill this data once. if (!isset($predictionvalues[$predictedvalue])) { $preddata = array(); - $preddata['predictiondisplayvalue'] = $this->model->get_target()->get_display_value($predictedvalue); + $preddata['predictiondisplayvalue'] = $target->get_display_value($predictedvalue); list($preddata['style'], $preddata['outcomeicon']) = - insight::get_calculation_display($this->model->get_target(), floatval($predictedvalue), $output); + insight::get_calculation_display($target, floatval($predictedvalue), $output); $predictionvalues[$predictedvalue] = $preddata; } @@ -123,7 +135,7 @@ class insights_list implements \renderable, \templatable { } // Order predicted values. - if ($this->model->get_target()->is_linear()) { + if ($target->is_linear()) { // During regression what we will be interested on most of the time is in low values so let's show them first. ksort($predictionvalues); } else { diff --git a/report/insights/done.php b/report/insights/done.php new file mode 100644 index 00000000000..d64d15120b4 --- /dev/null +++ b/report/insights/done.php @@ -0,0 +1,30 @@ +. + +/** + * Forwards the user to the action they selected. + * + * @package report_insights + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +require_once(__DIR__ . '/../../config.php'); + +require_login(); + +$url = new \moodle_url('/'); +redirect($url, get_string('actionsaved', 'report_insights'), null, \core\output\notification::NOTIFY_SUCCESS); diff --git a/report/insights/insights.php b/report/insights/insights.php index 452f9d9e6bd..c45aaf9e79b 100644 --- a/report/insights/insights.php +++ b/report/insights/insights.php @@ -106,6 +106,22 @@ if (!$model->uses_insights()) { $PAGE->set_title($insightinfo->insightname); $PAGE->set_heading($insightinfo->contextname); +// Some models generate one single prediction per context. We can directly show the prediction details in this case. +if ($model->get_analyser()::one_sample_per_analysable()) { + + // Param $perpage to 2 so we can detect if this model's analyser is using one_sample_per_analysable incorrectly. + $predictionsdata = $model->get_predictions($context, true, 0, 2); + if ($predictionsdata) { + list($total, $predictions) = $predictionsdata; + if ($total > 1) { + throw new \coding_exception('This model\'s analyser processed more than one sample for a single analysable element.' . + 'Therefore, the analyser\'s one_sample_per_analysable() method should return false.'); + } + $prediction = reset($predictions); + $redirecturl = new \moodle_url('/report/insights/prediction.php', ['id' => $prediction->get_prediction_data()->id]); + redirect($redirecturl); + } +} echo $OUTPUT->header(); $renderable = new \report_insights\output\insights_list($model, $context, $othermodels, $page, $perpage); diff --git a/report/insights/lang/en/report_insights.php b/report/insights/lang/en/report_insights.php index 2051e30ee65..e21e8dbf31a 100644 --- a/report/insights/lang/en/report_insights.php +++ b/report/insights/lang/en/report_insights.php @@ -22,6 +22,7 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +$string['actionsaved'] = 'Your action has been saved.'; $string['calculatedvalue'] = 'Calculated value'; $string['disabledmodel'] = 'Sorry, this model has been disabled by the administrator'; $string['indicator'] = 'Indicator'; diff --git a/report/insights/lib.php b/report/insights/lib.php index e63bc79768a..97c9ecb86e4 100644 --- a/report/insights/lib.php +++ b/report/insights/lib.php @@ -59,7 +59,7 @@ function report_insights_extend_navigation_course($navigation, $course, $context function report_insights_myprofile_navigation(core_user\output\myprofile\tree $tree, $user, $iscurrentuser, $course) { $context = \context_user::instance($user->id); - if (has_capability('moodle/analytics:listinsights', $context)) { + if (\core_analytics\manager::check_can_list_insights($context, true)) { $modelids = report_insights_context_insights($context); if (!empty($modelids)) { diff --git a/report/insights/templates/insight_details.mustache b/report/insights/templates/insight_details.mustache index 3b59962adea..ad61ef1b94a 100644 --- a/report/insights/templates/insight_details.mustache +++ b/report/insights/templates/insight_details.mustache @@ -75,7 +75,8 @@

{{#str}}insightprediction, report_insights, {{insightname}} {{/str}}

- + {{/showpredicionheading}} diff --git a/report/insights/templates/insights_list.mustache b/report/insights/templates/insights_list.mustache index c2db15eed44..5f8da0c1588 100644 --- a/report/insights/templates/insights_list.mustache +++ b/report/insights/templates/insights_list.mustache @@ -87,15 +87,17 @@ {{{ pagingbar }}} {{#predictions}}
+ {{#showpredicionheading}} + {{#str}}prediction, report_insights{{/str}}: {{#outcomeicon}} @@ -84,6 +85,7 @@ {{predictiondisplayvalue}}
{{#str}}name{{/str}}
- + {{#showpredicionheading}} + + {{/showpredicionheading}} diff --git a/user/classes/analytics/target/upcoming_activities_due.php b/user/classes/analytics/target/upcoming_activities_due.php new file mode 100644 index 00000000000..096005cc292 --- /dev/null +++ b/user/classes/analytics/target/upcoming_activities_due.php @@ -0,0 +1,179 @@ +. + +/** + * Upcoming activities due target. + * + * @package core + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core_user\analytics\target; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->dirroot . '/lib/enrollib.php'); + +/** + * Upcoming activities due target. + * + * @package core + * @copyright 2019 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class upcoming_activities_due extends \core_analytics\local\target\binary { + + /** + * Machine learning backends are not required to predict. + * + * @return bool + */ + public static function based_on_assumptions() { + return true; + } + + /** + * Only update last analysis time when analysables are processed. + * @return bool + */ + public function always_update_analysis_time(): bool { + return false; + } + + /** + * Returns the name. + * + * If there is a corresponding '_help' string this will be shown as well. + * + * @return \lang_string + */ + public static function get_name() : \lang_string { + return new \lang_string('target:upcomingactivitiesdue'); + } + + /** + * Overwritten to show a simpler language string. + * + * @param int $modelid + * @param \context $context + * @return string + */ + public function get_insight_subject(int $modelid, \context $context) { + return get_string('youhaveupcomingactivitiesdue'); + } + + /** + * classes_description + * + * @return string[] + */ + protected static function classes_description() { + return array( + get_string('no'), + get_string('yes'), + ); + } + + /** + * Returns the predicted classes that will be ignored. + * + * @return array + */ + public function ignored_predicted_classes() { + // No need to process users without upcoming activities due. + return array(0); + } + + /** + * get_analyser_class + * + * @return string + */ + public function get_analyser_class() { + return '\core\analytics\analyser\users'; + } + + /** + * All users are ok. + * + * @param \core_analytics\analysable $analysable + * @param mixed $fortraining + * @return true|string + */ + public function is_valid_analysable(\core_analytics\analysable $analysable, $fortraining = true) { + // The calendar API used by \core_course\analytics\indicator\activities_due is already checking + // if the user has any courses. + return true; + } + + /** + * Samples are users and all of them are ok. + * + * @param int $sampleid + * @param \core_analytics\analysable $analysable + * @param bool $fortraining + * @return bool + */ + public function is_valid_sample($sampleid, \core_analytics\analysable $analysable, $fortraining = true) { + return true; + } + + /** + * Calculation based on activities due indicator. + * + * @param int $sampleid + * @param \core_analytics\analysable $analysable + * @param int $starttime + * @param int $endtime + * @return float + */ + protected function calculate_sample($sampleid, \core_analytics\analysable $analysable, $starttime = false, $endtime = false) { + + $activitiesdueindicator = $this->retrieve('\core_course\analytics\indicator\activities_due', $sampleid); + if ($activitiesdueindicator == \core_course\analytics\indicator\activities_due::get_max_value()) { + return 1; + } + return 0; + } + + /** + * Adds a view upcoming events action. + * + * @param \core_analytics\prediction $prediction + * @param mixed $includedetailsaction + * @param bool $isinsightuser + * @return \core_analytics\prediction_action[] + */ + public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false, + $isinsightuser = false) { + global $CFG, $USER; + + $parentactions = parent::prediction_actions($prediction, $includedetailsaction); + + if (!$isinsightuser && $USER->id != $prediction->get_prediction_data()->sampleid) { + return $parentactions; + } + + // We force a lookahead of 30 days so we are sure that the upcoming activities due are shown. + $url = new \moodle_url('/calendar/view.php', ['view' => 'upcoming', 'lookahead' => '30']); + $pix = new \pix_icon('i/calendar', get_string('upcomingevents', 'calendar')); + $action = new \core_analytics\prediction_action('viewupcoming', $prediction, + $url, $pix, get_string('upcomingevents', 'calendar')); + + return array_merge([$action], $parentactions); + } +} diff --git a/version.php b/version.php index 08fe4374e63..5368791a889 100644 --- a/version.php +++ b/version.php @@ -29,11 +29,11 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2019040600.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2019041000.00; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes. -$release = '3.7dev+ (Build: 20190406)'; // Human-friendly version name +$release = '3.7dev+ (Build: 20190410)'; // Human-friendly version name $branch = '37'; // This version's branch. $maturity = MATURITY_ALPHA; // This version's maturity level.
- {{#str}}prediction, report_insights{{/str}}: - - {{#outcomeicon}} - {{> core/pix_icon}} - {{/outcomeicon}} - {{predictiondisplayvalue}} - - + {{#str}}prediction, report_insights{{/str}}: + + {{#outcomeicon}} + {{> core/pix_icon}} + {{/outcomeicon}} + {{predictiondisplayvalue}} + +
{{#str}}name{{/str}}