Merge branch 'master' into install_master

admin/settings/courses.php

@@ -458,4 +458,28 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) {
 
     $ADMIN->add('backups', $temp);
 
+    // Create a page for asynchronous backup and restore configuration and defaults.
+    if (!empty($CFG->enableasyncbackup)) {  // Only add settings if async mode is enable at site level.
+        $temp = new admin_settingpage('asyncgeneralsettings', new lang_string('asyncgeneralsettings', 'backup'));
+
+        $temp->add(new admin_setting_configcheckbox(
+                'backup/backup_async_message_users',
+                new lang_string('asyncemailenable', 'backup'),
+                new lang_string('asyncemailenabledetail', 'backup'), 0));
+
+        $temp->add(new admin_setting_configtext(
+                'backup/backup_async_message_subject',
+                new lang_string('asyncmessagesubject', 'backup'),
+                new lang_string('asyncmessagesubjectdetail', 'backup'),
+                new lang_string('asyncmessagesubjectdefault', 'backup')));
+
+        $temp->add(new admin_setting_confightmleditor(
+                'backup/backup_async_message',
+                new lang_string('asyncmessagebody', 'backup'),
+                new lang_string('asyncmessagebodydetail', 'backup'),
+                new lang_string('asyncmessagebodydefault', 'backup')));
+
+        $ADMIN->add('backups', $temp);
+    }
+
 }

admin/settings/subsystems.php

@@ -51,4 +51,7 @@ if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page
 
     $optionalsubsystems->add(new admin_setting_configcheckbox('enablecoursepublishing',
         new lang_string('enablecoursepublishing', 'hub'), new lang_string('enablecoursepublishing_help', 'hub'), 0));
+
+    $optionalsubsystems->add(new admin_setting_configcheckbox('enableasyncbackup', new lang_string('enableasyncbackup', 'backup'),
+        new lang_string('enableasyncbackup_help', 'backup'), 0, 1, 0));
 }

admin/tool/analytics/classes/output/form/edit_model.php

@@ -74,6 +74,7 @@ class edit_model extends \moodleform {
         );
         $mform->addElement('autocomplete', 'indicators', get_string('indicators', 'tool_analytics'), $indicators, $options);
         $mform->setType('indicators', PARAM_ALPHANUMEXT);
+        $mform->addHelpButton('indicators', 'indicators', 'tool_analytics');
 
         $timesplittings = array('' => '');
         foreach ($this->_customdata['timesplittings'] as $classname => $timesplitting) {

admin/tool/analytics/classes/output/helper.php

@@ -92,4 +92,17 @@ class helper {
         $PAGE->set_title($title);
         $PAGE->set_heading($title);
     }
+
+    /**
+     * Resets the current page.
+     *
+     * Note that this function can only be used by analytics pages that work at the system context.
+     *
+     * @return null
+     */
+    public static function reset_page() {
+        global $PAGE;
+        $PAGE->reset_theme_and_output();
+        $PAGE->set_context(\context_system::instance());
+    }
 }

admin/tool/analytics/classes/output/invalid_analysables.php

@@ -76,13 +76,17 @@ class invalid_analysables implements \renderable, \templatable {
 
         $offset = $this->page * $this->perpage;
 
-        $analysables = $this->model->get_analyser(['notimesplitting' => true])->get_analysables();
+        $analysables = $this->model->get_analyser(['notimesplitting' => true])->get_analysables_iterator();
 
         $skipped = 0;
         $enoughresults = false;
         $morepages = false;
         $results = array();
-        foreach ($analysables as $key => $analysable) {
+        foreach ($analysables as $analysable) {
+
+            if (!$analysable) {
+                continue;
+            }
 
             $validtraining = $this->model->get_target()->is_valid_analysable($analysable, true);
             if ($validtraining === true) {
@@ -117,8 +121,6 @@ class invalid_analysables implements \renderable, \templatable {
                 $morepages = true;
                 break;
             }
-
-            unset($analysables[$key]);
         }
 
         // Prepare the context object.

admin/tool/analytics/classes/task/predict_models.php

@@ -60,6 +60,10 @@ class predict_models extends \core\task\scheduled_task {
 
         foreach ($models as $model) {
             $result = $model->predict();
+
+            // Reset the page as some indicators may call external functions that overwrite the page context.
+            \tool_analytics\output\helper::reset_page();
+
             if ($result) {
                 echo $OUTPUT->heading(get_string('modelresults', 'tool_analytics', $model->get_target()->get_name()));
                 $renderer = $PAGE->get_renderer('tool_analytics');

admin/tool/analytics/classes/task/train_models.php

@@ -71,6 +71,10 @@ class train_models extends \core\task\scheduled_task {
             }
 
             $result = $model->train();
+
+            // Reset the page as some indicators may call external functions that overwrite the page context.
+            \tool_analytics\output\helper::reset_page();
+
             if ($result) {
                 echo $OUTPUT->heading(get_string('modelresults', 'tool_analytics', $model->get_target()->get_name()));
 

admin/tool/analytics/cli/evaluate_model.php

@@ -111,6 +111,9 @@ $analyseroptions = array(
 // Evaluate its suitability to predict accurately.
 $results = $model->evaluate($analyseroptions);
 
+// Reset the page as some indicators may call external functions that overwrite the page context.
+\tool_analytics\output\helper::reset_page();
+
 $renderer = $PAGE->get_renderer('tool_analytics');
 echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs());
 

admin/tool/analytics/lang/en/tool_analytics.php

@@ -78,6 +78,8 @@ $string['getpredictions'] = 'Get predictions';
 $string['goodmodel'] = 'This is a good model for using to obtain predictions. Enable it to start obtaining predictions.';
 $string['importmodel'] = 'Import model';
 $string['indicators'] = 'Indicators';
+$string['indicators_help'] = 'The indicators are what you think will lead to an accurate prediction of the target.';
+$string['indicators_link'] = 'Indicators';
 $string['info'] = 'Info';
 $string['ignoreversionmismatches'] = 'Ignore version mismatches';
 $string['ignoreversionmismatchescheckbox'] = 'Ignore the differences between this site version and the original site version.';
@@ -111,6 +113,7 @@ $string['samestartdate'] = 'Current start date is good';
 $string['sameenddate'] = 'Current end date is good';
 $string['target'] = 'Target';
 $string['target_help'] = 'The target is what the model will predict.';
+$string['target_link'] = 'Targets';
 $string['timesplittingnotdefined'] = 'Time splitting is not defined.';
 $string['timesplittingnotdefined_help'] = 'You need to select a time-splitting method before enabling the model.';
 $string['trainandpredictmodel'] = 'Training model and calculating predictions';

admin/tool/analytics/model.php

@@ -156,8 +156,6 @@ switch ($action) {
     case 'evaluate':
         confirm_sesskey();
 
-        echo $OUTPUT->header();
-
         if ($model->is_static()) {
             throw new moodle_exception('errornostaticevaluate', 'tool_analytics');
         }
@@ -176,6 +174,12 @@ switch ($action) {
             $options['mode'] = 'trainedmodel';
         }
         $results = $model->evaluate($options);
+
+        // We reset the theme and the output as some indicators may be using external functions
+        // which reset $PAGE.
+        \tool_analytics\output\helper::reset_page();
+        echo $OUTPUT->header();
+
         $renderer = $PAGE->get_renderer('tool_analytics');
         echo $renderer->render_evaluate_results($results, $model->get_analyser()->get_logs());
         break;
@@ -183,8 +187,6 @@ switch ($action) {
     case 'getpredictions':
         confirm_sesskey();
 
-        echo $OUTPUT->header();
-
         if ($onlycli) {
             throw new moodle_exception('erroronlycli', 'tool_analytics');
         }
@@ -202,6 +204,11 @@ switch ($action) {
             $predictlogs = array();
         }
 
+        // We reset the theme and the output as some indicators may be using external functions
+        // which reset $PAGE.
+        \tool_analytics\output\helper::reset_page();
+        echo $OUTPUT->header();
+
         $renderer = $PAGE->get_renderer('tool_analytics');
         echo $renderer->render_get_predictions_results($trainresults, $trainlogs, $predictresults, $predictlogs);
         break;

admin/tool/mobile/classes/external.php

@@ -262,6 +262,8 @@ class external extends external_api {
     /**
      * Creates an auto-login key for the current user. Is created only in https sites and is restricted by time and ip address.
      *
+     * Please note that it only works if the request comes from the Moodle mobile or desktop app.
+     *
      * @param string $privatetoken the user private token for validating the request
      * @return array with the settings and warnings
      * @since  Moodle 3.2
@@ -284,6 +286,12 @@ class external extends external_api {
             }
         }
 
+        // Only requests from the Moodle mobile or desktop app. This enhances security to avoid any type of XSS attack.
+        // This code goes intentionally here and not inside the check_autologin_prerequisites() function because it
+        // is used by other PHP scripts that can be opened in any browser.
+        if (!\core_useragent::is_moodle_app()) {
+            throw new moodle_exception('apprequired', 'tool_mobile');
+        }
         api::check_autologin_prerequisites($USER->id);
 
         if (isset($_GET['privatetoken']) or empty($privatetoken)) {

admin/tool/mobile/db/services.php

@@ -57,7 +57,8 @@ $functions = array(
         'classname'   => 'tool_mobile\external',
         'methodname'  => 'get_autologin_key',
         'description' => 'Creates an auto-login key for the current user.
-                            Is created only in https sites and is restricted by time and ip address.',
+                            Is created only in https sites and is restricted by time, ip address and only works if the request
+                            comes from the Moodle mobile or desktop app.',
         'type'        => 'write',
         'services'    => array(MOODLE_OFFICIAL_MOBILE_SERVICE),
     ),

admin/tool/mobile/lang/en/tool_mobile.php

@@ -27,6 +27,7 @@ $string['androidappid'] = 'Android app\'s unique identifier';
 $string['androidappid_desc'] = 'This setting may be left as default unless you have a custom Android app.';
 $string['apppolicy'] = 'App policy URL';
 $string['apppolicy_help'] = 'The URL of a policy for app users which is listed on the About page in the app. If the field is left empty, the site policy URL will be used instead.';
+$string['apprequired'] = 'This functionality is only available when accessed via the Moodle mobile or desktop app.';
 $string['autologinkeygenerationlockout'] = 'Auto-login key generation is blocked. You need to wait 6 minutes between requests.';
 $string['autologinnotallowedtoadmins'] = 'Auto-login is not allowed for site admins.';
 $string['cachedef_plugininfo'] = 'This stores the list of plugins with mobile addons';

admin/tool/mobile/tests/externallib_test.php

@@ -213,6 +213,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         // Enable requeriments.
         $_GET['wstoken'] = $token->token;   // Mock parameters.
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+                'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         // Even if we force the password change for the current user we should be able to retrieve the key.
         set_user_preference('auth_forcepasswordchange', 1, $user->id);
 
@@ -240,6 +244,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         global $CFG;
         $this->resetAfterTest(true);
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         // Need to disable webservices to verify that's checked.
         $CFG->enablewebservices = 0;
         $CFG->enablemobilewebservice = 0;
@@ -256,6 +264,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
     public function test_get_autologin_key_missing_https() {
         global $CFG;
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         // Need to simulate a non HTTPS site here.
         $CFG->wwwroot = str_replace('https:', 'http:', $CFG->wwwroot);
 
@@ -276,6 +288,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         $this->resetAfterTest(true);
         $this->setAdminUser();
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         $this->expectException('moodle_exception');
         $this->expectExceptionMessage(get_string('autologinnotallowedtoadmins', 'tool_mobile'));
         $result = external::get_autologin_key('');
@@ -296,6 +312,10 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         $token = external_generate_token_for_current_user($service);
         $_GET['wstoken'] = $token->token;   // Mock parameters.
 
+        // Fake the app.
+        core_useragent::instance(true, 'Mozilla/5.0 (Linux; Android 7.1.1; Moto G Play Build/NPIS26.48-43-2; wv) ' .
+            'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MoodleMobile');
+
         $result = external::get_autologin_key($token->privatetoken);
         $result = external_api::clean_returnvalue(external::get_autologin_key_returns(), $result);
 
@@ -311,6 +331,20 @@ class tool_mobile_external_testcase extends externallib_advanced_testcase {
         $result = external::get_autologin_key($token->privatetoken);
     }
 
+    /**
+     * Test get_autologin_key missing app_request.
+     */
+    public function test_get_autologin_key_missing_app_request() {
+        global $CFG;
+
+        $this->resetAfterTest(true);
+        $this->setAdminUser();
+
+        $this->expectException('moodle_exception');
+        $this->expectExceptionMessage(get_string('apprequired', 'tool_mobile'));
+        $result = external::get_autologin_key('');
+    }
+
     /**
      * Test get_content.
      */

admin/tool/mobile/upgrade.txt

@@ -4,6 +4,10 @@ Information provided here is intended especially for developers.
 === 3.7 ===
 
  * New external function tool_mobile::tool_mobile_call_external_function allows calling multiple external functions and returns all responses.
+ * External function tool_mobile::get_autologin_key now only works if the request comes from the Moodle mobile or desktop app.
+   This increases confidence that requests did originate from the mobile app, decreasing the likelihood of an XSS attack.
+   If you want to use this functionality, please override the Web Service via the override_webservice_execution callback although
+   this is not recommended or encouraged.
 
 === 3.5 ===
 

admin/tool/recyclebin/classes/category_bin.php

@@ -108,6 +108,16 @@ class category_bin extends base_bin {
 
         require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 
+        // As far as recycle bin is using MODE_AUTOMATED, it observes the backup_auto_storage
+        // setting (storing backups @ real location. For recycle bin we want to ensure that
+        // backup files are always stored in Moodle file area. In order to achieve that, we
+        // hack the setting here via $CFG->forced_plugin_settings, so it won't interfere other
+        // operations.  See MDL-65218 for more information.
+        // This hack will be removed once recycle bin switches to use its own backup mode, with
+        // own preferences and 100% appart from MODLE_AUTOMATED.
+        // TODO: Remove this as part of MDL-65228.
+        $CFG->forced_plugin_settings['backup'] = ['backup_auto_storage' => 0];
+
         // Backup the course.
         $user = get_admin();
         $controller = new \backup_controller(
@@ -115,11 +125,15 @@ class category_bin extends base_bin {
             $course->id,
             \backup::FORMAT_MOODLE,
             \backup::INTERACTIVE_NO,
-            \backup::MODE_GENERAL,
+            \backup::MODE_AUTOMATED,
             $user->id
         );
         $controller->execute_plan();
 
+        // We don't need the forced setting anymore, hence unsetting it.
+        // TODO: Remove this as part of MDL-65228.
+        unset($CFG->forced_plugin_settings['backup']);
+
         // Grab the result.
         $result = $controller->get_results();
         if (!isset($result['backup_destination'])) {
@@ -233,7 +247,7 @@ class category_bin extends base_bin {
             $tempdir,
             $course->id,
             \backup::INTERACTIVE_NO,
-            \backup::MODE_GENERAL,
+            \backup::MODE_AUTOMATED,
             $user->id,
             \backup::TARGET_NEW_COURSE
         );

admin/tool/recyclebin/classes/course_bin.php

@@ -112,6 +112,16 @@ class course_bin extends base_bin {
             return;
         }
 
+        // As far as recycle bin is using MODE_AUTOMATED, it observes the backup_auto_storage
+        // setting (storing backups @ real location. For recycle bin we want to ensure that
+        // backup files are always stored in Moodle file area. In order to achieve that, we
+        // hack the setting here via $CFG->forced_plugin_settings, so it won't interfere other
+        // operations.  See MDL-65218 for more information.
+        // This hack will be removed once recycle bin switches to use its own backup mode, with
+        // own preferences and 100% appart from MODLE_AUTOMATED.
+        // TODO: Remove this as part of MDL-65228.
+        $CFG->forced_plugin_settings['backup'] = ['backup_auto_storage' => 0];
+
         // Backup the activity.
         $user = get_admin();
         $controller = new \backup_controller(
@@ -124,6 +134,10 @@ class course_bin extends base_bin {
         );
         $controller->execute_plan();
 
+        // We don't need the forced setting anymore, hence unsetting it.
+        // TODO: Remove this as part of MDL-65228.
+        unset($CFG->forced_plugin_settings['backup']);
+
         // Grab the result.
         $result = $controller->get_results();
         if (!isset($result['backup_destination'])) {
@@ -344,4 +358,4 @@ class course_bin extends base_bin {
         $context = \context_course::instance($this->_courseid);
         return has_capability('tool/recyclebin:deleteitems', $context);
     }
-}
+}

admin/tool/recyclebin/tests/category_bin_test.php

@@ -176,4 +176,116 @@ class tool_recyclebin_category_bin_tests extends advanced_testcase {
         $course = reset($courses);
         $this->assertEquals('Test course 2', $course->fullname);
     }
+
+    /**
+     * Provider for test_course_restore_with_userdata() and test_course_restore_without_userdata()
+     *
+     * Used to verify that recycle bin is immune to various settings. Provides plugin, name, value for
+     * direct usage with set_config()
+     */
+    public function recycle_bin_settings_provider() {
+        return [
+            'backup/backup_auto_storage moodle' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 0],
+            ]],
+
+            'backup/backup_auto_storage external' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 1],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+
+            'backup/backup_auto_storage mixed' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 2],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+        ];
+    }
+
+    /**
+     * Tests that user data is restored when course is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
+     */
+    public function test_course_restore_with_userdata($settings) {
+        global $DB;
+
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
+        // We want user data to be included for this test.
+        set_config('backup_auto_users', true, 'backup');
+
+        $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
+
+        // Delete course.
+        delete_course($this->course, false);
+        $this->assertFalse($DB->record_exists('course', ['id' => $this->course->id]));
+
+        // Verify there is now a backup @ cat recycle bin file area.
+        $recyclebin = new \tool_recyclebin\category_bin($this->course->category);
+        $this->assertEquals(1, count($recyclebin->get_items()));
+
+        // Restore the recycle bin item.
+        $recyclebin->restore_item(current($recyclebin->get_items()));
+
+        // Get the new course.
+        $newcourse = $DB->get_record('course', ['shortname' => $this->course->shortname], '*', MUST_EXIST);
+
+        // Check that it was removed from the recycle bin.
+        $this->assertEquals(0, count($recyclebin->get_items()));
+
+        // Verify that student DOES continue enrolled.
+        $this->assertTrue(is_enrolled(context_course::instance($newcourse->id), $student->id));
+    }
+
+    /**
+     * Tests that user data is not restored when course is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
+     */
+    public function test_course_restore_without_userdata($settings) {
+        global $DB;
+
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
+        // We want user data to be included for this test.
+        set_config('backup_auto_users', false, 'backup');
+
+        $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
+
+        // Delete course.
+        delete_course($this->course, false);
+        $this->assertFalse($DB->record_exists('course', ['id' => $this->course->id]));
+
+        // Verify there is now a backup @ cat recycle bin file area.
+        $recyclebin = new \tool_recyclebin\category_bin($this->course->category);
+        $this->assertEquals(1, count($recyclebin->get_items()));
+
+        // Restore the recycle bin item.
+        $recyclebin->restore_item(current($recyclebin->get_items()));
+
+        // Get the new course.
+        $newcourse = $DB->get_record('course', ['shortname' => $this->course->shortname], '*', MUST_EXIST);
+
+        // Check that it was removed from the recycle bin.
+        $this->assertEquals(0, count($recyclebin->get_items()));
+
+        // Verify that student DOES NOT continue enrolled.
+        $this->assertFalse(is_enrolled(context_course::instance($newcourse->id), $student->id));
+    }
 }

admin/tool/recyclebin/tests/course_bin_test.php

@@ -175,9 +175,45 @@ class tool_recyclebin_course_bin_tests extends advanced_testcase {
     }
 
     /**
-     * Tests that user data is restored when module is restored.
+     * Provider for test_coursemodule_restore_with_userdata() and test_coursemodule_restore_without_userdata()
+     *
+     * Used to verify that recycle bin is immune to various settings. Provides plugin, name, value for
+     * direct usage with set_config()
      */
-    public function test_coursemodule_restore_with_userdata() {
+    public function recycle_bin_settings_provider() {
+        return [
+            'backup/backup_auto_storage moodle' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 0],
+            ]],
+
+            'backup/backup_auto_storage external' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 1],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+
+            'backup/backup_auto_storage mixed' => [[
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_storage', 'value' => 2],
+                (object)['plugin' => 'backup', 'name' => 'backup_auto_destination', 'value' => true],
+            ]],
+        ];
+    }
+
+    /**
+     * Tests that user data is restored when module is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
+     */
+    public function test_coursemodule_restore_with_userdata($settings) {
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
         $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
         $this->setUser($student);
 
@@ -211,8 +247,20 @@ class tool_recyclebin_course_bin_tests extends advanced_testcase {
 
     /**
      * Tests that user data is not restored when module is restored.
+     *
+     * @dataProvider recycle_bin_settings_provider
+     * @param array $settings array of plugin, name, value stdClass().
      */
-    public function test_coursemodule_restore_without_userdata() {
+    public function test_coursemodule_restore_without_userdata($settings) {
+        // Force configuration changes from provider.
+        foreach ($settings as $setting) {
+            // Need to create a directory for backup_auto_destination.
+            if ($setting->plugin === 'backup' && $setting->name === 'backup_auto_destination' && $setting->value === true) {
+                $setting->value = make_request_directory();
+            }
+            set_config($setting->name, $setting->value, $setting->plugin);
+        }
+
         $student = $this->getDataGenerator()->create_and_enrol($this->course, 'student');
         $this->setUser($student);
 

analytics/classes/analysable.php

@@ -29,10 +29,6 @@ defined('MOODLE_INTERNAL') || die();
 /**
  * Any element analysers can analyse.
  *
- * Analysers get_analysers method return all analysable elements in the site;
- * it is important that analysable elements implement lazy loading to avoid
- * big memory footprints. See \core_analytics\course example.
- *
  * @package   core_analytics
  * @copyright 2016 David Monllao {@link http://www.davidmonllao.com}
  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later

analytics/classes/analysis.php

@@ -0,0 +1,894 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Runs an analysis of the site.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Runs an analysis of the site.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class analysis {
+
+    /**
+     * @var \core_analytics\local\analyser\base
+     */
+    private $analyser;
+
+    /**
+     * @var bool Whether to calculate the target or not in this run.
+     */
+    private $includetarget;
+
+    /**
+     * @var \core_analytics\local\analysis\result
+     */
+    private $result;
+
+    /**
+     * @var \core\lock\lock
+     */
+    private $lock;
+
+    /**
+     * Constructor.
+     *
+     * @param \core_analytics\local\analyser\base   $analyser
+     * @param bool                                  $includetarget Whether to calculate the target or not.
+     * @param \core_analytics\local\analysis\result $result
+     */
+    public function __construct(\core_analytics\local\analyser\base $analyser, bool $includetarget,
+            \core_analytics\local\analysis\result $result) {
+        $this->analyser = $analyser;
+        $this->includetarget = $includetarget;
+        $this->result = $result;
+    }
+
+    /**
+     * Runs the analysis.
+     *
+     * @return null
+     */
+    public function run() {
+
+        $options = $this->analyser->get_options();
+
+        // Time limit control.
+        $modeltimelimit = intval(get_config('analytics', 'modeltimelimit'));
+
+        $filesbytimesplitting = array();
+
+        $alreadyprocessedanalysables = $this->get_processed_analysables();
+
+        if ($this->includetarget) {
+            $action = 'training';
+        } else {
+            $action = 'prediction';
+        }
+        $analysables = $this->analyser->get_analysables_iterator($action);
+
+        $inittime = microtime(true);
+        foreach ($analysables as $analysable) {
+            $processed = false;
+
+            if (!$analysable) {
+                continue;
+            }
+
+            $analysableresults = $this->process_analysable($analysable);
+            if ($analysableresults) {
+                $processed = $this->result->add_analysable_results($analysableresults);
+                if (!$processed) {
+                    $errors = array();
+                    foreach ($analysableresults as $timesplittingid => $result) {
+                        $str = '';
+                        if (count($analysableresults) > 1) {
+                            $str .= $timesplittingid . ': ';
+                        }
+                        $str .= $result->message;
+                        $errors[] = $str;
+                    }
+
+                    $a = new \stdClass();
+                    $a->analysableid = $analysable->get_name();
+                    $a->errors = implode(', ', $errors);
+                    $this->analyser->add_log(get_string('analysablenotused', 'analytics', $a));
+                }
+            }
+
+            // Updated regardless of how well the analysis went.
+            if ($this->analyser->get_target()->always_update_analysis_time() || $processed) {
+                $this->update_analysable_analysed_time($alreadyprocessedanalysables, $analysable->get_id());
+            }
+
+            // Apply time limit.
+            if (!$options['evaluation']) {
+                $timespent = microtime(true) - $inittime;
+                if ($modeltimelimit <= $timespent) {
+                    break;
+                }
+            }
+        }
+    }
+
+    /**
+     * Get analysables that have been already processed.
+     *
+     * @return \stdClass[]
+     */
+    protected function get_processed_analysables(): array {
+        global $DB;
+
+        $params = array('modelid' => $this->analyser->get_modelid());
+        $params['action'] = ($this->includetarget) ? 'training' : 'prediction';
+        $select = 'modelid = :modelid and action = :action';
+
+        // Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid).
+        return $DB->get_records_select('analytics_used_analysables', $select,
+            $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey');
+    }
+
+    /**
+     * Processes an analysable
+     *
+     * This method returns the general analysable status, an array of files by time splitting method and
+     * an error message if there is any problem.
+     *
+     * @param \core_analytics\analysable $analysable
+     * @return \stdClass[] Results objects by time splitting method
+     */
+    public function process_analysable(\core_analytics\analysable $analysable): array {
+
+        // Target instances scope is per-analysable (it can't be lower as calculations run once per
+        // analysable, not time splitting method nor time range).
+        $target = call_user_func(array($this->analyser->get_target(), 'instance'));
+
+        // We need to check that the analysable is valid for the target even if we don't include targets
+        // as we still need to discard invalid analysables for the target.
+        $isvalidresult = $target->is_valid_analysable($analysable, $this->includetarget);
+        if ($isvalidresult !== true) {
+            $a = new \stdClass();
+            $a->analysableid = $analysable->get_name();
+            $a->result = $isvalidresult;
+            $this->analyser->add_log(get_string('analysablenotvalidfortarget', 'analytics', $a));
+            return array();
+        }
+
+        // Process all provided time splitting methods.
+        $results = array();
+        foreach ($this->analyser->get_timesplittings() as $timesplitting) {
+
+            $cachedresult = $this->result->retrieve_cached_result($timesplitting, $analysable);
+            if ($cachedresult) {
+                $result = new \stdClass();
+                $result->result = $cachedresult;
+                $results[$timesplitting->get_id()] = $result;
+                continue;
+            }
+
+            $results[$timesplitting->get_id()] = $this->process_time_splitting($timesplitting, $analysable, $target);
+        }
+
+        return $results;
+    }
+
+    /**
+     * Processes the analysable samples using the provided time splitting method.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \core_analytics\analysable $analysable
+     * @param \core_analytics\local\target\base $target
+     * @return \stdClass Results object.
+     */
+    protected function process_time_splitting(\core_analytics\local\time_splitting\base $timesplitting,
+            \core_analytics\analysable $analysable, \core_analytics\local\target\base $target): \stdClass {
+
+        $options = $this->analyser->get_options();
+
+        $result = new \stdClass();
+
+        if (!$timesplitting->is_valid_analysable($analysable)) {
+            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+            $result->message = get_string('invalidanalysablefortimesplitting', 'analytics',
+                $timesplitting->get_name());
+            return $result;
+        }
+        $timesplitting->set_analysable($analysable);
+
+        if (CLI_SCRIPT && !PHPUNIT_TEST) {
+            mtrace('Analysing id "' . $analysable->get_id() . '" with "' . $timesplitting->get_name() .
+                '" time splitting method...');
+        }
+
+        // What is a sample is defined by the analyser, it can be an enrolment, a course, a user, a question
+        // attempt... it is on what we will base indicators calculations.
+        list($sampleids, $samplesdata) = $this->analyser->get_all_samples($analysable);
+
+        if (count($sampleids) === 0) {
+            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+            $result->message = get_string('nodata', 'analytics');
+            return $result;
+        }
+
+        if ($this->includetarget) {
+            // All ranges are used when we are calculating data for training.
+            $ranges = $timesplitting->get_training_ranges();
+        } else {
+            // The latest range that has not yet been used for prediction (it depends on the time range where we are right now).
+            $ranges = $timesplitting->get_most_recent_prediction_range();
+        }
+
+        // There is no need to keep track of the evaluated samples and ranges as we always evaluate the whole dataset.
+        if ($options['evaluation'] === false) {
+
+            if (empty($ranges)) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('noranges', 'analytics');
+                return $result;
+            }
+
+            // We skip all samples that are already part of a training dataset, even if they have not been used for prediction.
+            if (!$target::based_on_assumptions()) {
+                // Targets based on assumptions can not be trained.
+                $this->filter_out_train_samples($sampleids, $timesplitting);
+            }
+
+            if (count($sampleids) === 0) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('nonewdata', 'analytics');
+                return $result;
+            }
+
+            // Only when processing data for predictions.
+            if (!$this->includetarget) {
+                // We also filter out samples and ranges that have already been used for predictions.
+                $predictsamplesrecord = $this->filter_out_prediction_samples_and_ranges($sampleids, $ranges, $timesplitting);
+            }
+
+            if (count($sampleids) === 0) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('nonewdata', 'analytics');
+                return $result;
+            }
+
+            if (count($ranges) === 0) {
+                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+                $result->message = get_string('nonewranges', 'analytics');
+                return $result;
+            }
+        }
+
+        // Flag the model + analysable + timesplitting as being analysed (prevent concurrent executions).
+        if (!$this->init_analysable_analysis($timesplitting->get_id(), $analysable->get_id())) {
+            // If this model + analysable + timesplitting combination is being analysed we skip this process.
+            $result->status = \core_analytics\model::NO_DATASET;
+            $result->message = get_string('analysisinprogress', 'analytics');
+            return $result;
+        }
+
+        // Remove samples the target consider invalid.
+        try {
+            $target->add_sample_data($samplesdata);
+            $target->filter_out_invalid_samples($sampleids, $analysable, $this->includetarget);
+        } catch (\Throwable $e) {
+            $this->finish_analysable_analysis();
+            throw $e;
+        }
+
+        if (!$sampleids) {
+            $result->status = \core_analytics\model::NO_DATASET;
+            $result->message = get_string('novalidsamples', 'analytics');
+            $this->finish_analysable_analysis();
+            return $result;
+        }
+
+        try {
+            $indicators = $this->analyser->get_indicators();
+            foreach ($indicators as $key => $indicator) {
+                // The analyser attaches the main entities the sample depends on and are provided to the
+                // indicator to calculate the sample.
+                $indicators[$key]->add_sample_data($samplesdata);
+            }
+
+            // Here we start the memory intensive process that will last until $data var is
+            // unset (until the method is finished basically).
+            $data = $this->calculate($timesplitting, $sampleids, $ranges, $target);
+        } catch (\Throwable $e) {
+            $this->finish_analysable_analysis();
+            throw $e;
+        }
+
+        if (!$data) {
+            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
+            $result->message = get_string('novaliddata', 'analytics');
+            $this->finish_analysable_analysis();
+            return $result;
+        }
+
+        try {
+            // No need to keep track of analysed stuff when evaluating.
+            if ($options['evaluation'] === false) {
+                // Save the samples that have been already analysed so they are not analysed again in future.
+
+                if ($this->includetarget) {
+                    $this->save_train_samples($sampleids, $timesplitting);
+                } else {
+                    // The variable $predictsamplesrecord will always be set as filter_out_prediction_samples_and_ranges
+                    // will always be called before it (no evaluation mode and no includetarget).
+                    $this->save_prediction_samples($sampleids, $ranges, $timesplitting, $predictsamplesrecord);
+                }
+            }
+
+            // We need to pass all the analysis data.
+            $formattedresult = $this->result->format_result($data, $target, $timesplitting, $analysable);
+
+        } catch (\Throwable $e) {
+            $this->finish_analysable_analysis();
+            throw $e;
+        }
+
+        if (!$formattedresult) {
+            $this->finish_analysable_analysis();
+            throw new \moodle_exception('errorcannotwritedataset', 'analytics');
+        }
+
+        $result->status = \core_analytics\model::OK;
+        $result->message = get_string('successfullyanalysed', 'analytics');
+        $result->result = $formattedresult;
+
+        // Flag the model + analysable + timesplitting as analysed.
+        $this->finish_analysable_analysis();
+
+        return $result;
+    }
+
+    /**
+     * Calculates indicators and targets.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $sampleids
+     * @param array $ranges
+     * @param \core_analytics\local\target\base $target
+     * @return array|null
+     */
+    public function calculate(\core_analytics\local\time_splitting\base $timesplitting, array &$sampleids,
+            array $ranges, \core_analytics\local\target\base $target): ?array {
+
+        $calculatedtarget = null;
+        if ($this->includetarget) {
+            // We first calculate the target because analysable data may still be invalid or none
+            // of the analysable samples may be valid.
+            $calculatedtarget = $target->calculate($sampleids, $timesplitting->get_analysable());
+
+            // We remove samples we can not calculate their target.
+            $sampleids = array_filter($sampleids, function($sampleid) use ($calculatedtarget) {
+                if (is_null($calculatedtarget[$sampleid])) {
+                    return false;
+                }
+                return true;
+            });
+        }
+
+        // No need to continue calculating if the target couldn't be calculated for any sample.
+        if (empty($sampleids)) {
+            return null;
+        }
+
+        $dataset = $this->calculate_indicators($timesplitting, $sampleids, $ranges);
+
+        if (empty($dataset)) {
+            return null;
+        }
+
+        // Now that we have the indicators in place we can add the time range indicators (and target if provided) to each of them.
+        $this->fill_dataset($timesplitting, $dataset, $calculatedtarget);
+
+        $this->add_context_metadata($timesplitting, $dataset, $target);
+
+        if (!PHPUNIT_TEST && CLI_SCRIPT) {
+            echo PHP_EOL;
+        }
+
+        return $dataset;
+    }
+
+    /**
+     * Calculates indicators.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $sampleids
+     * @param array $ranges
+     * @return array
+     */
+    protected function calculate_indicators(\core_analytics\local\time_splitting\base $timesplitting, array $sampleids,
+            array $ranges): array {
+        global $DB;
+
+        $options = $this->analyser->get_options();
+
+        $dataset = array();
+
+        // Faster to run 1 db query per range.
+        $existingcalculations = array();
+        if ($timesplitting->cache_indicator_calculations()) {
+            foreach ($ranges as $rangeindex => $range) {
+                // Load existing calculations.
+                $existingcalculations[$rangeindex] = \core_analytics\manager::get_indicator_calculations(
+                    $timesplitting->get_analysable(), $range['start'], $range['end'], $this->analyser->get_samples_origin());
+            }
+        }
+
+        // Here we store samples which calculations are not all null.
+        $notnulls = array();
+
+        // Fill the dataset samples with indicators data.
+        $newcalculations = array();
+        foreach ($this->analyser->get_indicators() as $indicator) {
+
+            // Hook to allow indicators to store analysable-dependant data.
+            $indicator->fill_per_analysable_caches($timesplitting->get_analysable());
+
+            // Per-range calculations.
+            foreach ($ranges as $rangeindex => $range) {
+
+                // Indicator instances are per-range.
+                $rangeindicator = clone $indicator;
+
+                $prevcalculations = array();
+                if (!empty($existingcalculations[$rangeindex][$rangeindicator->get_id()])) {
+                    $prevcalculations = $existingcalculations[$rangeindex][$rangeindicator->get_id()];
+                }
+
+                // Calculate the indicator for each sample in this time range.
+                list($samplesfeatures, $newindicatorcalculations, $indicatornotnulls) = $rangeindicator->calculate($sampleids,
+                    $this->analyser->get_samples_origin(), $range['start'], $range['end'], $prevcalculations);
+
+                // Copy the features data to the dataset.
+                foreach ($samplesfeatures as $analysersampleid => $features) {
+
+                    $uniquesampleid = $timesplitting->append_rangeindex($analysersampleid, $rangeindex);
+
+                    if (!isset($notnulls[$uniquesampleid]) && !empty($indicatornotnulls[$analysersampleid])) {
+                        $notnulls[$uniquesampleid] = $uniquesampleid;
+                    }
+
+                    // Init the sample if it is still empty.
+                    if (!isset($dataset[$uniquesampleid])) {
+                        $dataset[$uniquesampleid] = array();
+                    }
+
+                    // Append the features indicator features at the end of the sample.
+                    $dataset[$uniquesampleid] = array_merge($dataset[$uniquesampleid], $features);
+                }
+
+                if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations()) {
+                    $timecreated = time();
+                    foreach ($newindicatorcalculations as $sampleid => $calculatedvalue) {
+                        // Prepare the new calculations to be stored into DB.
+
+                        $indcalc = new \stdClass();
+                        $indcalc->contextid = $timesplitting->get_analysable()->get_context()->id;
+                        $indcalc->starttime = $range['start'];
+                        $indcalc->endtime = $range['end'];
+                        $indcalc->sampleid = $sampleid;
+                        $indcalc->sampleorigin = $this->analyser->get_samples_origin();
+                        $indcalc->indicator = $rangeindicator->get_id();
+                        $indcalc->value = $calculatedvalue;
+                        $indcalc->timecreated = $timecreated;
+                        $newcalculations[] = $indcalc;
+                    }
+                }
+            }
+
+            if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations()) {
+                $batchsize = self::get_insert_batch_size();
+                if (count($newcalculations) > $batchsize) {
+                    // We don't want newcalculations array to grow too much as we already keep the
+                    // system memory busy storing $dataset contents.
+
+                    // Insert from the beginning.
+                    $remaining = array_splice($newcalculations, $batchsize);
+
+                    // Sorry mssql and oracle, this will be slow.
+                    $DB->insert_records('analytics_indicator_calc', $newcalculations);
+                    $newcalculations = $remaining;
+                }
+            }
+        }
+
+        if (!$options['evaluation'] && $timesplitting->cache_indicator_calculations() && $newcalculations) {
+            // Insert the remaining records.
+            $DB->insert_records('analytics_indicator_calc', $newcalculations);
+        }
+
+        // Delete rows where all calculations are null.
+        // We still store the indicator calculation and we still store the sample id as
+        // processed so we don't have to process this sample again, but we exclude it
+        // from the dataset because it is not useful.
+        $nulls = array_diff_key($dataset, $notnulls);
+        foreach ($nulls as $uniqueid => $ignoredvalues) {
+            unset($dataset[$uniqueid]);
+        }
+
+        return $dataset;
+    }
+
+    /**
+     * Adds time range indicators and the target to each sample.
+     *
+     * This will identify the sample as belonging to a specific range.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $dataset
+     * @param array|null $calculatedtarget
+     * @return null
+     */
+    protected function fill_dataset(\core_analytics\local\time_splitting\base $timesplitting,
+            array &$dataset, ?array $calculatedtarget = null) {
+
+        $nranges = count($timesplitting->get_distinct_ranges());
+
+        foreach ($dataset as $uniquesampleid => $unmodified) {
+
+            list($analysersampleid, $rangeindex) = $timesplitting->infer_sample_info($uniquesampleid);
+
+            // No need to add range features if this time splitting method only defines one time range.
+            if ($nranges > 1) {
+
+                // 1 column for each range.
+                $timeindicators = array_fill(0, $nranges, 0);
+
+                $timeindicators[$rangeindex] = 1;
+
+                $dataset[$uniquesampleid] = array_merge($timeindicators, $dataset[$uniquesampleid]);
+            }
+
+            if ($calculatedtarget) {
+                // Add this sampleid's calculated target and the end.
+                $dataset[$uniquesampleid][] = $calculatedtarget[$analysersampleid];
+
+            } else {
+                // Add this sampleid, it will be used to identify the prediction that comes back from
+                // the predictions processor.
+                array_unshift($dataset[$uniquesampleid], $uniquesampleid);
+            }
+        }
+    }
+
+    /**
+     * Updates the analysable analysis time.
+     *
+     * @param array $processedanalysables
+     * @param int $analysableid
+     * @return null
+     */
+    protected function update_analysable_analysed_time(array $processedanalysables, int $analysableid) {
+        global $DB;
+
+        if (!empty($processedanalysables[$analysableid])) {
+            $obj = $processedanalysables[$analysableid];
+
+            $obj->id = $obj->primarykey;
+            unset($obj->primarykey);
+
+            $obj->timeanalysed = time();
+            $DB->update_record('analytics_used_analysables', $obj);
+
+        } else {
+
+            $obj = new \stdClass();
+            $obj->modelid = $this->analyser->get_modelid();
+            $obj->action = ($this->includetarget) ? 'training' : 'prediction';
+            $obj->analysableid = $analysableid;
+            $obj->timeanalysed = time();
+
+            $DB->insert_record('analytics_used_analysables', $obj);
+        }
+    }
+
+    /**
+     * Adds dataset context info.
+     *
+     * The final dataset document will look like this:
+     * ----------------------------------------------------
+     * metadata1,metadata2,metadata3,.....
+     * value1, value2, value3,.....
+     *
+     * header1,header2,header3,header4,.....
+     * stud1value1,stud1value2,stud1value3,stud1value4,.....
+     * stud2value1,stud2value2,stud2value3,stud2value4,.....
+     * .....
+     * ----------------------------------------------------
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param array $dataset
+     * @param \core_analytics\local\target\base $target
+     * @return null
+     */
+    protected function add_context_metadata(\core_analytics\local\time_splitting\base $timesplitting, array &$dataset,
+            \core_analytics\local\target\base $target) {
+        $headers = $this->get_headers($timesplitting, $target);
+
+        // This will also reset samples' dataset keys.
+        array_unshift($dataset, $headers);
+    }
+
+    /**
+     * Returns the headers for the csv file based on the indicators and the target.
+     *
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \core_analytics\local\target\base $target
+     * @return string[]
+     */
+    public function get_headers(\core_analytics\local\time_splitting\base $timesplitting,
+            \core_analytics\local\target\base $target): array {
+        // 3rd column will contain the indicator ids.
+        $headers = array();
+
+        if (!$this->includetarget) {
+            // The first column is the sampleid.
+            $headers[] = 'sampleid';
+        }
+
+        // We always have 1 column for each time splitting method range, it does not depend on how
+        // many ranges we calculated.
+        $ranges = $timesplitting->get_distinct_ranges();
+        if (count($ranges) > 1) {
+            foreach ($ranges as $rangeindex) {
+                $headers[] = 'range/' . $rangeindex;
+            }
+        }
+
+        // Model indicators.
+        foreach ($this->analyser->get_indicators() as $indicator) {
+            $headers = array_merge($headers, $indicator::get_feature_headers());
+        }
+
+        // The target as well.
+        if ($this->includetarget) {
+            $headers[] = $target->get_id();
+        }
+
+        return $headers;
+    }
+
+    /**
+     * Filters out samples that have already been used for training.
+     *
+     * @param int[] $sampleids
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @return  null
+     */
+    protected function filter_out_train_samples(array &$sampleids, \core_analytics\local\time_splitting\base $timesplitting) {
+        global $DB;
+
+        $params = array('modelid' => $this->analyser->get_modelid(), 'analysableid' => $timesplitting->get_analysable()->get_id(),
+            'timesplitting' => $timesplitting->get_id());
+
+        $trainingsamples = $DB->get_records('analytics_train_samples', $params);
+
+        // Skip each file trained samples.
+        foreach ($trainingsamples as $trainingfile) {
+
+            $usedsamples = json_decode($trainingfile->sampleids, true);
+
+            if (!empty($usedsamples)) {
+                // Reset $sampleids to $sampleids minus this file's $usedsamples.
+                $sampleids = array_diff_key($sampleids, $usedsamples);
+            }
+        }
+    }
+
+    /**
+     * Filters out samples that have already been used for prediction.
+     *
+     * @param int[] $sampleids
+     * @param array $ranges
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @return  \stdClass|null The analytics_predict_samples record or null
+     */
+    protected function filter_out_prediction_samples_and_ranges(array &$sampleids, array &$ranges,
+            \core_analytics\local\time_splitting\base $timesplitting) {
+
+        if (count($ranges) > 1) {
+            throw new \coding_exception('$ranges argument should only contain one range');
+        }
+
+        $rangeindex = key($ranges);
+        $predictedrange = $this->get_predict_samples_record($timesplitting, $rangeindex);
+
+        if (!$predictedrange) {
+            // Nothing to filter out.
+            return null;
+        }
+
+        $predictedrange->sampleids = json_decode($predictedrange->sampleids, true);
+        $missingsamples = array_diff_key($sampleids, $predictedrange->sampleids);
+        if (count($missingsamples) === 0) {
+            // All samples already calculated.
+            unset($ranges[$rangeindex]);
+            return null;
+        }
+
+        // Replace the list of samples by the one excluding samples that already got predictions at this range.
+        $sampleids = $missingsamples;
+
+        return $predictedrange;
+    }
+
+    /**
+     * Returns a predict samples record.
+     *
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  int                                       $rangeindex
+     * @return \stdClass|false
+     */
+    private function get_predict_samples_record(\core_analytics\local\time_splitting\base $timesplitting, int $rangeindex) {
+        global $DB;
+
+        $params = array('modelid' => $this->analyser->get_modelid(), 'analysableid' => $timesplitting->get_analysable()->get_id(),
+            'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex);
+        $predictedrange = $DB->get_record('analytics_predict_samples', $params);
+
+        return $predictedrange;
+    }
+
+    /**
+     * Saves samples that have just been used for training.
+     *
+     * @param int[] $sampleids
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @return null
+     */
+    protected function save_train_samples(array $sampleids, \core_analytics\local\time_splitting\base $timesplitting) {
+        global $DB;
+
+        $trainingsamples = new \stdClass();
+        $trainingsamples->modelid = $this->analyser->get_modelid();
+        $trainingsamples->analysableid = $timesplitting->get_analysable()->get_id();
+        $trainingsamples->timesplitting = $timesplitting->get_id();
+
+        $trainingsamples->sampleids = json_encode($sampleids);
+        $trainingsamples->timecreated = time();
+
+        $DB->insert_record('analytics_train_samples', $trainingsamples);
+    }
+
+    /**
+     * Saves samples that have just been used for prediction.
+     *
+     * @param int[] $sampleids
+     * @param array $ranges
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \stdClass|null $predictsamplesrecord The existing record or null if there is no record yet.
+     * @return null
+     */
+    protected function save_prediction_samples(array $sampleids, array $ranges,
+            \core_analytics\local\time_splitting\base $timesplitting, ?\stdClass $predictsamplesrecord = null) {
+        global $DB;
+
+        if (count($ranges) > 1) {
+            throw new \coding_exception('$ranges argument should only contain one range');
+        }
+
+        $rangeindex = key($ranges);
+
+        if ($predictsamplesrecord) {
+            // Append the new samples used for prediction.
+            $predictsamplesrecord->sampleids = json_encode($predictsamplesrecord->sampleids + $sampleids);
+            $predictsamplesrecord->timemodified = time();
+            $DB->update_record('analytics_predict_samples', $predictsamplesrecord);
+        } else {
+            $predictsamplesrecord = (object)[
+                'modelid' => $this->analyser->get_modelid(),
+                'analysableid' => $timesplitting->get_analysable()->get_id(),
+                'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex
+            ];
+            $predictsamplesrecord->sampleids = json_encode($sampleids);
+            $predictsamplesrecord->timecreated = time();
+            $predictsamplesrecord->timemodified = $predictsamplesrecord->timecreated;
+            $DB->insert_record('analytics_predict_samples', $predictsamplesrecord);
+        }
+    }
+
+    /**
+     * Flags the analysable element as in-analysis and stores a lock for it.
+     *
+     * @param  string $timesplittingid
+     * @param  int    $analysableid
+     * @return bool Success or not
+     */
+    private function init_analysable_analysis(string $timesplittingid, int $analysableid) {
+
+        // Do not include $this->includetarget as we don't want the same analysable to be analysed for training
+        // and prediction at the same time.
+        $lockkey = 'modelid:' . $this->analyser->get_modelid() . '-analysableid:' . $analysableid .
+            '-timesplitting:' . self::clean_time_splitting_id($timesplittingid);
+
+        // Large timeout as processes may be quite long.
+        $lockfactory = \core\lock\lock_config::get_lock_factory('core_analytics');
+
+        // If it is not ready in 10 secs skip this model + analysable + timesplittingmethod combination
+        // it will attempt it again during next cron run.
+        if (!$this->lock = $lockfactory->get_lock($lockkey, 10)) {
+            return false;
+        }
+        return true;
+    }
+
+
+    /**
+     * Remove all possibly problematic chars from the time splitting method id (id = its full class name).
+     *
+     * @param string $timesplittingid
+     * @return string
+     */
+    public static function clean_time_splitting_id($timesplittingid) {
+        $timesplittingid = str_replace('\\', '-', $timesplittingid);
+        return clean_param($timesplittingid, PARAM_ALPHANUMEXT);
+    }
+
+    /**
+     * Mark the currently analysed analysable+timesplitting as analysed.
+     *
+     * @return null
+     */
+    private function finish_analysable_analysis() {
+        $this->lock->release();
+    }
+
+    /**
+     * Returns the batch size used for insert_records.
+     *
+     * This method tries to find the best batch size without getting
+     * into dml internals. Maximum 1000 records to save memory.
+     *
+     * @return int
+     */
+    private static function get_insert_batch_size(): int {
+        global $DB;
+
+        $dbconfig = $DB->export_dbconfig();
+
+        // 500 is pgsql default so using 1000 is fine, no other db driver uses a hardcoded value.
+        if (empty($dbconfig) || empty($dbconfig->dboptions) || empty($dbconfig->dboptions['bulkinsertsize'])) {
+            return 1000;
+        }
+
+        $bulkinsert = $dbconfig->dboptions['bulkinsertsize'];
+        if ($bulkinsert < 1000) {
+            return $bulkinsert;
+        }
+
+        while ($bulkinsert > 1000) {
+            $bulkinsert = round($bulkinsert / 2, 0);
+        }
+
+        return (int)$bulkinsert;
+    }
+}

analytics/classes/course.php

@@ -132,12 +132,11 @@ class course implements \core_analytics\analysable {
      * Use self::instance() instead to get cached copies of the course. Instances obtained
      * through this constructor will not be cached.
      *
-     * Lazy load of course data, students and teachers.
-     *
-     * @param int|\stdClass $course Course id
+     * @param int|\stdClass $course Course id or mdl_course record
+     * @param \context|null $context
      * @return void
      */
-    public function __construct($course) {
+    public function __construct($course, ?\context $context = null) {
 
         if (is_scalar($course)) {
             $this->course = new \stdClass();
@@ -145,6 +144,10 @@ class course implements \core_analytics\analysable {
         } else {
             $this->course = $course;
         }
+
+        if (!is_null($context)) {
+            $this->coursecontext = $context;
+        }
     }
 
     /**
@@ -153,9 +156,10 @@ class course implements \core_analytics\analysable {
      * Lazy load of course data, students and teachers.
      *
      * @param int|\stdClass $course Course object or course id
+     * @param \context|null $context
      * @return \core_analytics\course
      */
-    public static function instance($course) {
+    public static function instance($course, ?\context $context = null) {
 
         $courseid = $course;
         if (!is_scalar($courseid)) {
@@ -166,7 +170,7 @@ class course implements \core_analytics\analysable {
             return self::$cachedinstance;
         }
 
-        $cachedinstance = new \core_analytics\course($course);
+        $cachedinstance = new \core_analytics\course($course, $context);
         self::$cachedinstance = $cachedinstance;
         self::$cachedid = (int)$courseid;
         return self::$cachedinstance;

analytics/classes/dataset_manager.php

@@ -109,31 +109,8 @@ class dataset_manager {
         $this->modelid = $modelid;
         $this->analysableid = $analysableid;
         $this->timesplittingid = $timesplittingid;
-        $this->evaluation = $evaluation;
         $this->filearea = $filearea;
-    }
-
-    /**
-     * Mark the analysable as being analysed.
-     *
-     * @return bool Could we get the lock or not.
-     */
-    public function init_process() {
-
-        // Do not include $this->includetarget as we don't want the same analysable to be analysed for training
-        // and prediction at the same time.
-        $lockkey = 'modelid:' . $this->modelid . '-analysableid:' . $this->analysableid .
-            '-timesplitting:' . self::clean_time_splitting_id($this->timesplittingid);
-
-        // Large timeout as processes may be quite long.
-        $lockfactory = \core\lock\lock_config::get_lock_factory('core_analytics');
-
-        // If it is not ready in 10 secs skip this model + analysable + timesplittingmethod combination
-        // it will attempt it again during next cron run.
-        if (!$this->lock = $lockfactory->get_lock($lockkey, 10)) {
-            return false;
-        }
-        return true;
+        $this->evaluation = $evaluation;
     }
 
     /**
@@ -152,7 +129,8 @@ class dataset_manager {
             'filearea' => $this->filearea,
             'itemid' => $this->modelid,
             'contextid' => \context_system::instance()->id,
-            'filepath' => '/analysable/' . $this->analysableid . '/' . self::clean_time_splitting_id($this->timesplittingid) . '/',
+            'filepath' => '/analysable/' . $this->analysableid . '/' .
+                \core_analytics\analysis::clean_time_splitting_id($this->timesplittingid) . '/',
             'filename' => self::get_filename($this->evaluation)
         ];
 
@@ -167,8 +145,7 @@ class dataset_manager {
         $filepath = make_request_directory() . DIRECTORY_SEPARATOR . $filerecord['filename'];
         $fh = fopen($filepath, 'w+');
         if (!$fh) {
-            $this->close_process();
-            throw new \moodle_exception('errorcannotwritedataset', 'analytics', '', $filepath);
+            return false;
         }
         foreach ($data as $line) {
             fputcsv($fh, $line);
@@ -178,15 +155,6 @@ class dataset_manager {
         return $fs->create_file_from_pathname($filerecord, $filepath);
     }
 
-    /**
-     * Mark as analysed.
-     *
-     * @return void
-     */
-    public function close_process() {
-        $this->lock->release();
-    }
-
     /**
      * Returns the previous evaluation file.
      *
@@ -200,8 +168,9 @@ class dataset_manager {
     public static function get_previous_evaluation_file($modelid, $timesplittingid) {
         $fs = get_file_storage();
         // Evaluation data is always labelled.
+        $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
         return $fs->get_file(\context_system::instance()->id, 'analytics', self::LABELLED_FILEAREA, $modelid,
-            '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/', self::EVALUATION_FILENAME);
+            $filepath, self::EVALUATION_FILENAME);
     }
 
     /**
@@ -233,7 +202,7 @@ class dataset_manager {
         $filesbytimesplitting = array();
         foreach ($timesplittingids as $timesplittingid) {
 
-            $filepath = '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/';
+            $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
             $files = $fs->get_directory_files(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath);
             foreach ($files as $file) {
 
@@ -291,7 +260,8 @@ class dataset_manager {
         // Always evaluation.csv and labelled as it is an evaluation file.
         $filearea = self::LABELLED_FILEAREA;
         $filename = self::get_filename(true);
-        $filepath = '/analysable/' . $analysableid . '/' . self::clean_time_splitting_id($timesplittingid) . '/';
+        $filepath = '/analysable/' . $analysableid . '/' .
+            \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
         return $fs->get_file(\context_system::instance()->id, 'analytics', $filearea, $modelid, $filepath, $filename);
     }
 
@@ -372,7 +342,7 @@ class dataset_manager {
             'filearea' => $filearea,
             'itemid' => $modelid,
             'contextid' => \context_system::instance()->id,
-            'filepath' => '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/',
+            'filepath' => '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/',
             'filename' => self::get_filename($evaluation)
         ];
 
@@ -393,7 +363,7 @@ class dataset_manager {
         $fs = get_file_storage();
 
         $contextid = \context_system::instance()->id;
-        $filepath = '/timesplitting/' . self::clean_time_splitting_id($timesplittingid) . '/';
+        $filepath = '/timesplitting/' . \core_analytics\analysis::clean_time_splitting_id($timesplittingid) . '/';
 
         $files = $fs->get_directory_files($contextid, 'analytics', self::LABELLED_FILEAREA, $modelid,
             $filepath, true, false);
@@ -467,17 +437,6 @@ class dataset_manager {
         return $fs->delete_area_files(\context_system::instance()->id, 'analytics', false, $modelid);
     }
 
-    /**
-     * Remove all possibly problematic chars from the time splitting method id (id = its full class name).
-     *
-     * @param string $timesplittingid
-     * @return string
-     */
-    protected static function clean_time_splitting_id($timesplittingid) {
-        $timesplittingid = str_replace('\\', '-', $timesplittingid);
-        return clean_param($timesplittingid, PARAM_ALPHANUMEXT);
-    }
-
     /**
      * Returns the file name to be used.
      *

analytics/classes/insights_generator.php

@@ -0,0 +1,236 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Insights generator.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics;
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once($CFG->dirroot . '/lib/messagelib.php');
+
+/**
+ * Insights generator.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class insights_generator {
+
+    /**
+     * @var int
+     */
+    private $modelid;
+
+    /**
+     * @var \core_analytics\local\target\base
+     */
+    private $target;
+
+    /**
+     * @var int[]
+     */
+    private $contextcourseids;
+
+    /**
+     * Constructor.
+     *
+     * @param int $modelid
+     * @param \core_analytics\local\target\base $target
+     */
+    public function __construct(int $modelid, \core_analytics\local\target\base $target) {
+        $this->modelid = $modelid;
+        $this->target = $target;
+    }
+
+    /**
+     * Generates insight notifications.
+     *
+     * @param array                         $samplecontexts    The contexts these predictions belong to
+     * @param \core_analytics\prediction[]  $predictions       The prediction records
+     * @return  null
+     */
+    public function generate($samplecontexts, $predictions) {
+        global $OUTPUT;
+
+        $analyserclass = $this->target->get_analyser_class();
+
+        // We will need to restore it later.
+        $actuallanguage = current_language();
+
+        if ($analyserclass::one_sample_per_analysable()) {
+
+            // Iterate through the predictions and the users in each prediction (likely to be just one).
+            foreach ($predictions as $prediction) {
+
+                $context = $samplecontexts[$prediction->get_prediction_data()->contextid];
+
+                $users = $this->target->get_insights_users($context);
+                foreach ($users as $user) {
+
+                    $this->set_notification_language($user);
+                    list($insighturl, $fullmessage, $fullmessagehtml) = $this->prediction_info($prediction);
+                    $this->notification($context, $user, $insighturl, $fullmessage, $fullmessagehtml);
+                }
+            }
+
+        } else {
+
+            // Iterate through the context and the users in each context.
+            foreach ($samplecontexts as $context) {
+
+                $users = $this->target->get_insights_users($context);
+                foreach ($users as $user) {
+
+                    $this->set_notification_language($user);
+
+                    $insighturl = $this->target->get_insight_context_url($this->modelid, $context);
+
+                    $fullmessage = get_string('insightinfomessage', 'analytics', $insighturl->out(false));
+                    $fullmessagehtml = $OUTPUT->render_from_template('core_analytics/insight_info_message',
+                        ['url' => $insighturl->out(false)]
+                    );
+
+                    $this->notification($context, $user, $insighturl, $fullmessage, $fullmessagehtml);
+                }
+            }
+        }
+
+        force_current_language($actuallanguage);
+    }
+
+    /**
+     * Generates a insight notification for the user.
+     *
+     * @param  \context    $context
+     * @param  \stdClass   $user
+     * @param  \moodle_url $insighturl    The insight URL
+     * @param  string      $fullmessage
+     * @param  string      $fullmessagehtml
+     * @return null
+     */
+    private function notification(\context $context, \stdClass $user, \moodle_url $insighturl, string $fullmessage, string $fullmessagehtml) {
+
+        $message = new \core\message\message();
+        $message->component = 'moodle';
+        $message->name = 'insights';
+
+        $message->userfrom = \core_user::get_noreply_user();
+        $message->userto = $user;
+
+        $message->subject = $this->target->get_insight_subject($this->modelid, $context);
+
+        // Same than the subject.
+        $message->contexturlname = $message->subject;
+        $message->courseid = $this->get_context_courseid($context);
+
+        $message->fullmessage = $fullmessage;
+        $message->fullmessageformat = FORMAT_PLAIN;
+        $message->fullmessagehtml = $fullmessagehtml;
+        $message->smallmessage = $fullmessage;
+        $message->contexturl = $insighturl->out(false);
+
+        message_send($message);
+    }
+
+    /**
+     * Returns the course context of the provided context reading an internal cache first.
+     *
+     * @param  \context $context
+     * @return int
+     */
+    private function get_context_courseid(\context $context) {
+
+        if (empty($this->contextcourseids[$context->id])) {
+
+            $coursecontext = $context->get_course_context(false);
+            if (!$coursecontext) {
+                // Default to the frontpage course context.
+                $coursecontext = \context_course::instance(SITEID);
+            }
+            $this->contextcourseids[$context->id] = $coursecontext->instanceid;
+        }
+
+        return $this->contextcourseids[$context->id];
+    }
+
+    /**
+     * Extracts info from the prediction for display purposes.
+     *
+     * @param  \core_analytics\prediction $prediction
+     * @return array Three items array with formats [\moodle_url, string, string]
+     */
+    private function prediction_info(\core_analytics\prediction $prediction) {
+        global $OUTPUT;
+
+        $predictionactions = $this->target->prediction_actions($prediction, true, true);
+
+        // For FORMAT_PLAIN.
+        $fullmessageplaintext  = '';
+
+        // For FORMAT_HTML.
+        $messageactions  = [];
+        $insighturl = null;
+        foreach ($predictionactions as $action) {
+            $actionurl = $action->get_url();
+            if (!$actionurl->get_param('forwardurl')) {
+
+                $actiondoneurl = new \moodle_url('/report/insights/done.php');
+                // Set the forward url to the 'done' script.
+                $actionurl->param('forwardurl', $actiondoneurl->out(false));
+            }
+
+            if (empty($insighturl)) {
+                // We use the primary action url as insight url so we log that the user followed the provided link.
+                $insighturl = $action->get_url();
+            }
+            $actiondata = (object)['url' => $action->get_url()->out(false), 'text' => $action->get_text()];
+            $fullmessageplaintext .= get_string('insightinfomessageaction', 'analytics', $actiondata) . PHP_EOL;
+            $messageactions[] = $actiondata;
+        }
+
+        $fullmessagehtml = $OUTPUT->render_from_template('core_analytics/insight_info_message_prediction',
+            ['actions' => $messageactions]);
+        return [$insighturl, $fullmessageplaintext, $fullmessagehtml];
+    }
+
+    /**
+     * Sets the session language to the language used by the notification receiver.
+     *
+     * @param  \stdClass $user The user who will receive the message
+     * @return null
+     */
+    private function set_notification_language($user) {
+        global $CFG;
+
+        // Copied from current_language().
+        if (!empty($user->lang)) {
+            $lang = $user->lang;
+        } else if (isset($CFG->lang)) {
+            $lang = $CFG->lang;
+        } else {
+            $lang = 'en';
+        }
+        force_current_language($lang);
+    }
+}

analytics/classes/local/analyser/base.php

@@ -47,13 +47,6 @@ abstract class base {
      */
     protected $target;
 
-    /**
-     * A $this->$target copy loaded with the ongoing analysis analysable.
-     *
-     * @var \core_analytics\local\target\base
-     */
-    protected $analysabletarget;
-
     /**
      * The model indicators.
      *
@@ -118,9 +111,37 @@ abstract class base {
      * \core_analytics\local\analyser\by_course and \core_analytics\local\analyser\sitewide are implementing
      * this method returning site courses (by_course) and the whole system (sitewide) as analysables.
      *
+     * @todo MDL-65284 This will be removed in Moodle 4.1
+     * @deprecated
+     * @see get_analysables_iterator
+     * @throws  \coding_exception
      * @return \core_analytics\analysable[] Array of analysable elements using the analysable id as array key.
      */
-    abstract public function get_analysables();
+    public function get_analysables() {
+        // This function should only be called from get_analysables_iterator and we keep it here until Moodle 4.1
+        // for backwards compatibility.
+        throw new \coding_exception('This method is deprecated in favour of get_analysables_iterator.');
+    }
+
+    /**
+     * Returns the list of analysable elements available on the site.
+     *
+     * A relatively complex SQL query should be set so that we take into account which analysable elements
+     * have already been processed and the order in which they have been processed. Helper methods are available
+     * to ease to implementation of get_analysables_iterator: get_iterator_sql and order_sql.
+     *
+     * @param string|null $action 'prediction', 'training' or null if no specific action needed.
+     * @return \Iterator
+     */
+    public function get_analysables_iterator(?string $action = null) {
+
+        debugging('Please overwrite get_analysables_iterator with your own implementation, we only keep this default
+            implementation for backwards compatibility purposes with get_analysables(). note that $action param will
+            be ignored so the analysable elements will be processed using get_analysables order, regardless of the
+            last time they were processed.');
+
+        return new \ArrayIterator($this->get_analysables());
+    }
 
     /**
      * This function returns this analysable list of samples.
@@ -128,7 +149,7 @@ abstract class base {
      * @param \core_analytics\analysable $analysable
      * @return array array[0] = int[] (sampleids) and array[1] = array (samplesdata)
      */
-    abstract protected function get_all_samples(\core_analytics\analysable $analysable);
+    abstract public function get_all_samples(\core_analytics\analysable $analysable);
 
     /**
      * This function returns the samples data from a list of sample ids.
@@ -174,70 +195,46 @@ abstract class base {
     abstract public function sample_description($sampleid, $contextid, $sampledata);
 
     /**
-     * Main analyser method which processes the site analysables.
-     *
-     * @param bool $includetarget
-     * @return \stored_file[]
+     * Model id getter.
+     * @return int
      */
-    public function get_analysable_data($includetarget) {
-        global $DB;
+    public function get_modelid(): int {
+        return $this->modelid;
+    }
 
-        // Time limit control.
-        $modeltimelimit = intval(get_config('analytics', 'modeltimelimit'));
+    /**
+     * Options getter.
+     * @return array
+     */
+    public function get_options(): array {
+        return $this->options;
+    }
 
-        $filesbytimesplitting = array();
+    /**
+     * Returns the analysed target.
+     *
+     * @return \core_analytics\local\target\base
+     */
+    public function get_target(): \core_analytics\local\target\base {
+        return $this->target;
+    }
 
-        list($analysables, $processedanalysables) = $this->get_sorted_analysables($includetarget);
+    /**
+     * Getter for time splittings.
+     *
+     * @return \core_analytics\local\time_splitting\base
+     */
+    public function get_timesplittings(): array {
+        return $this->timesplittings;
+    }
 
-        $inittime = time();
-        foreach ($analysables as $key => $analysable) {
-
-            $files = $this->process_analysable($analysable, $includetarget);
-
-            // Later we will need to aggregate data by time splitting method.
-            foreach ($files as $timesplittingid => $file) {
-                $filesbytimesplitting[$timesplittingid][] = $file;
-            }
-
-            $this->update_analysable_analysed_time($processedanalysables, $analysable->get_id(), $includetarget);
-
-            // Apply time limit.
-            if (!$this->options['evaluation']) {
-                $timespent = time() - $inittime;
-                if ($modeltimelimit <= $timespent) {
-                    break;
-                }
-            }
-
-            unset($analysables[$key]);
-        }
-
-        if ($this->options['evaluation'] === false) {
-            // Look for previous training and prediction files we generated and couldn't be used
-            // by machine learning backends because they weren't big enough.
-
-            $pendingfiles = \core_analytics\dataset_manager::get_pending_files($this->modelid, $includetarget,
-                array_keys($filesbytimesplitting));
-            foreach ($pendingfiles as $timesplittingid => $files) {
-                foreach ($files as $file) {
-                    $filesbytimesplitting[$timesplittingid][] = $file;
-                }
-            }
-        }
-
-        // We join the datasets by time splitting method.
-        $timesplittingfiles = $this->merge_analysable_files($filesbytimesplitting, $includetarget);
-
-        if (!empty($pendingfiles)) {
-            // We must remove them now as they are already part of another dataset.
-            foreach ($pendingfiles as $timesplittingid => $files) {
-                foreach ($files as $file) {
-                    $file->delete();
-                }
-            }
-        }
-
-        return $timesplittingfiles;
+    /**
+     * Getter for indicators.
+     *
+     * @return \core_analytics\local\indicator\base
+     */
+    public function get_indicators(): array {
+        return $this->indicators;
     }
 
     /**
@@ -252,19 +249,39 @@ abstract class base {
     /**
      * Returns labelled data (training and evaluation).
      *
-     * @return array
+     * @return \stored_file[]
      */
     public function get_labelled_data() {
-        return $this->get_analysable_data(true);
+        // Delegates all processing to the analysis.
+        $result = new \core_analytics\local\analysis\result_file($this->get_modelid(), true, $this->get_options());
+        $analysis = new \core_analytics\analysis($this, true, $result);
+        $analysis->run();
+        return $result->get();
     }
 
     /**
      * Returns unlabelled data (prediction).
      *
-     * @return array
+     * @return \stored_file[]
      */
     public function get_unlabelled_data() {
-        return $this->get_analysable_data(false);
+        // Delegates all processing to the analysis.
+        $result = new \core_analytics\local\analysis\result_file($this->get_modelid(), false, $this->get_options());
+        $analysis = new \core_analytics\analysis($this, false, $result);
+        $analysis->run();
+        return $result->get();
+    }
+
+    /**
+     * Returns indicator calculations as an array.
+     * @return array
+     */
+    public function get_static_data() {
+        // Delegates all processing to the analysis.
+        $result = new \core_analytics\local\analysis\result_array($this->get_modelid(), false, $this->get_options());
+        $analysis = new \core_analytics\analysis($this, false, $result);
+        $analysis->run();
+        return $result->get();
     }
 
     /**
@@ -284,36 +301,6 @@ abstract class base {
         }
     }
 
-    /**
-     * Merges analysable dataset files into 1.
-     *
-     * @param array $filesbytimesplitting
-     * @param bool $includetarget
-     * @return \stored_file[]
-     */
-    protected function merge_analysable_files($filesbytimesplitting, $includetarget) {
-
-        $timesplittingfiles = array();
-        foreach ($filesbytimesplitting as $timesplittingid => $files) {
-
-            if ($this->options['evaluation'] === true) {
-                // Delete the previous copy. Only when evaluating.
-                \core_analytics\dataset_manager::delete_previous_evaluation_file($this->modelid, $timesplittingid);
-            }
-
-            // Merge all course files into one.
-            if ($includetarget) {
-                $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
-            } else {
-                $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
-            }
-            $timesplittingfiles[$timesplittingid] = \core_analytics\dataset_manager::merge_datasets($files,
-                $this->modelid, $timesplittingid, $filearea, $this->options['evaluation']);
-        }
-
-        return $timesplittingfiles;
-    }
-
     /**
      * Checks that this analyser satisfies the provided indicator requirements.
      *
@@ -338,83 +325,6 @@ abstract class base {
         return $missingrequired;
     }
 
-    /**
-     * Processes an analysable
-     *
-     * This method returns the general analysable status, an array of files by time splitting method and
-     * an error message if there is any problem.
-     *
-     * @param \core_analytics\analysable $analysable
-     * @param bool $includetarget
-     * @return \stored_file[] Files by time splitting method
-     */
-    public function process_analysable($analysable, $includetarget) {
-
-        // Default returns.
-        $files = array();
-        $message = null;
-
-        // Target instances scope is per-analysable (it can't be lower as calculations run once per
-        // analysable, not time splitting method nor time range).
-        $this->analysabletarget = call_user_func(array($this->target, 'instance'));
-
-        // We need to check that the analysable is valid for the target even if we don't include targets
-        // as we still need to discard invalid analysables for the target.
-        $result = $this->analysabletarget->is_valid_analysable($analysable, $includetarget);
-        if ($result !== true) {
-            $a = new \stdClass();
-            $a->analysableid = $analysable->get_name();
-            $a->result = $result;
-            $this->add_log(get_string('analysablenotvalidfortarget', 'analytics', $a));
-            return array();
-        }
-
-        // Process all provided time splitting methods.
-        $results = array();
-        foreach ($this->timesplittings as $timesplitting) {
-
-            // For evaluation purposes we don't need to be that strict about how updated the data is,
-            // if this analyser was analysed less that 1 week ago we skip generating a new one. This
-            // helps scale the evaluation process as sites with tons of courses may a lot of time to
-            // complete an evaluation.
-            if (!empty($this->options['evaluation']) && !empty($this->options['reuseprevanalysed'])) {
-
-                $previousanalysis = \core_analytics\dataset_manager::get_evaluation_analysable_file($this->modelid,
-                    $analysable->get_id(), $timesplitting->get_id());
-                // 1 week is a partly random time interval, no need to worry about DST.
-                $boundary = time() - WEEKSECS;
-                if ($previousanalysis && $previousanalysis->get_timecreated() > $boundary) {
-                    // Recover the previous analysed file and avoid generating a new one.
-
-                    // Don't bother filling a result object as it is only useful when there are no files generated.
-                    $files[$timesplitting->get_id()] = $previousanalysis;
-                    continue;
-                }
-            }
-
-            $result = $this->process_time_splitting($timesplitting, $analysable, $includetarget);
-
-            if (!empty($result->file)) {
-                $files[$timesplitting->get_id()] = $result->file;
-            }
-            $results[] = $result;
-        }
-
-        if (empty($files)) {
-            $errors = array();
-            foreach ($results as $timesplittingid => $result) {
-                $errors[] = $timesplittingid . ': ' . $result->message;
-            }
-
-            $a = new \stdClass();
-            $a->analysableid = $analysable->get_name();
-            $a->errors = implode(', ', $errors);
-            $this->add_log(get_string('analysablenotused', 'analytics', $a));
-        }
-
-        return $files;
-    }
-
     /**
      * Adds a register to the analysis log.
      *
@@ -482,427 +392,83 @@ abstract class base {
     }
 
     /**
-     * Processes the analysable samples using the provided time splitting method.
+     * Do this analyser's analysables have 1 single sample each?
      *
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @param \core_analytics\analysable $analysable
-     * @param bool $includetarget
-     * @return \stdClass Results object.
+     * Overwrite and return true if your analysables only have
+     * one sample. The insights generated by models using this
+     * analyser will then include the suggested actions in the
+     * notification.
+     *
+     * @return bool
      */
-    protected function process_time_splitting($timesplitting, $analysable, $includetarget = false) {
-
-        $result = new \stdClass();
-
-        if (!$timesplitting->is_valid_analysable($analysable)) {
-            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-            $result->message = get_string('invalidanalysablefortimesplitting', 'analytics',
-                $timesplitting->get_name());
-            return $result;
-        }
-        $timesplitting->set_analysable($analysable);
-
-        if (CLI_SCRIPT && !PHPUNIT_TEST) {
-            mtrace('Analysing id "' . $analysable->get_id() . '" with "' . $timesplitting->get_name() .
-                '" time splitting method...');
-        }
-
-        // What is a sample is defined by the analyser, it can be an enrolment, a course, a user, a question
-        // attempt... it is on what we will base indicators calculations.
-        list($sampleids, $samplesdata) = $this->get_all_samples($analysable);
-
-        if (count($sampleids) === 0) {
-            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-            $result->message = get_string('nodata', 'analytics');
-            return $result;
-        }
-
-        if ($includetarget) {
-            // All ranges are used when we are calculating data for training.
-            $ranges = $timesplitting->get_all_ranges();
-        } else {
-            // The latest range that has not yet been used for prediction (it depends on the time range where we are right now).
-            $ranges = $this->get_most_recent_prediction_range($timesplitting);
-        }
-
-        // There is no need to keep track of the evaluated samples and ranges as we always evaluate the whole dataset.
-        if ($this->options['evaluation'] === false) {
-
-            if (empty($ranges)) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('noranges', 'analytics');
-                return $result;
-            }
-
-            // We skip all samples that are already part of a training dataset, even if they have not been used for prediction.
-            $this->filter_out_train_samples($sampleids, $timesplitting);
-
-            if (count($sampleids) === 0) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('nonewdata', 'analytics');
-                return $result;
-            }
-
-            // Only when processing data for predictions.
-            if (!$includetarget) {
-                // We also filter out samples and ranges that have already been used for predictions.
-                $this->filter_out_prediction_samples_and_ranges($sampleids, $ranges, $timesplitting);
-            }
-
-            if (count($sampleids) === 0) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('nonewdata', 'analytics');
-                return $result;
-            }
-
-            if (count($ranges) === 0) {
-                $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-                $result->message = get_string('nonewranges', 'analytics');
-                return $result;
-            }
-        }
-
-        if (!empty($includetarget)) {
-            $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
-        } else {
-            $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
-        }
-        $dataset = new \core_analytics\dataset_manager($this->modelid, $analysable->get_id(), $timesplitting->get_id(),
-            $filearea, $this->options['evaluation']);
-
-        // Flag the model + analysable + timesplitting as being analysed (prevent concurrent executions).
-        if (!$dataset->init_process()) {
-            // If this model + analysable + timesplitting combination is being analysed we skip this process.
-            $result->status = \core_analytics\model::NO_DATASET;
-            $result->message = get_string('analysisinprogress', 'analytics');
-            return $result;
-        }
-
-        try {
-            // Remove samples the target consider invalid.
-            $this->analysabletarget->add_sample_data($samplesdata);
-            $this->analysabletarget->filter_out_invalid_samples($sampleids, $analysable, $includetarget);
-        } catch (\Throwable $e) {
-            $dataset->close_process();
-            throw $e;
-        }
-
-        if (!$sampleids) {
-            $result->status = \core_analytics\model::NO_DATASET;
-            $result->message = get_string('novalidsamples', 'analytics');
-            $dataset->close_process();
-            return $result;
-        }
-
-        try {
-            foreach ($this->indicators as $key => $indicator) {
-                // The analyser attaches the main entities the sample depends on and are provided to the
-                // indicator to calculate the sample.
-                $this->indicators[$key]->add_sample_data($samplesdata);
-            }
-
-            // Here we start the memory intensive process that will last until $data var is
-            // unset (until the method is finished basically).
-            if ($includetarget) {
-                $data = $timesplitting->calculate($sampleids, $this->get_samples_origin(), $this->indicators, $ranges,
-                    $this->analysabletarget);
-            } else {
-                $data = $timesplitting->calculate($sampleids, $this->get_samples_origin(), $this->indicators, $ranges);
-            }
-        } catch (\Throwable $e) {
-            $dataset->close_process();
-            throw $e;
-        }
-
-        if (!$data) {
-            $result->status = \core_analytics\model::ANALYSABLE_REJECTED_TIME_SPLITTING_METHOD;
-            $result->message = get_string('novaliddata', 'analytics');
-            $dataset->close_process();
-            return $result;
-        }
-
-        try {
-            // Add extra metadata.
-            $this->add_model_metadata($data);
-
-            // Write all calculated data to a file.
-            $file = $dataset->store($data);
-        } catch (\Throwable $e) {
-            $dataset->close_process();
-            throw $e;
-        }
-
-        // Flag the model + analysable + timesplitting as analysed.
-        $dataset->close_process();
-
-        // No need to keep track of analysed stuff when evaluating.
-        if ($this->options['evaluation'] === false) {
-            // Save the samples that have been already analysed so they are not analysed again in future.
-
-            if ($includetarget) {
-                $this->save_train_samples($sampleids, $timesplitting, $file);
-            } else {
-                $this->save_prediction_samples($sampleids, $ranges, $timesplitting);
-            }
-        }
-
-        $result->status = \core_analytics\model::OK;
-        $result->message = get_string('successfullyanalysed', 'analytics');
-        $result->file = $file;
-        return $result;
+    public static function one_sample_per_analysable() {
+        return false;
     }
 
     /**
-     * Returns the most recent range that can be used to predict.
+     * Get the sql of a default implementation of the iterator.
      *
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @return array
+     * This method only works for analysers that return analysable elements which ids map to a context instance ids.
+     *
+     * @param  string      $tablename    The name of the table
+     * @param  int         $contextlevel The context level of the analysable
+     * @param  string|null $action
+     * @param  string|null $tablealias   The table alias
+     * @return array                     [0] => sql and [1] => params array
      */
-    protected function get_most_recent_prediction_range($timesplitting) {
+    protected function get_iterator_sql(string $tablename, int $contextlevel, ?string $action = null, ?string $tablealias = null) {
 
-        $now = time();
-        $ranges = $timesplitting->get_all_ranges();
-
-        // Opposite order as we are interested in the last range that can be used for prediction.
-        krsort($ranges);
-
-        // We already provided the analysable to the time splitting method, there is no need to feed it back.
-        foreach ($ranges as $rangeindex => $range) {
-            if ($timesplitting->ready_to_predict($range)) {
-                // We need to maintain the same indexes.
-                return array($rangeindex => $range);
-            }
+        if (!$tablealias) {
+            $tablealias = 'analysable';
         }
 
-        return array();
+        $params = ['contextlevel' => $contextlevel, 'modelid' => $this->get_modelid()];
+        $select = $tablealias . '.*, ' . \context_helper::get_preload_record_columns_sql('ctx');
+
+        // We add the action filter on ON instead of on WHERE because otherwise records are not returned if there are existing
+        // records for another action or model.
+        $usedanalysablesjoin = ' LEFT JOIN {analytics_used_analysables} aua ON ' . $tablealias . '.id = aua.analysableid AND ' .
+            '(aua.modelid = :modelid OR aua.modelid IS NULL)';
+
+        if ($action) {
+            $usedanalysablesjoin .= " AND aua.action = :action";
+            $params = $params + ['action' => $action];
+        }
+
+        // Adding the 1 = 1 just to have the WHERE part so that all further conditions added by callers can be
+        // appended to $sql with and ' AND'.
+        $sql = 'SELECT ' . $select . '
+                  FROM {' . $tablename . '} ' . $tablealias . '
+                  ' . $usedanalysablesjoin . '
+                  JOIN {context} ctx ON (ctx.contextlevel = :contextlevel AND ctx.instanceid = ' . $tablealias . '.id)
+                  WHERE 1 = 1';
+
+        return [$sql, $params];
     }
 
     /**
-     * Filters out samples that have already been used for training.
+     * Returns the order by clause.
      *
-     * @param int[] $sampleids
-     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param  string|null $fieldname  The field name
+     * @param  string      $order      'ASC' or 'DESC'
+     * @param  string|null $tablealias The table alias of the field
+     * @return string
      */
-    protected function filter_out_train_samples(&$sampleids, $timesplitting) {
-        global $DB;
+    protected function order_sql(?string $fieldname = null, string $order = 'ASC', ?string $tablealias = null) {
 
-        $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(),
-            'timesplitting' => $timesplitting->get_id());
-
-        $trainingsamples = $DB->get_records('analytics_train_samples', $params);
-
-        // Skip each file trained samples.
-        foreach ($trainingsamples as $trainingfile) {
-
-            $usedsamples = json_decode($trainingfile->sampleids, true);
-
-            if (!empty($usedsamples)) {
-                // Reset $sampleids to $sampleids minus this file's $usedsamples.
-                $sampleids = array_diff_key($sampleids, $usedsamples);
-            }
-        }
-    }
-
-    /**
-     * Filters out samples that have already been used for prediction.
-     *
-     * @param int[] $sampleids
-     * @param array $ranges
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     */
-    protected function filter_out_prediction_samples_and_ranges(&$sampleids, &$ranges, $timesplitting) {
-        global $DB;
-
-        if (count($ranges) > 1) {
-            throw new \coding_exception('$ranges argument should only contain one range');
+        if (!$tablealias) {
+            $tablealias = 'analysable';
         }
 
-        $rangeindex = key($ranges);
-
-        $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(),
-            'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex);
-        $predictedrange = $DB->get_record('analytics_predict_samples', $params);
-
-        if (!$predictedrange) {
-            // Nothing to filter out.
-            return;
+        if ($order != 'ASC' && $order != 'DESC') {
+            throw new \coding_exception('The order can only be ASC or DESC');
         }
 
-        $predictedrange->sampleids = json_decode($predictedrange->sampleids, true);
-        $missingsamples = array_diff_key($sampleids, $predictedrange->sampleids);
-        if (count($missingsamples) === 0) {
-            // All samples already calculated.
-            unset($ranges[$rangeindex]);
-            return;
+        $ordersql = ' ORDER BY (CASE WHEN aua.timeanalysed IS NULL THEN 0 ELSE aua.timeanalysed END) ASC';
+        if ($fieldname) {
+            $ordersql .= ', ' . $tablealias . '.' . $fieldname .' ' . $order;
         }
 
-        // Replace the list of samples by the one excluding samples that already got predictions at this range.
-        $sampleids = $missingsamples;
-    }
-
-    /**
-     * Saves samples that have just been used for training.
-     *
-     * @param int[] $sampleids
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @param \stored_file $file
-     * @return void
-     */
-    protected function save_train_samples($sampleids, $timesplitting, $file) {
-        global $DB;
-
-        $trainingsamples = new \stdClass();
-        $trainingsamples->modelid = $this->modelid;
-        $trainingsamples->analysableid = $timesplitting->get_analysable()->get_id();
-        $trainingsamples->timesplitting = $timesplitting->get_id();
-        $trainingsamples->fileid = $file->get_id();
-
-        $trainingsamples->sampleids = json_encode($sampleids);
-        $trainingsamples->timecreated = time();
-
-        $DB->insert_record('analytics_train_samples', $trainingsamples);
-    }
-
-    /**
-     * Saves samples that have just been used for prediction.
-     *
-     * @param int[] $sampleids
-     * @param array $ranges
-     * @param \core_analytics\local\time_splitting\base $timesplitting
-     * @return void
-     */
-    protected function save_prediction_samples($sampleids, $ranges, $timesplitting) {
-        global $DB;
-
-        if (count($ranges) > 1) {
-            throw new \coding_exception('$ranges argument should only contain one range');
-        }
-
-        $rangeindex = key($ranges);
-
-        $params = array('modelid' => $this->modelid, 'analysableid' => $timesplitting->get_analysable()->get_id(),
-            'timesplitting' => $timesplitting->get_id(), 'rangeindex' => $rangeindex);
-        if ($predictionrange = $DB->get_record('analytics_predict_samples', $params)) {
-            // Append the new samples used for prediction.
-            $prevsamples = json_decode($predictionrange->sampleids, true);
-            $predictionrange->sampleids = json_encode($prevsamples + $sampleids);
-            $predictionrange->timemodified = time();
-            $DB->update_record('analytics_predict_samples', $predictionrange);
-        } else {
-            $predictionrange = (object)$params;
-            $predictionrange->sampleids = json_encode($sampleids);
-            $predictionrange->timecreated = time();
-            $predictionrange->timemodified = $predictionrange->timecreated;
-            $DB->insert_record('analytics_predict_samples', $predictionrange);
-        }
-    }
-
-    /**
-     * Adds target metadata to the dataset.
-     *
-     * @param array $data
-     * @return void
-     */
-    protected function add_model_metadata(&$data) {
-        global $CFG;
-
-        $metadata = array(
-            'moodleversion' => $CFG->version,
-            'targetcolumn' => $this->analysabletarget->get_id()
-        );
-        if ($this->analysabletarget->is_linear()) {
-            $metadata['targettype'] = 'linear';
-            $metadata['targetmin'] = $this->analysabletarget::get_min_value();
-            $metadata['targetmax'] = $this->analysabletarget::get_max_value();
-        } else {
-            $metadata['targettype'] = 'discrete';
-            $metadata['targetclasses'] = json_encode($this->analysabletarget::get_classes());
-        }
-
-        foreach ($metadata as $varname => $value) {
-            $data[0][] = $varname;
-            $data[1][] = $value;
-        }
-    }
-
-    /**
-     * Returns the list of analysables sorted in processing priority order.
-     *
-     * It will first return analysables that have never been analysed before
-     * and it will continue with the ones we have already seen by timeanalysed DESC
-     * order.
-     *
-     * @param bool $includetarget
-     * @return array(0 => \core_analytics\analysable[], 1 => \stdClass[])
-     */
-    protected function get_sorted_analysables($includetarget) {
-
-        $analysables = $this->get_analysables();
-
-        // Get the list of analysables that have been already processed.
-        $processedanalysables = $this->get_processed_analysables($includetarget);
-
-        // We want to start processing analysables we have not yet processed and later continue
-        // with analysables that we already processed.
-        $unseen = array_diff_key($analysables, $processedanalysables);
-
-        // Var $processed first as we want to respect its timeanalysed DESC order so analysables that
-        // have recently been processed are on the bottom of the stack.
-        $seen = array_intersect_key($processedanalysables, $analysables);
-        array_walk($seen, function(&$value, $analysableid) use ($analysables) {
-            // We replace the analytics_used_analysables record by the analysable object.
-            $value = $analysables[$analysableid];
-        });
-
-        return array($unseen + $seen, $processedanalysables);
-    }
-
-    /**
-     * Get analysables that have been already processed.
-     *
-     * @param bool $includetarget
-     * @return \stdClass[]
-     */
-    protected function get_processed_analysables($includetarget) {
-        global $DB;
-
-        $params = array('modelid' => $this->modelid);
-        $params['action'] = ($includetarget) ? 'training' : 'prediction';
-        $select = 'modelid = :modelid and action = :action';
-
-        // Weird select fields ordering for performance (analysableid key matching, analysableid is also unique by modelid).
-        return $DB->get_records_select('analytics_used_analysables', $select,
-            $params, 'timeanalysed DESC', 'analysableid, modelid, action, timeanalysed, id AS primarykey');
-    }
-
-    /**
-     * Updates the analysable analysis time.
-     *
-     * @param array $processedanalysables
-     * @param int $analysableid
-     * @param bool $includetarget
-     * @return null
-     */
-    protected function update_analysable_analysed_time($processedanalysables, $analysableid, $includetarget) {
-        global $DB;
-
-        if (!empty($processedanalysables[$analysableid])) {
-            $obj = $processedanalysables[$analysableid];
-
-            $obj->id = $obj->primarykey;
-            unset($obj->primarykey);
-
-            $obj->timeanalysed = time();
-            $DB->update_record('analytics_used_analysables', $obj);
-
-        } else {
-
-            $obj = new \stdClass();
-            $obj->modelid = $this->modelid;
-            $obj->action = ($includetarget) ? 'training' : 'prediction';
-            $obj->analysableid = $analysableid;
-            $obj->timeanalysed = time();
-
-            $DB->insert_record('analytics_used_analysables', $obj);
-        }
+        return $ordersql;
     }
 }

analytics/classes/local/analyser/by_course.php

@@ -38,34 +38,43 @@ abstract class by_course extends base {
     /**
      * Return the list of courses to analyse.
      *
-     * @return \core_analytics\course[]
+     * @param string|null $action 'prediction', 'training' or null if no specific action needed.
+     * @return \Iterator
      */
-    public function get_analysables() {
+    public function get_analysables_iterator(?string $action = null) {
+        global $DB;
 
-        // Default to all system courses.
+        list($sql, $params) = $this->get_iterator_sql('course', CONTEXT_COURSE, $action, 'c');
+
+        // This will be updated to filter by context as part of MDL-64739.
         if (!empty($this->options['filter'])) {
             $courses = array();
             foreach ($this->options['filter'] as $courseid) {
                 $courses[$courseid] = new \stdClass();
                 $courses[$courseid]->id = $courseid;
             }
-        } else {
-            // Iterate through all potentially valid courses.
-            $courses = get_courses('all', 'c.sortorder ASC', 'c.id');
-        }
-        unset($courses[SITEID]);
 
-        $analysables = array();
-        foreach ($courses as $course) {
-            // Skip the frontpage course.
-            $analysable = \core_analytics\course::instance($course->id);
-            $analysables[$analysable->get_id()] = $analysable;
+            list($coursesql, $courseparams) = $DB->get_in_or_equal($courses, SQL_PARAMS_NAMED);
+            $sql .= " AND c.id IN $coursesql";
+            $params = $params + $courseparams;
         }
 
-        if (empty($analysables)) {
-            $this->log[] = get_string('nocourses', 'analytics');
+        $ordersql = $this->order_sql('sortorder', 'ASC', 'c');
+
+        $recordset = $DB->get_recordset_sql($sql . $ordersql, $params);
+
+        if (!$recordset->valid()) {
+            $this->add_log(get_string('nocourses', 'analytics'));
+            return new \ArrayIterator([]);
         }
 
-        return $analysables;
+        return new \core\dml\recordset_walk($recordset, function($record) {
+
+            if ($record->id == SITEID) {
+                return false;
+            }
+            $context = \context_helper::preload_from_record($record);
+            return \core_analytics\course::instance($record, $context);
+        });
     }
-}
+}

analytics/classes/local/analyser/sitewide.php

@@ -36,12 +36,13 @@ defined('MOODLE_INTERNAL') || die();
 abstract class sitewide extends base {
 
     /**
-     * Returns one single analysable element, the site.
+     * Return the list of analysables to analyse.
      *
-     * @return \core_analytics\analysable[]
+     * @param string|null $action 'prediction', 'training' or null if no specific action needed.
+     * @return \Iterator
      */
-    public function get_analysables() {
-        $analysable = new \core_analytics\site();
-        return array(SYSCONTEXTID => $analysable);
+    public function get_analysables_iterator(?string $action = null) {
+        // We can safely ignore $action as we have 1 single analysable element in this analyser.
+        return new \ArrayIterator([new \core_analytics\site()]);
     }
 }

analytics/classes/local/analysis/result.php

@@ -0,0 +1,102 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Keeps track of the analysis results.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\analysis;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Keeps track of the analysis results.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class result {
+
+    /**
+     * @var int
+     */
+    protected $modelid;
+
+    /**
+     * @var bool
+     */
+    protected $includetarget;
+
+    /**
+     * @var array Analysis options
+     */
+    protected $options;
+
+    /**
+     * Stores analysis data at instance level.
+     * @param int   $modelid
+     * @param bool  $includetarget
+     * @param array $options
+     */
+    public function __construct(int $modelid, bool $includetarget, array $options) {
+        $this->modelid = $modelid;
+        $this->includetarget = $includetarget;
+        $this->options = $options;
+    }
+
+    /**
+     * Retrieves cached results during evaluation.
+     *
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed It can be in whatever format the result uses.
+     */
+    public function retrieve_cached_result(\core_analytics\local\time_splitting\base $timesplitting,
+        \core_analytics\analysable $analysable) {
+        return false;
+    }
+
+    /**
+     * Stores the analysis results.
+     *
+     * @param  array $results
+     * @return bool            True if anything was successfully analysed
+     */
+    abstract public function add_analysable_results(array $results): bool;
+
+    /**
+     * Formats the result.
+     *
+     * @param  array                                     $data
+     * @param  \core_analytics\local\target\base         $target
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed It can be in whatever format the result uses
+     */
+    abstract public function format_result(array $data, \core_analytics\local\target\base $target,
+            \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable);
+
+    /**
+     * Returns the results of the analysis.
+     * @return array
+     */
+    abstract public function get(): array;
+}

analytics/classes/local/analysis/result_array.php

@@ -0,0 +1,100 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Keeps track of the analysis results by storing the results in an array.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\analysis;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Keeps track of the analysis results by storing the results in an array.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class result_array extends result {
+
+    /**
+     * Stores the analysis results by time-splitting method.
+     * @var array
+     */
+    private $resultsbytimesplitting = [];
+
+    /**
+     * Stores the analysis results.
+     * @param  array $results
+     * @return bool            True if anything was successfully analysed
+     */
+    public function add_analysable_results(array $results): bool {
+
+        $any = false;
+
+        // Process all provided time splitting methods.
+        foreach ($results as $timesplittingid => $result) {
+            if (!empty($result->result)) {
+                if (empty($this->resultsbytimesplitting[$timesplittingid])) {
+                    $this->resultsbytimesplitting[$timesplittingid] = [];
+                }
+                $this->resultsbytimesplitting[$timesplittingid] += $result->result;
+                $any = true;
+            }
+        }
+        if (empty($any)) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Formats the result.
+     *
+     * @param  array                                     $data
+     * @param  \core_analytics\local\target\base         $target
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed The data as it comes
+     */
+    public function format_result(array $data, \core_analytics\local\target\base $target,
+            \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable) {
+        return $data;
+    }
+
+    /**
+     * Returns the results of the analysis.
+     * @return array
+     */
+    public function get(): array {
+
+        // We join the datasets by time splitting method.
+        $timesplittingresults = array();
+        foreach ($this->resultsbytimesplitting as $timesplittingid => $results) {
+            if (empty($timesplittingresults[$timesplittingid])) {
+                $timesplittingresults[$timesplittingid] = [];
+            }
+            $timesplittingresults[$timesplittingid] += $results;
+        }
+
+        return $timesplittingresults;
+    }
+}

analytics/classes/local/analysis/result_file.php

@@ -0,0 +1,228 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Keeps track of the analysis results by storing the results in files.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\analysis;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Keeps track of the analysis results by storing the results in files.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class result_file extends result {
+
+    /**
+     * Stores the analysis results by time-splitting method.
+     * @var array
+     */
+    private $filesbytimesplitting = [];
+
+    /**
+     * Stores the analysis results.
+     * @param  array $results
+     * @return bool            True if anything was successfully analysed
+     */
+    public function add_analysable_results(array $results): bool {
+
+        $any = false;
+
+        // Process all provided time splitting methods.
+        foreach ($results as $timesplittingid => $result) {
+            if (!empty($result->result)) {
+                $this->filesbytimesplitting[$timesplittingid][] = $result->result;
+                $any = true;
+            }
+        }
+
+        if (empty($any)) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Retrieves cached results during evaluation.
+     *
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed A \stored_file in this case.
+     */
+    public function retrieve_cached_result(\core_analytics\local\time_splitting\base $timesplitting,
+        \core_analytics\analysable $analysable) {
+
+        // For evaluation purposes we don't need to be that strict about how updated the data is,
+        // if this analyser was analysed less that 1 week ago we skip generating a new one. This
+        // helps scale the evaluation process as sites with tons of courses may need a lot of time to
+        // complete an evaluation.
+        if (!empty($options['evaluation']) && !empty($options['reuseprevanalysed'])) {
+
+            $previousanalysis = \core_analytics\dataset_manager::get_evaluation_analysable_file($this->analyser->get_modelid(),
+                $analysable->get_id(), $timesplitting->get_id());
+            // 1 week is a partly random time interval, no need to worry about DST.
+            $boundary = time() - WEEKSECS;
+            if ($previousanalysis && $previousanalysis->get_timecreated() > $boundary) {
+                // Recover the previous analysed file and avoid generating a new one.
+                return $previousanalysis;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Formats the result.
+     *
+     * @param  array                                     $data
+     * @param  \core_analytics\local\target\base         $target
+     * @param  \core_analytics\local\time_splitting\base $timesplitting
+     * @param  \core_analytics\analysable                $analysable
+     * @return mixed A \stored_file in this case
+     */
+    public function format_result(array $data, \core_analytics\local\target\base $target,
+            \core_analytics\local\time_splitting\base $timesplitting, \core_analytics\analysable $analysable) {
+
+        if (!empty($this->includetarget)) {
+            $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
+        } else {
+            $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
+        }
+        $dataset = new \core_analytics\dataset_manager($this->modelid, $analysable->get_id(),
+            $timesplitting->get_id(), $filearea, $this->options['evaluation']);
+
+        // Add extra metadata.
+        $this->add_model_metadata($data, $timesplitting, $target);
+
+        // Write all calculated data to a file.
+        if (!$result = $dataset->store($data)) {
+            return false;
+        }
+
+        return $result;
+    }
+
+    /**
+     * Returns the results of the analysis.
+     * @return array
+     */
+    public function get(): array {
+
+        if ($this->options['evaluation'] === false) {
+            // Look for previous training and prediction files we generated and couldn't be used
+            // by machine learning backends because they weren't big enough.
+
+            $pendingfiles = \core_analytics\dataset_manager::get_pending_files($this->modelid, $this->includetarget,
+                array_keys($this->filesbytimesplitting));
+            foreach ($pendingfiles as $timesplittingid => $files) {
+                foreach ($files as $file) {
+                    $this->filesbytimesplitting[$timesplittingid][] = $file;
+                }
+            }
+        }
+
+        // We join the datasets by time splitting method.
+        $timesplittingfiles = array();
+        foreach ($this->filesbytimesplitting as $timesplittingid => $files) {
+
+            if ($this->options['evaluation'] === true) {
+                // Delete the previous copy. Only when evaluating.
+                \core_analytics\dataset_manager::delete_previous_evaluation_file($this->modelid, $timesplittingid);
+            }
+
+            // Merge all course files into one.
+            if ($this->includetarget) {
+                $filearea = \core_analytics\dataset_manager::LABELLED_FILEAREA;
+            } else {
+                $filearea = \core_analytics\dataset_manager::UNLABELLED_FILEAREA;
+            }
+            $timesplittingfiles[$timesplittingid] = \core_analytics\dataset_manager::merge_datasets($files,
+                $this->modelid, $timesplittingid, $filearea, $this->options['evaluation']);
+        }
+
+        if (!empty($pendingfiles)) {
+            // We must remove them now as they are already part of another dataset.
+            foreach ($pendingfiles as $timesplittingid => $files) {
+                foreach ($files as $file) {
+                    $file->delete();
+                }
+            }
+        }
+
+        return $timesplittingfiles;
+    }
+
+    /**
+     * Adds target metadata to the dataset.
+     *
+     * The final dataset document will look like this:
+     * ----------------------------------------------------
+     * metadata1,metadata2,metadata3,.....
+     * value1, value2, value3,.....
+     *
+     * header1,header2,header3,header4,.....
+     * stud1value1,stud1value2,stud1value3,stud1value4,.....
+     * stud2value1,stud2value2,stud2value3,stud2value4,.....
+     * .....
+     * ----------------------------------------------------
+     *
+     * @param array $data
+     * @param \core_analytics\local\time_splitting\base $timesplitting
+     * @param \core_analytics\local\target\base         $target
+     * @return null
+     */
+    private function add_model_metadata(array &$data, \core_analytics\local\time_splitting\base $timesplitting,
+            \core_analytics\local\target\base $target) {
+        global $CFG;
+
+        // If no target the first column is the sampleid, if target the last column is the target.
+        // This will need to be updated when we support unsupervised learning models.
+        $metadata = array(
+            'timesplitting' => $timesplitting->get_id(),
+            'nfeatures' => count(current($data)) - 1,
+            'moodleversion' => $CFG->version,
+            'targetcolumn' => $target->get_id()
+        );
+        if ($target->is_linear()) {
+            $metadata['targettype'] = 'linear';
+            $metadata['targetmin'] = $target::get_min_value();
+            $metadata['targetmax'] = $target::get_max_value();
+        } else {
+            $metadata['targettype'] = 'discrete';
+            $metadata['targetclasses'] = json_encode($target::get_classes());
+        }
+
+        // The first 2 samples will be used to store metadata about the dataset.
+        $metadatacolumns = [];
+        $metadatavalues = [];
+        foreach ($metadata as $key => $value) {
+            $metadatacolumns[] = $key;
+            $metadatavalues[] = $value;
+        }
+
+        // This will also reset samples' dataset keys.
+        array_unshift($data, $metadatacolumns, $metadatavalues);
+    }
+}

analytics/classes/local/target/base.php

@@ -104,14 +104,29 @@ abstract class base extends \core_analytics\calculable {
         return false;
     }
 
+    /**
+     * Update the last analysis time on analysable processed or always.
+     *
+     * If you overwrite this method to return false the last analysis time
+     * will only be recorded in DB when the element successfully analysed. You can
+     * safely return false for lightweight targets.
+     *
+     * @return bool
+     */
+    public function always_update_analysis_time(): bool {
+        return true;
+    }
+
     /**
      * Suggested actions for a user.
      *
      * @param \core_analytics\prediction $prediction
      * @param bool $includedetailsaction
+     * @param bool $isinsightuser
      * @return \core_analytics\prediction_action[]
      */
-    public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false) {
+    public function prediction_actions(\core_analytics\prediction $prediction, $includedetailsaction = false,
+            $isinsightuser = false) {
         global $PAGE;
 
         $predictionid = $prediction->get_prediction_data()->id;
@@ -122,12 +137,12 @@ abstract class base extends \core_analytics\calculable {
 
         if ($includedetailsaction) {
 
-            $predictionurl = new \moodle_url('/report/insights/prediction.php',
-                array('id' => $predictionid));
+            $predictionurl = new \moodle_url('/report/insights/prediction.php', array('id' => $predictionid));
+            $detailstext = $this->get_view_details_text();
 
             $actions[] = new \core_analytics\prediction_action(\core_analytics\prediction::ACTION_PREDICTION_DETAILS, $prediction,
-                $predictionurl, new \pix_icon('t/preview', get_string('viewprediction', 'analytics')),
-                get_string('viewprediction', 'analytics'));
+                $predictionurl, new \pix_icon('t/preview', $detailstext),
+                $detailstext);
         }
 
         // Flag as fixed / solved.
@@ -151,6 +166,25 @@ abstract class base extends \core_analytics\calculable {
         return $actions;
     }
 
+    /**
+     * Returns the view details link text.
+     * @return string
+     */
+    private function get_view_details_text() {
+        if ($this->based_on_assumptions()) {
+            $analyserclass = $this->get_analyser_class();
+            if ($analyserclass::one_sample_per_analysable()) {
+                $detailstext = get_string('viewinsightdetails', 'analytics');
+            } else {
+                $detailstext = get_string('viewdetails', 'analytics');
+            }
+        } else {
+            $detailstext = get_string('viewprediction', 'analytics');
+        }
+
+        return $detailstext;
+    }
+
     /**
      * Callback to execute once a prediction has been returned from the predictions processor.
      *
@@ -173,61 +207,28 @@ abstract class base extends \core_analytics\calculable {
      *
      * @param int $modelid
      * @param \context[] $samplecontexts
+     * @param  \core_analytics\prediction[] $predictions
      * @return void
      */
-    public function generate_insight_notifications($modelid, $samplecontexts) {
-
-        foreach ($samplecontexts as $context) {
-
-            $insightinfo = new \stdClass();
-            $insightinfo->insightname = $this->get_name();
-            $insightinfo->contextname = $context->get_context_name();
-            $subject = get_string('insightmessagesubject', 'analytics', $insightinfo);
-
-            $users = $this->get_insights_users($context);
-
-            if (!$coursecontext = $context->get_course_context(false)) {
-                $coursecontext = \context_course::instance(SITEID);
-            }
-
-            foreach ($users as $user) {
-
-                $message = new \core\message\message();
-                $message->component = 'moodle';
-                $message->name = 'insights';
-
-                $message->userfrom = \core_user::get_noreply_user();
-                $message->userto = $user;
-
-                $insighturl = new \moodle_url('/report/insights/insights.php?modelid=' . $modelid . '&contextid=' . $context->id);
-                $message->subject = $subject;
-                // Same than the subject.
-                $message->contexturlname = $message->subject;
-                $message->courseid = $coursecontext->instanceid;
-
-                $message->fullmessage = get_string('insightinfomessage', 'analytics', $insighturl->out(false));
-                $message->fullmessageformat = FORMAT_PLAIN;
-                $message->fullmessagehtml = get_string('insightinfomessagehtml', 'analytics', $insighturl->out());
-                $message->smallmessage = get_string('insightinfomessage', 'analytics', $insighturl->out(false));
-                $message->contexturl = $insighturl->out(false);
-
-                message_send($message);
-            }
-        }
-
+    public function generate_insight_notifications($modelid, $samplecontexts, array $predictions = []) {
+        // Delegate the processing of insights to the insights_generator.
+        $insightsgenerator = new \core_analytics\insights_generator($modelid, $this);
+        $insightsgenerator->generate($samplecontexts, $predictions);
     }
 
     /**
      * Returns the list of users that will receive insights notifications.
      *
      * Feel free to overwrite if you need to but keep in mind that moodle/analytics:listinsights
-     * capability is required to access the list of insights.
+     * or moodle/analytics:listowninsights capability is required to access the list of insights.
      *
      * @param \context $context
      * @return array
      */
-    protected function get_insights_users(\context $context) {
-        if ($context->contextlevel >= CONTEXT_COURSE) {
+    public function get_insights_users(\context $context) {
+        if ($context->contextlevel === CONTEXT_USER) {
+            $users = [$context->instanceid => \core_user::get_user($context->instanceid)];
+        } else if ($context->contextlevel >= CONTEXT_COURSE) {
             // At course level or below only enrolled users although this is not ideal for
             // teachers assigned at category level.
             $users = get_enrolled_users($context, 'moodle/analytics:listinsights');
@@ -237,6 +238,30 @@ abstract class base extends \core_analytics\calculable {
         return $users;
     }
 
+    /**
+     * URL to the insight.
+     *
+     * @param  int $modelid
+     * @param  \context $context
+     * @return \moodle_url
+     */
+    public function get_insight_context_url($modelid, $context) {
+        return new \moodle_url('/report/insights/insights.php?modelid=' . $modelid . '&contextid=' . $context->id);
+    }
+
+    /**
+     * The insight notification subject.
+     *
+     * This is just a default message, you should overwrite it for a custom insight message.
+     *
+     * @param  int $modelid
+     * @param  \context $context
+     * @return string
+     */
+    public function get_insight_subject(int $modelid, \context $context) {
+        return get_string('insightmessagesubject', 'analytics', $context->get_context_name());
+    }
+
     /**
      * Returns an instance of the child class.
      *

analytics/classes/local/target/binary.php

@@ -60,7 +60,7 @@ abstract class binary extends discrete {
      *
      * @return array
      */
-    protected function ignored_predicted_classes() {
+    public function ignored_predicted_classes() {
         // Zero-value class is usually ignored in binary classifiers.
         return array(0);
     }

analytics/classes/local/target/discrete.php

@@ -145,7 +145,7 @@ abstract class discrete extends base {
      *
      * @return array List of values that will be ignored (array keys are ignored).
      */
-    protected function ignored_predicted_classes() {
+    public function ignored_predicted_classes() {
         // Coding exception as this will only be called if this target have non-linear values.
         throw new \coding_exception('Overwrite ignored_predicted_classes() and return an array with the classes that should not ' .
             'trigger the callback');

analytics/classes/local/time_splitting/base.php

@@ -40,42 +40,29 @@ abstract class base {
      */
     protected $id;
 
+    /**
+     * The model id.
+     *
+     * @var int
+     */
+    protected $modelid;
+
     /**
      * @var \core_analytics\analysable
      */
     protected $analysable;
 
-
-    /**
-     * @var int[]
-     */
-    protected $sampleids;
-
-    /**
-     * @var string
-     */
-    protected $samplesorigin;
-
     /**
      * @var array
      */
     protected $ranges = [];
 
-    /**
-     * @var \core_analytics\local\indicator\base
-     */
-    protected static $indicators = [];
-
-    /**
-     * @var bool
-     */
-    protected $evaluation = false;
-
     /**
      * Define the time splitting methods ranges.
      *
      * 'time' value defines when predictions are executed, their values will be compared with
-     * the current time in ready_to_predict
+     * the current time in ready_to_predict. The ranges should be sorted by 'time' in
+     * ascending order.
      *
      * @return array('start' => time(), 'end' => time(), 'time' => time())
      */
@@ -101,24 +88,6 @@ abstract class base {
         return '\\' . get_class($this);
     }
 
-    /**
-     * Returns current evaluation value.
-     *
-     * @return bool
-     */
-    public function is_evaluating() {
-        return $this->evaluation;
-    }
-
-    /**
-     * Sets the evaluation flag.
-     *
-     * @param bool $evaluation
-     */
-    public function set_evaluating($evaluation) {
-        $this->evaluation = (bool)$evaluation;
-    }
-
     /**
      * Assigns the analysable and updates the time ranges according to the analysable start and end dates.
      *
@@ -164,252 +133,17 @@ abstract class base {
     }
 
     /**
-     * Calculates indicators and targets.
+     * Should we use this time range for training?
      *
-     * @param array $sampleids
-     * @param string $samplesorigin
-     * @param \core_analytics\local\indicator\base[] $indicators
-     * @param array $ranges
-     * @param \core_analytics\local\target\base $target
-     * @return array|bool
+     * @param array $range
+     * @return bool
      */
-    public function calculate(&$sampleids, $samplesorigin, $indicators, $ranges, $target = false) {
-
-        $calculatedtarget = false;
-        if ($target) {
-            // We first calculate the target because analysable data may still be invalid or none
-            // of the analysable samples may be valid ($sampleids is also passed by reference).
-            $calculatedtarget = $target->calculate($sampleids, $this->analysable);
-
-            // We remove samples we can not calculate their target.
-            $sampleids = array_filter($sampleids, function($sampleid) use ($calculatedtarget) {
-                if (is_null($calculatedtarget[$sampleid])) {
-                    return false;
-                }
-                return true;
-            });
+    public function ready_to_train($range) {
+        $now = time();
+        if ($range['time'] <= $now && $range['end'] <= $now) {
+            return true;
         }
-
-        // No need to continue calculating if the target couldn't be calculated for any sample.
-        if (empty($sampleids)) {
-            return false;
-        }
-
-        $dataset = $this->calculate_indicators($sampleids, $samplesorigin, $indicators, $ranges);
-
-        if (empty($dataset)) {
-            return false;
-        }
-
-        // Now that we have the indicators in place we can add the time range indicators (and target if provided) to each of them.
-        $this->fill_dataset($dataset, $calculatedtarget);
-
-        $this->add_metadata($dataset, $indicators, $target);
-
-        if (!PHPUNIT_TEST && CLI_SCRIPT) {
-            echo PHP_EOL;
-        }
-
-        return $dataset;
-    }
-
-    /**
-     * Calculates indicators.
-     *
-     * @param array $sampleids
-     * @param string $samplesorigin
-     * @param \core_analytics\local\indicator\base[] $indicators
-     * @param array $ranges
-     * @return array
-     */
-    protected function calculate_indicators($sampleids, $samplesorigin, $indicators, $ranges) {
-        global $DB;
-
-        $dataset = array();
-
-        // Faster to run 1 db query per range.
-        $existingcalculations = array();
-        foreach ($ranges as $rangeindex => $range) {
-            // Load existing calculations.
-            $existingcalculations[$rangeindex] = \core_analytics\manager::get_indicator_calculations($this->analysable,
-                $range['start'], $range['end'], $samplesorigin);
-        }
-
-        // Here we store samples which calculations are not all null.
-        $notnulls = array();
-
-        // Fill the dataset samples with indicators data.
-        $newcalculations = array();
-        foreach ($indicators as $indicator) {
-
-            // Hook to allow indicators to store analysable-dependant data.
-            $indicator->fill_per_analysable_caches($this->analysable);
-
-            // Per-range calculations.
-            foreach ($ranges as $rangeindex => $range) {
-
-                // Indicator instances are per-range.
-                $rangeindicator = clone $indicator;
-
-                $prevcalculations = array();
-                if (!empty($existingcalculations[$rangeindex][$rangeindicator->get_id()])) {
-                    $prevcalculations = $existingcalculations[$rangeindex][$rangeindicator->get_id()];
-                }
-
-                // Calculate the indicator for each sample in this time range.
-                list($samplesfeatures, $newindicatorcalculations, $indicatornotnulls) = $rangeindicator->calculate($sampleids,
-                    $samplesorigin, $range['start'], $range['end'], $prevcalculations);
-
-                // Copy the features data to the dataset.
-                foreach ($samplesfeatures as $analysersampleid => $features) {
-
-                    $uniquesampleid = $this->append_rangeindex($analysersampleid, $rangeindex);
-
-                    if (!isset($notnulls[$uniquesampleid]) && !empty($indicatornotnulls[$analysersampleid])) {
-                        $notnulls[$uniquesampleid] = $uniquesampleid;
-                    }
-
-                    // Init the sample if it is still empty.
-                    if (!isset($dataset[$uniquesampleid])) {
-                        $dataset[$uniquesampleid] = array();
-                    }
-
-                    // Append the features indicator features at the end of the sample.
-                    $dataset[$uniquesampleid] = array_merge($dataset[$uniquesampleid], $features);
-                }
-
-                if (!$this->is_evaluating()) {
-                    $timecreated = time();
-                    foreach ($newindicatorcalculations as $sampleid => $calculatedvalue) {
-                        // Prepare the new calculations to be stored into DB.
-
-                        $indcalc = new \stdClass();
-                        $indcalc->contextid = $this->analysable->get_context()->id;
-                        $indcalc->starttime = $range['start'];
-                        $indcalc->endtime = $range['end'];
-                        $indcalc->sampleid = $sampleid;
-                        $indcalc->sampleorigin = $samplesorigin;
-                        $indcalc->indicator = $rangeindicator->get_id();
-                        $indcalc->value = $calculatedvalue;
-                        $indcalc->timecreated = $timecreated;
-                        $newcalculations[] = $indcalc;
-                    }
-                }
-            }
-
-            if (!$this->is_evaluating()) {
-                $batchsize = self::get_insert_batch_size();
-                if (count($newcalculations) > $batchsize) {
-                    // We don't want newcalculations array to grow too much as we already keep the
-                    // system memory busy storing $dataset contents.
-
-                    // Insert from the beginning.
-                    $remaining = array_splice($newcalculations, $batchsize);
-
-                    // Sorry mssql and oracle, this will be slow.
-                    $DB->insert_records('analytics_indicator_calc', $newcalculations);
-                    $newcalculations = $remaining;
-                }
-            }
-        }
-
-        if (!$this->is_evaluating() && $newcalculations) {
-            // Insert the remaining records.
-            $DB->insert_records('analytics_indicator_calc', $newcalculations);
-        }
-
-        // Delete rows where all calculations are null.
-        // We still store the indicator calculation and we still store the sample id as
-        // processed so we don't have to process this sample again, but we exclude it
-        // from the dataset because it is not useful.
-        $nulls = array_diff_key($dataset, $notnulls);
-        foreach ($nulls as $uniqueid => $ignoredvalues) {
-            unset($dataset[$uniqueid]);
-        }
-
-        return $dataset;
-    }
-
-    /**
-     * Adds time range indicators and the target to each sample.
-     *
-     * This will identify the sample as belonging to a specific range.
-     *
-     * @param array $dataset
-     * @param array $calculatedtarget
-     * @return void
-     */
-    protected function fill_dataset(&$dataset, $calculatedtarget = false) {
-
-        $nranges = count($this->get_all_ranges());
-
-        foreach ($dataset as $uniquesampleid => $unmodified) {
-
-            list($analysersampleid, $rangeindex) = $this->infer_sample_info($uniquesampleid);
-
-            // No need to add range features if this time splitting method only defines one time range.
-            if ($nranges > 1) {
-
-                // 1 column for each range.
-                $timeindicators = array_fill(0, $nranges, 0);
-
-                $timeindicators[$rangeindex] = 1;
-
-                $dataset[$uniquesampleid] = array_merge($timeindicators, $dataset[$uniquesampleid]);
-            }
-
-            if ($calculatedtarget) {
-                // Add this sampleid's calculated target and the end.
-                $dataset[$uniquesampleid][] = $calculatedtarget[$analysersampleid];
-
-            } else {
-                // Add this sampleid, it will be used to identify the prediction that comes back from
-                // the predictions processor.
-                array_unshift($dataset[$uniquesampleid], $uniquesampleid);
-            }
-        }
-    }
-
-    /**
-     * Adds dataset context info.
-     *
-     * The final dataset document will look like this:
-     * ----------------------------------------------------
-     * metadata1,metadata2,metadata3,.....
-     * value1, value2, value3,.....
-     *
-     * indicator1,indicator2,indicator3,indicator4,.....
-     * stud1value1,stud1value2,stud1value3,stud1value4,.....
-     * stud2value1,stud2value2,stud2value3,stud2value4,.....
-     * .....
-     * ----------------------------------------------------
-     *
-     * @param array $dataset
-     * @param \core_analytics\local\indicator\base[] $indicators
-     * @param \core_analytics\local\target\base|false $target
-     * @return void
-     */
-    protected function add_metadata(&$dataset, $indicators, $target = false) {
-
-        $metadata = array(
-            'timesplitting' => $this->get_id(),
-            // If no target the first column is the sampleid, if target the last column is the target.
-            // This will need to be updated when we support unsupervised learning models.
-            'nfeatures' => count(current($dataset)) - 1
-        );
-
-        // The first 2 samples will be used to store metadata about the dataset.
-        $metadatacolumns = [];
-        $metadatavalues = [];
-        foreach ($metadata as $key => $value) {
-            $metadatacolumns[] = $key;
-            $metadatavalues[] = $value;
-        }
-
-        $headers = $this->get_headers($indicators, $target);
-
-        // This will also reset samples' dataset keys.
-        array_unshift($dataset, $metadatacolumns, $metadatavalues, $headers);
+        return false;
     }
 
     /**
@@ -421,6 +155,53 @@ abstract class base {
         return $this->ranges;
     }
 
+    /**
+     * By default all ranges are for training.
+     *
+     * @return array
+     */
+    public function get_training_ranges() {
+        return $this->ranges;
+    }
+
+    /**
+     * Returns the distinct range indexes in this time splitting method.
+     *
+     * @return int[]
+     */
+    public function get_distinct_ranges() {
+        if ($this->include_range_info_in_training_data()) {
+            return array_keys($this->ranges);
+        } else {
+            return [0];
+        }
+    }
+
+    /**
+     * Returns the most recent range that can be used to predict.
+     *
+     * This method is only called when calculating predictions.
+     *
+     * @return array
+     */
+    public function get_most_recent_prediction_range() {
+
+        $ranges = $this->get_all_ranges();
+
+        // Opposite order as we are interested in the last range that can be used for prediction.
+        krsort($ranges);
+
+        // We already provided the analysable to the time splitting method, there is no need to feed it back.
+        foreach ($ranges as $rangeindex => $range) {
+            if ($this->ready_to_predict($range)) {
+                // We need to maintain the same indexes.
+                return array($rangeindex => $range);
+            }
+        }
+
+        return array();
+    }
+
     /**
      * Returns range data by its index.
      *
@@ -441,7 +222,7 @@ abstract class base {
      * @param int $rangeindex
      * @return string
      */
-    public function append_rangeindex($sampleid, $rangeindex) {
+    public final function append_rangeindex($sampleid, $rangeindex) {
         return $sampleid . '-' . $rangeindex;
     }
 
@@ -451,46 +232,47 @@ abstract class base {
      * @param string $uniquesampleid
      * @return array array($sampleid, $rangeindex)
      */
-    public function infer_sample_info($uniquesampleid) {
+    public final function infer_sample_info($uniquesampleid) {
         return explode('-', $uniquesampleid);
     }
 
     /**
-     * Returns the headers for the csv file based on the indicators and the target.
+     * Whether to include the range index in the training data or not.
      *
-     * @param \core_analytics\local\indicator\base[] $indicators
-     * @param \core_analytics\local\target\base|false $target
-     * @return string[]
+     * By default, we consider that the different time ranges included in a time splitting method may not be
+     * compatible between them (i.e. the indicators calculated at the end of the course can easily
+     * differ from indicators calculated at the beginning of the course). So we include the range index as
+     * one of the variables that the machine learning backend uses to generate predictions.
+     *
+     * If the indicators calculated using the different time ranges available in this time splitting method
+     * are comparable you can overwrite this method to return false.
+     *
+     * Note that:
+     *  - This is only relevant for models whose predictions are not based on assumptions
+     *    (i.e. the ones using a machine learning backend to generate predictions).
+     *  - The ranges can only be included in the training data when
+     *    we know the final number of ranges the time splitting method will have. E.g.
+     *    We can not know the final number of ranges of a 'daily' time splitting method
+     *    as we will have one new range every day.
+     * @return bool
      */
-    protected function get_headers($indicators, $target = false) {
-        // 3rd column will contain the indicator ids.
-        $headers = array();
+    public function include_range_info_in_training_data() {
+        return true;
+    }
 
-        if (!$target) {
-            // The first column is the sampleid.
-            $headers[] = 'sampleid';
-        }
-
-        // We always have 1 column for each time splitting method range, it does not depend on how
-        // many ranges we calculated.
-        $ranges = $this->get_all_ranges();
-        if (count($ranges) > 1) {
-            foreach ($ranges as $rangeindex => $range) {
-                $headers[] = 'range/' . $rangeindex;
-            }
-        }
-
-        // Model indicators.
-        foreach ($indicators as $indicator) {
-            $headers = array_merge($headers, $indicator::get_feature_headers());
-        }
-
-        // The target as well.
-        if ($target) {
-            $headers[] = $target->get_id();
-        }
-
-        return $headers;
+    /**
+     * Whether to cache or not the indicator calculations.
+     *
+     * Indicator calculations are stored to be reused across models. The calculations
+     * are indexed by the calculation start and end time, and these times depend on the
+     * time-splitting method. You should overwrite this method and return false if the time
+     * frames generated by your time-splitting method are unique and / or can hardly be
+     * reused by further models.
+     *
+     * @return bool
+     */
+    public function cache_indicator_calculations(): bool {
+        return true;
     }
 
     /**
@@ -508,32 +290,4 @@ abstract class base {
             }
         }
     }
-
-    /**
-     * Returns the batch size used for insert_records.
-     *
-     * This method tries to find the best batch size without getting
-     * into dml internals. Maximum 1000 records to save memory.
-     *
-     * @return int
-     */
-    private static function get_insert_batch_size() {
-        global $DB;
-
-        // 500 is pgsql default so using 1000 is fine, no other db driver uses a hardcoded value.
-        if (empty($DB->dboptions['bulkinsertsize'])) {
-            return 1000;
-        }
-
-        $bulkinsert = $DB->dboptions['bulkinsertsize'];
-        if ($bulkinsert < 1000) {
-            return $bulkinsert;
-        }
-
-        while ($bulkinsert > 1000) {
-            $bulkinsert = round($bulkinsert / 2, 0);
-        }
-
-        return (int)$bulkinsert;
-    }
 }

analytics/classes/local/time_splitting/periodic.php

@@ -0,0 +1,143 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Time splitting method that generates predictions regularly.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\time_splitting;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Time splitting method that generates predictions periodically.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class periodic extends base {
+
+    /**
+     * The periodicity of the predictions / training data generation.
+     *
+     * @return \DateInterval
+     */
+    abstract protected function periodicity();
+
+    /**
+     * Returns whether the analysable can be processed by this time splitting method or not.
+     *
+     * @param \core_analytics\analysable $analysable
+     * @return bool
+     */
+    public function is_valid_analysable(\core_analytics\analysable $analysable) {
+        if (!$analysable->get_start()) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * define_ranges
+     *
+     * @return array
+     */
+    protected function define_ranges() {
+
+        $periodicity = $this->periodicity();
+
+        $now = new \DateTimeImmutable('now', \core_date::get_server_timezone_object());
+
+        if ($this->analysable->get_end()) {
+            $end = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_end());
+        }
+        $next = (new \DateTimeImmutable())->setTimestamp($this->analysable->get_start());
+
+        $ranges = [];
+        while ($next < $now &&
+                (empty($end) || $next < $end)) {
+            $range = $this->get_next_range($next);
+            if ($range) {
+                $ranges[] = $range;
+            }
+            $next = $next->add($periodicity);
+        }
+
+        $nextrange = $this->get_next_range($next);
+        if ($this->ready_to_predict($nextrange) && (empty($end) || $next < $end)) {
+            // Add the next one if we have not reached the analysable end yet.
+            // It will be used to get predictions.
+            $ranges[] = $nextrange;
+        }
+
+        return $ranges;
+    }
+
+    /**
+     * Overwritten as all generated rows are comparable.
+     *
+     * @return bool
+     */
+    public function include_range_info_in_training_data() {
+        return false;
+    }
+
+    /**
+     * Overwritting as the last range may be for prediction.
+     *
+     * @return array
+     */
+    public function get_training_ranges() {
+        // Cloning the array.
+        $trainingranges = $this->ranges;
+
+        foreach ($trainingranges as $rangeindex => $range) {
+            if (!$this->ready_to_train($range)) {
+                unset($trainingranges[$rangeindex]);
+            }
+        }
+
+        return $trainingranges;
+    }
+
+    /**
+     * The next range is based on the past period.
+     *
+     * @param  \DateTimeImmutable $next
+     * @return array
+     */
+    protected function get_next_range(\DateTimeImmutable $next) {
+
+        $end = $next->getTimestamp();
+        $start = $next->sub($this->periodicity())->getTimestamp();
+
+        if ($start < $this->analysable->get_start()) {
+            // We skip the first range generated as its start is prior to the analysable start.
+            return false;
+        }
+
+        return [
+            'start' => $start,
+            'end' => $end,
+            'time' => $end
+        ];
+    }
+}

analytics/classes/local/time_splitting/upcoming_periodic.php

@@ -0,0 +1,62 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Time splitting method that generates predictions periodically.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics\local\time_splitting;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Time splitting method that generates predictions periodically.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+abstract class upcoming_periodic extends periodic {
+
+    /**
+     * The next range indicator calculations should be based on upcoming dates.
+     *
+     * @param  \DateTimeImmutable $next
+     * @return array
+     */
+    protected function get_next_range(\DateTimeImmutable $next) {
+
+        $start = $next->getTimestamp();
+        $end = $next->add($this->periodicity())->getTimestamp();
+        return [
+            'start' => $start,
+            'end' => $end,
+            'time' => $start
+        ];
+    }
+
+    /**
+     * Whether to cache or not the indicator calculations.
+     * @return bool
+     */
+    public function cache_indicator_calculations(): bool {
+        return false;
+    }
+}

analytics/classes/manager.php

@@ -80,10 +80,23 @@ class manager {
      *
      * @throws \required_capability_exception
      * @param \context $context
+     * @param  bool $return The method returns a bool if true.
      * @return void
      */
-    public static function check_can_list_insights(\context $context) {
-        require_capability('moodle/analytics:listinsights', $context);
+    public static function check_can_list_insights(\context $context, bool $return = false) {
+        global $USER;
+
+        if ($context->contextlevel === CONTEXT_USER && $context->instanceid == $USER->id) {
+            $capability = 'moodle/analytics:listowninsights';
+        } else {
+            $capability = 'moodle/analytics:listinsights';
+        }
+
+        if ($return) {
+            return has_capability($capability, $context);
+        } else {
+            require_capability($capability, $context);
+        }
     }
 
     /**
@@ -552,15 +565,19 @@ class manager {
         $models = self::get_all_models();
         foreach ($models as $model) {
             $analyser = $model->get_analyser(array('notimesplitting' => true));
-            $analysables = $analyser->get_analysables();
-            if (!$analysables) {
+            $analysables = $analyser->get_analysables_iterator();
+
+            $analysableids = [];
+            foreach ($analysables as $analysable) {
+                if (!$analysable) {
+                    continue;
+                }
+                $analysableids[] = $analysable->get_id();
+            }
+            if (empty($analysableids)) {
                 continue;
             }
 
-            $analysableids = array_map(function($analysable) {
-                return $analysable->get_id();
-            }, $analysables);
-
             list($notinsql, $params) = $DB->get_in_or_equal($analysableids, SQL_PARAMS_NAMED, 'param', false);
             $params['modelid'] = $model->get_id();
 

analytics/classes/model.php

@@ -306,12 +306,6 @@ class model {
             }
         }
 
-        if (!empty($options['evaluation'])) {
-            foreach ($timesplittings as $timesplitting) {
-                $timesplitting->set_evaluating(true);
-            }
-        }
-
         $classname = $target->get_analyser_class();
         if (!class_exists($classname)) {
             throw new \coding_exception($classname . ' class does not exists');
@@ -541,7 +535,7 @@ class model {
             $this->get_analyser()->add_log(get_string('noevaluationbasedassumptions', 'analytics'));
             $result = new \stdClass();
             $result->status = self::NO_DATASET;
-            return array($this->get_time_splitting()->get_id() => $result);
+            return array($result);
         }
 
         $options['evaluation'] = true;
@@ -736,50 +730,67 @@ class model {
         // Before get_unlabelled_data call so we get an early exception if it is not writable.
         $outputdir = $this->get_output_dir(array('execution'));
 
-        // Before get_unlabelled_data call so we get an early exception if it is not ready.
         if (!$this->is_static()) {
+            // Predictions using a machine learning backend.
+
+            // Before get_unlabelled_data call so we get an early exception if it is not ready.
             $predictor = $this->get_predictions_processor();
-        }
 
-        $samplesdata = $this->get_analyser()->get_unlabelled_data();
+            $samplesdata = $this->get_analyser()->get_unlabelled_data();
 
-        // Get the prediction samples file.
-        if (empty($samplesdata) || empty($samplesdata[$this->model->timesplitting])) {
+            // Get the prediction samples file.
+            if (empty($samplesdata) || empty($samplesdata[$this->model->timesplitting])) {
 
-            $result = new \stdClass();
-            $result->status = self::NO_DATASET;
-            $result->info = $this->get_analyser()->get_logs();
-            return $result;
-        }
-        $samplesfile = $samplesdata[$this->model->timesplitting];
+                $result = new \stdClass();
+                $result->status = self::NO_DATASET;
+                $result->info = $this->get_analyser()->get_logs();
+                return $result;
+            }
+            $samplesfile = $samplesdata[$this->model->timesplitting];
 
-        // We need to throw an exception if we are trying to predict stuff that was already predicted.
-        $params = array('modelid' => $this->model->id, 'action' => 'predicted', 'fileid' => $samplesfile->get_id());
-        if ($predicted = $DB->get_record('analytics_used_files', $params)) {
-            throw new \moodle_exception('erroralreadypredict', 'analytics', '', $samplesfile->get_id());
-        }
+            // We need to throw an exception if we are trying to predict stuff that was already predicted.
+            $params = array('modelid' => $this->model->id, 'action' => 'predicted', 'fileid' => $samplesfile->get_id());
+            if ($predicted = $DB->get_record('analytics_used_files', $params)) {
+                throw new \moodle_exception('erroralreadypredict', 'analytics', '', $samplesfile->get_id());
+            }
 
-        $indicatorcalculations = \core_analytics\dataset_manager::get_structured_data($samplesfile);
+            $indicatorcalculations = \core_analytics\dataset_manager::get_structured_data($samplesfile);
 
-        // Prepare the results object.
-        $result = new \stdClass();
-
-        if ($this->is_static()) {
-            // Prediction based on assumptions.
-            $result->status = self::OK;
-            $result->info = [];
-            $result->predictions = $this->get_static_predictions($indicatorcalculations);
-
-        } else {
             // Estimation and classification processes run on the machine learning backend side.
             if ($this->get_target()->is_linear()) {
                 $predictorresult = $predictor->estimate($this->get_unique_id(), $samplesfile, $outputdir);
             } else {
                 $predictorresult = $predictor->classify($this->get_unique_id(), $samplesfile, $outputdir);
             }
+
+            // Prepare the results object.
+            $result = new \stdClass();
             $result->status = $predictorresult->status;
             $result->info = $predictorresult->info;
             $result->predictions = $this->format_predictor_predictions($predictorresult);
+
+        } else {
+            // Predictions based on assumptions.
+
+            $indicatorcalculations = $this->get_analyser()->get_static_data();
+            // Get the prediction samples file.
+            if (empty($indicatorcalculations) || empty($indicatorcalculations[$this->model->timesplitting])) {
+
+                $result = new \stdClass();
+                $result->status = self::NO_DATASET;
+                $result->info = $this->get_analyser()->get_logs();
+                return $result;
+            }
+
+            // Same as reset($indicatorcalculations) as models based on assumptions only analyse 1 single
+            // time-splitting method.
+            $indicatorcalculations = $indicatorcalculations[$this->model->timesplitting];
+
+            // Prepare the results object.
+            $result = new \stdClass();
+            $result->status = self::OK;
+            $result->info = [];
+            $result->predictions = $this->get_static_predictions($indicatorcalculations);
         }
 
         if ($result->status !== self::OK) {
@@ -787,14 +798,17 @@ class model {
         }
 
         if ($result->predictions) {
-            $samplecontexts = $this->execute_prediction_callbacks($result->predictions, $indicatorcalculations);
+            list($samplecontexts, $predictionrecords) = $this->execute_prediction_callbacks($result->predictions,
+                $indicatorcalculations);
         }
 
         if (!empty($samplecontexts) && $this->uses_insights()) {
-            $this->trigger_insights($samplecontexts);
+            $this->trigger_insights($samplecontexts, $predictionrecords);
         }
 
-        $this->flag_file_as_used($samplesfile, 'predicted');
+        if (!$this->is_static()) {
+            $this->flag_file_as_used($samplesfile, 'predicted');
+        }
 
         return $result;
     }
@@ -855,7 +869,7 @@ class model {
      * @param array $indicatorcalculations
      * @return array
      */
-    protected function execute_prediction_callbacks($predictions, $indicatorcalculations) {
+    protected function execute_prediction_callbacks(&$predictions, $indicatorcalculations) {
 
         // Here we will store all predictions' contexts, this will be used to limit which users will see those predictions.
         $samplecontexts = array();
@@ -865,7 +879,6 @@ class model {
 
             // The unique sample id contains both the sampleid and the rangeindex.
             list($sampleid, $rangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid);
-
             if ($this->get_target()->triggers_callback($prediction->prediction, $prediction->predictionscore)) {
 
                 // Prepare the record to store the predicted values.
@@ -887,19 +900,38 @@ class model {
             $this->save_predictions($records);
         }
 
-        return $samplecontexts;
+        return [$samplecontexts, $records];
     }
 
     /**
      * Generates insights and updates the cache.
      *
      * @param \context[] $samplecontexts
+     * @param  \stdClass[] $predictionrecords
      * @return void
      */
-    protected function trigger_insights($samplecontexts) {
+    protected function trigger_insights($samplecontexts, $predictionrecords) {
 
         // Notify the target that all predictions have been processed.
-        $this->get_target()->generate_insight_notifications($this->model->id, $samplecontexts);
+        if ($this->get_analyser()::one_sample_per_analysable()) {
+
+            // We need to do something unusual here. self::save_predictions uses the bulk-insert function (insert_records()) for
+            // performance reasons and that function does not return us the inserted ids. We need to retrieve them from
+            // the database, and we need to do it using one single database query (for performance reasons as well).
+            $predictionrecords = $this->add_prediction_ids($predictionrecords);
+
+            // Get \core_analytics\prediction objects also fetching the samplesdata. This costs us
+            // 1 db read, but we have to pay it if we want that our insights include links to the
+            // suggested actions.
+            $predictions = array_map(function($predictionobj) {
+                $prediction = new \core_analytics\prediction($predictionobj, $this->prediction_sample_data($predictionobj));
+                return $prediction;
+            }, $predictionrecords);
+        } else {
+            $predictions = [];
+        }
+
+        $this->get_target()->generate_insight_notifications($this->model->id, $samplecontexts, $predictions);
 
         // Update cache.
         $cache = \cache::make('core', 'contextwithinsights');
@@ -926,12 +958,23 @@ class model {
      */
     protected function get_static_predictions(&$indicatorcalculations) {
 
+        $headers = array_shift($indicatorcalculations);
+
+        // Get rid of the sampleid header.
+        array_shift($headers);
+
         // Group samples by analysable for \core_analytics\local\target::calculate.
         $analysables = array();
         // List all sampleids together.
         $sampleids = array();
 
         foreach ($indicatorcalculations as $uniquesampleid => $indicators) {
+
+            // Get rid of the sampleid column.
+            unset($indicators[0]);
+            $indicators = array_combine($headers, $indicators);
+            $indicatorcalculations[$uniquesampleid] = $indicators;
+
             list($sampleid, $rangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid);
 
             $analysable = $this->get_analyser()->get_sample_analysable($sampleid);
@@ -946,6 +989,7 @@ class model {
                     'sampleids' => array()
                 ];
             }
+
             // Using the sampleid as a key so we can easily merge indicators data later.
             $analysables[$analysableclass][$rangeindex]->indicatorsdata[$sampleid] = $indicators;
             // We could use indicatorsdata keys but the amount of redundant data is not that big and leaves code below cleaner.
@@ -969,16 +1013,20 @@ class model {
                 $this->get_target()->add_sample_data($data->indicatorsdata);
 
                 // Append new elements (we can not get duplicates because sample-analysable relation is N-1).
-                $range = $this->get_time_splitting()->get_range_by_index($rangeindex);
+                $timesplitting = $this->get_time_splitting();
+                $timesplitting->set_analysable($data->analysable);
+                $range = $timesplitting->get_range_by_index($rangeindex);
+
                 $this->get_target()->filter_out_invalid_samples($data->sampleids, $data->analysable, false);
                 $calculations = $this->get_target()->calculate($data->sampleids, $data->analysable, $range['start'], $range['end']);
 
                 // Missing $indicatorcalculations values in $calculations are caused by is_valid_sample. We need to remove
                 // these $uniquesampleid from $indicatorcalculations because otherwise they will be stored as calculated
                 // by self::save_prediction.
-                $indicatorcalculations = array_filter($indicatorcalculations, function($indicators, $uniquesampleid) use ($calculations) {
-                    list($sampleid, $rangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid);
-                    if (!isset($calculations[$sampleid])) {
+                $indicatorcalculations = array_filter($indicatorcalculations, function($indicators, $uniquesampleid)
+                        use ($calculations, $rangeindex) {
+                    list($sampleid, $indicatorsrangeindex) = $this->get_time_splitting()->infer_sample_info($uniquesampleid);
+                    if ($rangeindex == $indicatorsrangeindex && !isset($calculations[$sampleid])) {
                         return false;
                     }
                     return true;
@@ -1642,6 +1690,44 @@ class model {
         $DB->update_record('analytics_models', $this->model);
     }
 
+    /**
+     * Adds the id from {analytics_predictions} db table to the prediction \stdClass objects.
+     *
+     * @param  \stdClass[] $predictionrecords
+     * @return \stdClass[] The prediction records including their ids in {analytics_predictions} db table.
+     */
+    private function add_prediction_ids($predictionrecords) {
+        global $DB;
+
+        $firstprediction = reset($predictionrecords);
+
+        $contextids = array_map(function($predictionobj) {
+            return $predictionobj->contextid;
+        }, $predictionrecords);
+        list($contextsql, $contextparams) = $DB->get_in_or_equal($contextids, SQL_PARAMS_NAMED);
+
+        // We select the fields that will allow us to map ids to $predictionrecords. Given that we already filter by modelid
+        // we have enough with sampleid and rangeindex. The reason is that the sampleid relation to a site is N - 1.
+        $fields = 'id, sampleid, rangeindex';
+
+        // We include the contextid and the timecreated filter to reduce the number of records in $dbpredictions. We can not
+        // add as many OR conditions as records in $predictionrecords.
+        $sql = "SELECT $fields
+                  FROM {analytics_predictions}
+                 WHERE modelid = :modelid
+                       AND contextid $contextsql
+                       AND timecreated >= :firsttimecreated";
+        $params = $contextparams + ['modelid' => $this->model->id, 'firsttimecreated' => $firstprediction->timecreated];
+        $dbpredictions = $DB->get_recordset_sql($sql, $params);
+        foreach ($dbpredictions as $id => $dbprediction) {
+            // The append_rangeindex implementation is the same regardless of the time splitting method in use.
+            $uniqueid = $this->get_time_splitting()->append_rangeindex($dbprediction->sampleid, $dbprediction->rangeindex);
+            $predictionrecords[$uniqueid]->id = $dbprediction->id;
+        }
+
+        return $predictionrecords;
+    }
+
     /**
      * Purges the insights cache.
      */

analytics/classes/prediction_action.php

@@ -40,6 +40,11 @@ class prediction_action {
      */
     protected $actionname = null;
 
+    /**
+     * @var \moodle_url
+     */
+    protected $url = null;
+
     /**
      * @var \action_menu_link
      */
@@ -50,7 +55,7 @@ class prediction_action {
      *
      * @param string $actionname They should match a-zA-Z_0-9-, as we apply a PARAM_ALPHANUMEXT filter
      * @param \core_analytics\prediction $prediction
-     * @param \moodle_url $actionurl
+     * @param \moodle_url $actionurl The final URL where the user should be forwarded.
      * @param \pix_icon $icon Link icon
      * @param string $text Link text
      * @param bool $primary Primary button or secondary.
@@ -61,16 +66,17 @@ class prediction_action {
                                 $text, $primary = false, $attributes = array()) {
 
         $this->actionname = $actionname;
+        $this->text = $text;
 
         // We want to track how effective are our suggested actions, we pass users through a script that will log these actions.
         $params = array('action' => $this->actionname, 'predictionid' => $prediction->get_prediction_data()->id,
             'forwardurl' => $actionurl->out(false));
-        $url = new \moodle_url('/report/insights/action.php', $params);
+        $this->url = new \moodle_url('/report/insights/action.php', $params);
 
         if ($primary === false) {
-            $this->actionlink = new \action_menu_link_secondary($url, $icon, $text, $attributes);
+            $this->actionlink = new \action_menu_link_secondary($this->url, $icon, $this->text, $attributes);
         } else {
-            $this->actionlink = new \action_menu_link_primary($url, $icon, $text, $attributes);
+            $this->actionlink = new \action_menu_link_primary($this->url, $icon, $this->text, $attributes);
         }
     }
 
@@ -83,6 +89,15 @@ class prediction_action {
         return $this->actionname;
     }
 
+    /**
+     * Returns the url to the action.
+     *
+     * @return \moodle_url
+     */
+    public function get_url() {
+        return $this->url;
+    }
+
     /**
      * Returns the link to the action.
      *
@@ -91,4 +106,12 @@ class prediction_action {
     public function get_action_link() {
         return $this->actionlink;
     }
+
+    /**
+     * Returns the action text.
+     * @return string
+     */
+    public function get_text() {
+        return $this->text;
+    }
 }

analytics/classes/user.php

@@ -0,0 +1,207 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Moodle user analysable
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_analytics;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Moodle user analysable
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class user implements \core_analytics\analysable {
+
+    /**
+     * @var bool Has this user data been already loaded.
+     */
+    protected $loaded = false;
+
+    /**
+     * @var int $cachedid self::$cachedinstance analysable id.
+     */
+    protected static $cachedid = 0;
+
+    /**
+     * @var \core_analytics\user $cachedinstance
+     */
+    protected static $cachedinstance = null;
+
+    /**
+     * User object
+     *
+     * @var \stdClass
+     */
+    protected $user = null;
+
+    /**
+     * The user context.
+     *
+     * @var \context_user
+     */
+    protected $usercontext = null;
+
+    /**
+     * Constructor.
+     *
+     * Use self::instance() instead to get cached copies of the class. Instances obtained
+     * through this constructor will not be cached.
+     *
+     * @param int|\stdClass $user User id
+     * @param \context|null $context
+     * @return void
+     */
+    public function __construct($user, ?\context $context = null) {
+
+        if (is_scalar($user)) {
+            $this->user = new \stdClass();
+            $this->user->id = $user;
+        } else {
+            $this->user = $user;
+        }
+
+        if (!is_null($context)) {
+            $this->usercontext = $context;
+        }
+    }
+
+    /**
+     * Returns an analytics user instance.
+     *
+     * Lazy load of analysable data.
+     *
+     * @param int|\stdClass $user User object or user id
+     * @param \context|null $context
+     * @return \core_analytics\user
+     */
+    public static function instance($user, ?\context $context = null) {
+
+        $userid = $user;
+        if (!is_scalar($userid)) {
+            $userid = $user->id;
+        }
+
+        if (self::$cachedid === $userid) {
+            return self::$cachedinstance;
+        }
+
+        $cachedinstance = new \core_analytics\user($user, $context);
+        self::$cachedinstance = $cachedinstance;
+        self::$cachedid = (int)$userid;
+        return self::$cachedinstance;
+    }
+
+    /**
+     * get_id
+     *
+     * @return int
+     */
+    public function get_id() {
+        return $this->user->id;
+    }
+
+    /**
+     * Loads the analytics user object.
+     *
+     * @return void
+     */
+    protected function load() {
+
+        // The instance constructor could be already loaded with the full user object. Using email
+        // because it is a required user field.
+        if (empty($this->user->email)) {
+            $this->user = \core_user::get_user($this->user->id);
+        }
+
+        $this->usercontext = $this->get_context();
+
+        $this->now = time();
+
+        // Flag the instance as loaded.
+        $this->loaded = true;
+    }
+
+    /**
+     * The user full name.
+     *
+     * @return string
+     */
+    public function get_name() {
+
+        if (!$this->loaded) {
+            $this->load();
+        }
+        return fullname($this->user);
+    }
+
+    /**
+     * get_context
+     *
+     * @return \context
+     */
+    public function get_context() {
+        if ($this->usercontext === null) {
+            $this->usercontext = \context_user::instance($this->user->id);
+        }
+        return $this->usercontext;
+    }
+
+    /**
+     * Get the start timestamp.
+     *
+     * @return int
+     */
+    public function get_start() {
+
+        if (!$this->loaded) {
+            $this->load();
+        }
+        return $this->user->timecreated;
+    }
+
+    /**
+     * Get the end timestamp.
+     *
+     * @return int
+     */
+    public function get_end() {
+        return self::MAX_TIME;
+    }
+
+    /**
+     * Returns a user plain object.
+     *
+     * @return \stdClass
+     */
+    public function get_user_data() {
+
+        if (!$this->loaded) {
+            $this->load();
+        }
+
+        return $this->user;
+    }
+}

analytics/templates/insight_info_message.mustache

@@ -0,0 +1,60 @@
+{{!
+    This file is part of Moodle - http://moodle.org/
+
+    Moodle is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Moodle is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+}}
+{{!
+    @template core_analytics/insight_info_message
+
+    HTML message for insights
+
+    Classes required for JS:
+    * none
+
+    Data attributes required for JS:
+    * none
+
+    Example context (json):
+    {
+        "url": "https://moodle.org"
+    }
+}}
+<style>
+
+{{! Default btn-default styles. These styles are not applied to Moodle's web UI as there is a body:not(.dir-ltr):not(.dir-rtl)}}
+
+body:not(.dir-ltr):not(.dir-rtl) .btn-insight {
+    margin-bottom: 1rem!important;
+    margin-right: 1rem!important;
+    color: #212529;
+    background-color: #e9ecef;
+    border-color: #e9ecef
+    display: inline-block;
+    font-weight: 400;
+    text-align: center;
+    white-space: nowrap;
+    vertical-align: middle;
+    user-select: none;
+    border: 1px solid transparent;
+    padding: .375rem .75rem;
+    font-size: .9375rem;
+    line-height: 1.5;
+    border-radius: .25rem;
+    text-decoration: none;
+}
+</style>
+
+{{#str}} insightinfomessagehtml, analytics {{/str}}
+<br/><br/>
+<a class="btn btn-default btn-insight" href="{{url}}">{{#str}} viewinsight, analytics {{/str}}</a>

analytics/templates/insight_info_message_prediction.mustache

@@ -0,0 +1,69 @@
+{{!
+    This file is part of Moodle - http://moodle.org/
+
+    Moodle is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Moodle is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+}}
+{{!
+    @template core_analytics/insight_info_message_prediction
+
+    HTML message for insights with one single prediction
+
+    Classes required for JS:
+    * none
+
+    Data attributes required for JS:
+    * none
+
+    Example context (json):
+    {
+        "actions": [
+            {
+                "url": "https://moodle.org",
+                "text": "Moodle"
+            }, {
+                "url": "https://en.wikipedia.org/wiki/Noodle",
+                "text": "Noodle"
+            }
+        ]
+    }
+}}
+
+{{! Default btn-default styles. These styles are not applied to Moodle's web UI as there is a body:not(.dir-ltr):not(.dir-rtl)}}
+
+<style>
+body:not(.dir-ltr):not(.dir-rtl) .btn-insight {
+    margin-bottom: 1rem!important;
+    margin-right: 1rem!important;
+    color: #212529;
+    background-color: #e9ecef;
+    border-color: #e9ecef
+    display: inline-block;
+    font-weight: 400;
+    text-align: center;
+    white-space: nowrap;
+    vertical-align: middle;
+    user-select: none;
+    border: 1px solid transparent;
+    padding: .375rem .75rem;
+    font-size: .9375rem;
+    line-height: 1.5;
+    border-radius: .25rem;
+    text-decoration: none;
+}
+</style>
+
+<br/>
+{{#actions}}
+    <a class="btn btn-default m-r-1 m-b-1 btn-insight" href="{{url}}">{{text}}</a>
+{{/actions}}

analytics/tests/dataset_manager_test.php

@@ -56,10 +56,8 @@ class dataset_manager_testcase extends advanced_testcase {
     public function test_create_dataset() {
 
         $dataset1 = new \core_analytics\dataset_manager(1, 1, 'whatever', \core_analytics\dataset_manager::LABELLED_FILEAREA, false);
-        $dataset1->init_process();
         $dataset1data = array_merge($this->sharedtoprows, array(array('yeah', 'yeah', 'yeah')));
         $f1 = $dataset1->store($dataset1data);
-        $dataset1->close_process();
 
         $f1contents = $f1->get_content();
         $this->assertContains('yeah', $f1contents);
@@ -76,16 +74,12 @@ class dataset_manager_testcase extends advanced_testcase {
     public function test_merge_datasets() {
 
         $dataset1 = new \core_analytics\dataset_manager(1, 1, 'whatever', \core_analytics\dataset_manager::LABELLED_FILEAREA, false);
-        $dataset1->init_process();
         $dataset1data = array_merge($this->sharedtoprows, array(array('yeah', 'yeah', 'yeah')));
         $f1 = $dataset1->store($dataset1data);
-        $dataset1->close_process();
 
         $dataset2 = new \core_analytics\dataset_manager(1, 2, 'whatever', \core_analytics\dataset_manager::LABELLED_FILEAREA, false);
-        $dataset2->init_process();
         $dataset2data = array_merge($this->sharedtoprows, array(array('no', 'no', 'no')));
         $f2 = $dataset2->store($dataset2data);
-        $dataset2->close_process();
 
         $files = array($f1, $f2);
         $merged = \core_analytics\dataset_manager::merge_datasets($files, 1, 'whatever',
@@ -123,10 +117,8 @@ class dataset_manager_testcase extends advanced_testcase {
         // merged into training and prediction files).
         $analysabledataset = new \core_analytics\dataset_manager($fakemodelid, 1, 'whatever',
             \core_analytics\dataset_manager::LABELLED_FILEAREA, false);
-        $analysabledataset->init_process();
         $analysabledatasetdata = array_merge($this->sharedtoprows, array(array('yeah', 'yeah', 'yeah')));
         $file = $analysabledataset->store($analysabledatasetdata);
-        $analysabledataset->close_process();
 
         // Evaluation files ignored.
         $evaluationdataset = \core_analytics\dataset_manager::merge_datasets(array($file), $fakemodelid,

analytics/tests/fixtures/test_analysis.php

@@ -31,18 +31,17 @@ defined('MOODLE_INTERNAL') || die();
  * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
-class test_analyser extends \core\analytics\analyser\courses {
+class test_analysis extends \core_analytics\analysis {
 
     /**
      * Overwritten to add a delay.
      *
      * @param \core_analytics\analysable $analysable
-     * @param mixed $includetarget
-     * @return null
+     * @return array
      */
-    public function process_analysable($analysable, $includetarget) {
-        // A bit more than 1 second.
-        usleep(1100000);
-        return parent::process_analysable($analysable, $includetarget);
+    public function process_analysable(\core_analytics\analysable $analysable): array {
+        // Half a second.
+        usleep(500000);
+        return parent::process_analysable($analysable);
     }
 }

analytics/tests/fixtures/test_indicator_null.php

@@ -0,0 +1,62 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Test indicator. Always null.
+ *
+ * @package   core_analytics
+ * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Test indicator. Always null.
+ *
+ * @package   core_analytics
+ * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class test_indicator_null extends \core_analytics\local\indicator\binary {
+
+    /**
+     * Returns a lang_string object representing the name for the indicator.
+     *
+     * Used as column identificator.
+     *
+     * If there is a corresponding '_help' string this will be shown as well.
+     *
+     * @return \lang_string
+     */
+    public static function get_name() : \lang_string {
+        // Using a string that exists and contains a corresponding '_help' string.
+        return new \lang_string('allowstealthmodules');
+    }
+
+    /**
+     * calculate_sample
+     *
+     * @param int $sampleid
+     * @param string $samplesorigin
+     * @param int $starttime
+     * @param int $endtime
+     * @return float
+     */
+    protected function calculate_sample($sampleid, $samplesorigin, $starttime, $endtime) {
+        return null;
+    }
+}

analytics/tests/fixtures/test_site_users_analyser.php

@@ -77,7 +77,7 @@ class test_site_users_analyser extends \core_analytics\local\analyser\sitewide {
      * @param \core_analytics\analysable $site
      * @return array
      */
-    protected function get_all_samples(\core_analytics\analysable $site) {
+    public function get_all_samples(\core_analytics\analysable $site) {
         global $DB;
 
         $users = $DB->get_records('user');
@@ -121,7 +121,7 @@ class test_site_users_analyser extends \core_analytics\local\analyser\sitewide {
      * @return array array(string, \renderable)
      */
     public function sample_description($sampleid, $contextid, $sampledata) {
-        $description = fullname($samplesdata['user']);
+        $description = fullname($sampledata['user']);
         $userimage = new \pix_icon('i/user', get_string('user'));
         return array($description, $userimage);
     }

analytics/tests/fixtures/test_target_shortname.php

@@ -87,7 +87,7 @@ class test_target_shortname extends \core_analytics\local\target\binary {
      * We don't want to discard results.
      * @return array
      */
-    protected function ignored_predicted_classes() {
+    public function ignored_predicted_classes() {
         return array();
     }
 

analytics/tests/fixtures/test_target_site_users.php

@@ -89,7 +89,7 @@ class test_target_site_users extends \core_analytics\local\target\binary {
      * We don't want to discard results.
      * @return array
      */
-    protected function ignored_predicted_classes() {
+    public function ignored_predicted_classes() {
         return array();
     }
 

analytics/tests/fixtures/test_timesplitting_seconds.php

@@ -0,0 +1,52 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Test time splitting.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Test time splitting.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllaó {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class test_timesplitting_seconds extends \core_analytics\local\time_splitting\periodic {
+
+    /**
+     * Every second.
+     * @return \DateInterval
+     */
+    public function periodicity() {
+        return new \DateInterval('PT1S');
+    }
+
+    /**
+     * Just to comply with the interface.
+     *
+     * @return \lang_string
+     */
+    public static function get_name() : \lang_string {
+        return new \lang_string('error');
+    }
+}

analytics/tests/fixtures/test_timesplitting_weekly.php

@@ -0,0 +1,51 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Time splitting method that generates weekly predictions.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Time splitting method that generates weekly predictions.
+ *
+ * @package   core_analytics
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class test_timesplitting_weekly extends \core_analytics\local\time_splitting\periodic {
+
+    /**
+     * The time splitting method name.
+     * @return \lang_string
+     */
+    public static function get_name() : \lang_string {
+        return new \lang_string('error');
+    }
+
+    /**
+     * Once per week.
+     * @return \DateInterval
+     */
+    public function periodicity() {
+        return new \DateInterval('P1W');
+    }
+}

analytics/tests/manager_test.php

@@ -348,9 +348,11 @@ class analytics_manager_testcase extends advanced_testcase {
 
         $noteaching = \core_analytics\manager::get_target('\core\analytics\target\no_teaching');
         $dropout = \core_analytics\manager::get_target('\core\analytics\target\course_dropout');
+        $upcomingactivities = \core_analytics\manager::get_target('\core_user\analytics\target\upcoming_activities_due');
 
         $this->assertTrue(\core_analytics\model::exists($noteaching));
         $this->assertTrue(\core_analytics\model::exists($dropout));
+        $this->assertTrue(\core_analytics\model::exists($upcomingactivities));
 
         foreach (\core_analytics\manager::get_all_models() as $model) {
             $model->delete();
@@ -358,14 +360,17 @@ class analytics_manager_testcase extends advanced_testcase {
 
         $this->assertFalse(\core_analytics\model::exists($noteaching));
         $this->assertFalse(\core_analytics\model::exists($dropout));
+        $this->assertFalse(\core_analytics\model::exists($upcomingactivities));
 
         $updated = \core_analytics\manager::update_default_models_for_component('moodle');
 
-        $this->assertEquals(2, count($updated));
+        $this->assertEquals(3, count($updated));
+        $this->assertTrue(array_pop($updated) instanceof \core_analytics\model);
         $this->assertTrue(array_pop($updated) instanceof \core_analytics\model);
         $this->assertTrue(array_pop($updated) instanceof \core_analytics\model);
         $this->assertTrue(\core_analytics\model::exists($noteaching));
         $this->assertTrue(\core_analytics\model::exists($dropout));
+        $this->assertTrue(\core_analytics\model::exists($upcomingactivities));
 
         $repeated = \core_analytics\manager::update_default_models_for_component('moodle');
 

analytics/tests/model_test.php

@@ -30,7 +30,7 @@ require_once(__DIR__ . '/fixtures/test_indicator_fullname.php');
 require_once(__DIR__ . '/fixtures/test_target_shortname.php');
 require_once(__DIR__ . '/fixtures/test_static_target_shortname.php');
 require_once(__DIR__ . '/fixtures/test_target_course_level_shortname.php');
-require_once(__DIR__ . '/fixtures/test_analyser.php');
+require_once(__DIR__ . '/fixtures/test_analysis.php');
 
 /**
  * Unit tests for the model.
@@ -292,7 +292,7 @@ class analytics_model_testcase extends advanced_testcase {
 
         $this->resetAfterTest(true);
 
-        set_config('modeltimelimit', 2, 'analytics');
+        set_config('modeltimelimit', 1, 'analytics');
 
         $courses = array();
         for ($i = 0; $i < 5; $i++) {
@@ -302,19 +302,22 @@ class analytics_model_testcase extends advanced_testcase {
         }
 
         $target = new test_target_course_level_shortname();
-        $analyser = new test_analyser(1, $target, [], [], []);
+        $analyser = new \core\analytics\analyser\courses(1, $target, [], [], []);
 
-        // Each analysable element takes 1.1 secs, so the max (and likely) number of analysable
+        $result = new \core_analytics\local\analysis\result_array(1, false, []);
+        $analysis = new test_analysis($analyser, false, $result);
+
+        // Each analysable element takes 0.5 secs minimum (test_analysis), so the max (and likely) number of analysable
         // elements that will be processed is 2.
-        $analyser->get_analysable_data(false);
+        $analysis->run();
         $params = array('modelid' => 1, 'action' => 'prediction');
         $this->assertLessThanOrEqual(2, $DB->count_records('analytics_used_analysables', $params));
 
-        $analyser->get_analysable_data(false);
+        $analysis->run();
         $this->assertLessThanOrEqual(4, $DB->count_records('analytics_used_analysables', $params));
 
         // Check that analysable elements have been processed following the analyser order
-        // (course->sortorder here). We can not check this nicely after next get_analysable_data round
+        // (course->sortorder here). We can not check this nicely after next get_unlabelled_data round
         // because the first analysed element will be analysed again.
         $analysedelems = $DB->get_records('analytics_used_analysables', $params, 'timeanalysed ASC');
         // Just a default for the first checked element.
@@ -326,16 +329,21 @@ class analytics_model_testcase extends advanced_testcase {
             $last = $courses[$analysed->analysableid];
         }
 
-        $analyser->get_analysable_data(false);
-        $this->assertGreaterThanOrEqual(5, $DB->count_records('analytics_used_analysables', $params));
+        // No time limit now to process the rest.
+        set_config('modeltimelimit', 1000, 'analytics');
+
+        $analysis->run();
+        $this->assertEquals(5, $DB->count_records('analytics_used_analysables', $params));
 
         // New analysable elements are immediately pulled.
         $this->getDataGenerator()->create_course();
-        $analyser->get_analysable_data(false);
-        $this->assertGreaterThanOrEqual(6, $DB->count_records('analytics_used_analysables', $params));
+        $analysis->run();
+        $this->assertEquals(6, $DB->count_records('analytics_used_analysables', $params));
 
         // Training and prediction data do not get mixed.
-        $analyser->get_analysable_data(true);
+        $result = new \core_analytics\local\analysis\result_array(1, false, []);
+        $analysis = new test_analysis($analyser, false, $result);
+        $analysis->run();
         $params = array('modelid' => 1, 'action' => 'training');
         $this->assertLessThanOrEqual(2, $DB->count_records('analytics_used_analysables', $params));
     }

analytics/tests/prediction_test.php

@@ -27,6 +27,7 @@ defined('MOODLE_INTERNAL') || die();
 global $CFG;
 require_once(__DIR__ . '/fixtures/test_indicator_max.php');
 require_once(__DIR__ . '/fixtures/test_indicator_min.php');
+require_once(__DIR__ . '/fixtures/test_indicator_null.php');
 require_once(__DIR__ . '/fixtures/test_indicator_fullname.php');
 require_once(__DIR__ . '/fixtures/test_indicator_random.php');
 require_once(__DIR__ . '/fixtures/test_target_shortname.php');
@@ -86,18 +87,14 @@ class core_analytics_prediction_testcase extends advanced_testcase {
         // 1 range for each analysable.
         $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id()));
         $this->assertCount(2, $predictedranges);
-        $this->assertEquals(1, $DB->count_records('analytics_used_files',
-            array('modelid' => $model->get_id(), 'action' => 'predicted')));
         // 2 predictions for each range.
         $this->assertEquals(2, $DB->count_records('analytics_predictions',
             array('modelid' => $model->get_id())));
 
-        // No new generated files nor records as there are no new courses available.
+        // No new generated records as there are no new courses available.
         $model->predict();
         $predictedranges = $DB->get_records('analytics_predict_samples', array('modelid' => $model->get_id()));
         $this->assertCount(2, $predictedranges);
-        $this->assertEquals(1, $DB->count_records('analytics_used_files',
-            array('modelid' => $model->get_id(), 'action' => 'predicted')));
         $this->assertEquals(2, $DB->count_records('analytics_predictions',
             array('modelid' => $model->get_id())));
     }
@@ -553,8 +550,7 @@ class core_analytics_prediction_testcase extends advanced_testcase {
     public function test_not_null_samples() {
         $this->resetAfterTest(true);
 
-        $classname = '\core\analytics\time_splitting\quarters';
-        $timesplitting = \core_analytics\manager::get_time_splitting($classname);
+        $timesplitting = \core_analytics\manager::get_time_splitting('\core\analytics\time_splitting\quarters');
         $timesplitting->set_analysable(new \core_analytics\site());
 
         $ranges = array(
@@ -563,35 +559,49 @@ class core_analytics_prediction_testcase extends advanced_testcase {
         );
         $samples = array(123 => 123, 321 => 321);
 
-        $indicator1 = $this->getMockBuilder('test_indicator_max')
-            ->setMethods(['calculate_sample'])
-            ->getMock();
-        $indicator1->method('calculate_sample')
-            ->willReturn(null);
+        $target = \core_analytics\manager::get_target('test_target_shortname');
+        $indicators = array('test_indicator_null', 'test_indicator_min');
+        foreach ($indicators as $key => $indicator) {
+            $indicators[$key] = \core_analytics\manager::get_indicator($indicator);
+        }
+        $model = \core_analytics\model::create($target, $indicators, '\core\analytics\time_splitting\no_splitting');
 
-        $indicator2 = \core_analytics\manager::get_indicator('test_indicator_min');
+        $analyser = $model->get_analyser();
+        $result = new \core_analytics\local\analysis\result_array($model->get_id(), false, $analyser->get_options());
+        $analysis = new \core_analytics\analysis($analyser, false, $result);
 
         // Samples with at least 1 not null value are returned.
         $params = array(
+            $timesplitting,
             $samples,
-            'whatever',
-            array($indicator1, $indicator2),
             $ranges
         );
-        $dataset = phpunit_util::call_internal_method($timesplitting, 'calculate_indicators', $params, $classname);
+        $dataset = phpunit_util::call_internal_method($analysis, 'calculate_indicators', $params,
+            '\core_analytics\analysis');
         $this->assertArrayHasKey('123-0', $dataset);
         $this->assertArrayHasKey('123-1', $dataset);
         $this->assertArrayHasKey('321-0', $dataset);
         $this->assertArrayHasKey('321-1', $dataset);
 
+
+        $indicators = array('test_indicator_null');
+        foreach ($indicators as $key => $indicator) {
+            $indicators[$key] = \core_analytics\manager::get_indicator($indicator);
+        }
+        $model = \core_analytics\model::create($target, $indicators, '\core\analytics\time_splitting\no_splitting');
+
+        $analyser = $model->get_analyser();
+        $result = new \core_analytics\local\analysis\result_array($model->get_id(), false, $analyser->get_options());
+        $analysis = new \core_analytics\analysis($analyser, false, $result);
+
         // Samples with only null values are not returned.
         $params = array(
+            $timesplitting,
             $samples,
-            'whatever',
-            array($indicator1),
             $ranges
         );
-        $dataset = phpunit_util::call_internal_method($timesplitting, 'calculate_indicators', $params, $classname);
+        $dataset = phpunit_util::call_internal_method($analysis, 'calculate_indicators', $params,
+            '\core_analytics\analysis');
         $this->assertArrayNotHasKey('123-0', $dataset);
         $this->assertArrayNotHasKey('123-1', $dataset);
         $this->assertArrayNotHasKey('321-0', $dataset);

analytics/tests/privacy_test.php

@@ -34,7 +34,6 @@ require_once(__DIR__ . '/fixtures/test_indicator_max.php');
 require_once(__DIR__ . '/fixtures/test_indicator_min.php');
 require_once(__DIR__ . '/fixtures/test_target_site_users.php');
 require_once(__DIR__ . '/fixtures/test_target_course_users.php');
-require_once(__DIR__ . '/fixtures/test_analyser.php');
 
 /**
  * Unit tests for privacy.

analytics/upgrade.txt

@@ -13,6 +13,36 @@ information provided here is intended especially for developers.
   has been replaced with automatic update of models provided by the core moodle component. There
   is no need to call this method explicitly any more. Instead, adding new models can be achieved
   by updating the lib/db/analytics.php file and bumping the core version.
+* \core_analytics\model::execute_prediction_callbacks now returns an array with both sample's contexts
+  and the prediction records.
+* Time splitting methods:
+    * \core_analytics\local\time_splitting\base::append_rangeindex and
+      \core_analytics\local\time_splitting\base::infer_sample_info are now marked as final and can not
+      be overwritten.
+    * Can now overwrite include_range_info_in_training_data() and
+      get_training_ranges() methods. They can be used to create time splitting methods with a pre-defined
+      number of ranges.
+    * Can now overwrite cache_indicator_calculations(). You should return false if the time frames generated
+      by your time-splitting method are unique and / or can hardly be reused by further models.
+    * \core_analytics\local\analyser\base::get_most_recent_prediction_range has been moved to
+      \core_analytics\local\time_splitting\base::get_most_recent_prediction_range and it is not overwritable
+      by time splitting methods.
+* Targets:
+    * The visibility of the following methods must now be public: ignored_predicted_classes()
+      and get_insights_users()
+    * Prediction_actions() has now a 3rd parameter $isinsightuser. This parameter is true
+      when we are listing actions for the user that will receives the insight.
+    * Can now implement a always_update_analysis_time() method so analysable elements' timeanalysed is
+      only updated when analysable elements have been successfully evaluated. It is useful for lightweight targets.
+    * Can not implement two new methods to tune the insights generated by the model: get_insight_subject()
+      and get_insight_context_url().
+* Analysers:
+    * The visibility of get_all_samples() method must now be public.
+    * get_analysables() method has been deprecated in favour of a new get_analysables_interator()
+      for performance reasons.
+    * Can overwrite a new one_sample_per_analysable() method if the analysables they use only have
+      one sample. The insights generated by models will then include the suggested actions in
+      the notification.
 
 === 3.5 ===
 

backup/backup.class.php

@@ -74,6 +74,12 @@ abstract class backup implements checksumable {
     const MODE_AUTOMATED = 50;
     const MODE_CONVERTED = 60;
 
+    /**
+     * This mode is for asynchronous backups.
+     * These backups will run via adhoc scheduled tasks.
+     */
+    const MODE_ASYNC = 70;
+
     // Target (new/existing/current/adding/deleting)
     const TARGET_CURRENT_DELETING = 0;
     const TARGET_CURRENT_ADDING   = 1;

backup/backup.php

@@ -30,16 +30,27 @@ require_once('../config.php');
 require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 require_once($CFG->dirroot . '/backup/moodle2/backup_plan_builder.class.php');
 
+// Backup of large courses requires extra memory. Use the amount configured
+// in admin settings.
+raise_memory_limit(MEMORY_EXTRA);
 
 $courseid = required_param('id', PARAM_INT);
 $sectionid = optional_param('section', null, PARAM_INT);
 $cmid = optional_param('cm', null, PARAM_INT);
 $cancel      = optional_param('cancel', '', PARAM_ALPHA);
+$previous = optional_param('previous', false, PARAM_BOOL);
 /**
  * Part of the forms in stages after initial, is POST never GET
  */
 $backupid = optional_param('backup', false, PARAM_ALPHANUM);
 
+// Determine if we are performing realtime for asynchronous backups.
+$backupmode = backup::MODE_GENERAL;
+if (async_helper::is_async_enabled()) {
+    $backupmode = backup::MODE_ASYNC;
+}
+
+$courseurl = new moodle_url('/course/view.php', array('id' => $courseid));
 $url = new moodle_url('/backup/backup.php', array('id'=>$courseid));
 if ($sectionid !== null) {
     $url->param('section', $sectionid);
@@ -53,6 +64,8 @@ $PAGE->set_pagelayout('admin');
 $id = $courseid;
 $cm = null;
 $course = $DB->get_record('course', array('id'=>$courseid), '*', MUST_EXIST);
+$coursecontext = context_course::instance($course->id);
+$contextid = $coursecontext->id;
 $type = backup::TYPE_1COURSE;
 if (!is_null($sectionid)) {
     $section = $DB->get_record('course_sections', array('course'=>$course->id, 'id'=>$sectionid), '*', MUST_EXIST);
@@ -68,11 +81,10 @@ require_login($course, false, $cm);
 
 switch ($type) {
     case backup::TYPE_1COURSE :
-        require_capability('moodle/backup:backupcourse', context_course::instance($course->id));
+        require_capability('moodle/backup:backupcourse', $coursecontext);
         $heading = get_string('backupcourse', 'backup', $course->shortname);
         break;
     case backup::TYPE_1SECTION :
-        $coursecontext = context_course::instance($course->id);
         require_capability('moodle/backup:backupsection', $coursecontext);
         if ((string)$section->name !== '') {
             $sectionname = format_string($section->name, true, array('context' => $coursecontext));
@@ -84,102 +96,145 @@ switch ($type) {
         }
         break;
     case backup::TYPE_1ACTIVITY :
-        require_capability('moodle/backup:backupactivity', context_module::instance($cm->id));
+        $activitycontext = context_module::instance($cm->id);
+        require_capability('moodle/backup:backupactivity', $activitycontext);
+        $contextid = $activitycontext->id;
         $heading = get_string('backupactivity', 'backup', $cm->name);
         break;
     default :
         print_error('unknownbackuptype');
 }
 
-// Backup of large courses requires extra memory. Use the amount configured
-// in admin settings.
-raise_memory_limit(MEMORY_EXTRA);
-
-if (!($bc = backup_ui::load_controller($backupid))) {
-    $bc = new backup_controller($type, $id, backup::FORMAT_MOODLE,
-                            backup::INTERACTIVE_YES, backup::MODE_GENERAL, $USER->id);
-}
-$backup = new backup_ui($bc);
-
 $PAGE->set_title($heading);
 $PAGE->set_heading($heading);
 
-$renderer = $PAGE->get_renderer('core','backup');
 if (empty($cancel)) {
     // Do not print the header if user cancelled the process, as we are going to redirect the user.
     echo $OUTPUT->header();
 }
 
-// Prepare a progress bar which can display optionally during long-running
-// operations while setting up the UI.
-$slowprogress = new \core\progress\display_if_slow(get_string('preparingui', 'backup'));
+// Only let user perform a backup if we aren't in async mode, or if we are
+// and there are no pending backups for this item for this user.
+if (!async_helper::is_async_pending($id, 'course', 'backup')) {
 
-$previous = optional_param('previous', false, PARAM_BOOL);
-if ($backup->get_stage() == backup_ui::STAGE_SCHEMA && !$previous) {
-    // After schema stage, we are probably going to get to the confirmation stage,
-    // The confirmation stage has 2 sets of progress, so this is needed to prevent
-    // it showing 2 progress bars.
-    $twobars = true;
-    $slowprogress->start_progress('', 2);
-} else {
-    $twobars = false;
-}
-$backup->get_controller()->set_progress($slowprogress);
-$backup->process();
+    // The mix of business logic and display elements below makes me sad.
+    // This needs to refactored into the renderer and seperated out.
 
-if ($backup->enforce_changed_dependencies()) {
-    debugging('Your settings have been altered due to unmet dependencies', DEBUG_DEVELOPER);
-}
-
-$loghtml = '';
-if ($backup->get_stage() == backup_ui::STAGE_FINAL) {
-    // Display an extra backup step bar so that we can show the 'processing' step first.
-    echo html_writer::start_div('', array('id' => 'executionprogress'));
-    echo $renderer->progress_bar($backup->get_progress_bar());
-    $backup->get_controller()->set_progress(new \core\progress\display());
-
-    // Prepare logger and add to end of chain.
-    $logger = new core_backup_html_logger($CFG->debugdeveloper ? backup::LOG_DEBUG : backup::LOG_INFO);
-    $backup->get_controller()->add_logger($logger);
-
-    // Carry out actual backup.
-    $backup->execute();
-
-    // Backup controller gets saved/loaded so the logger object changes and we
-    // have to retrieve it.
-    $logger = $backup->get_controller()->get_logger();
-    while (!is_a($logger, 'core_backup_html_logger')) {
-        $logger = $logger->get_next();
+    if (!($bc = backup_ui::load_controller($backupid))) {
+        $bc = new backup_controller($type, $id, backup::FORMAT_MOODLE,
+                backup::INTERACTIVE_YES, $backupmode, $USER->id);
     }
 
-    // Get HTML from logger.
-    if ($CFG->debugdisplay) {
-        $loghtml = $logger->get_html();
+    // Prepare a progress bar which can display optionally during long-running
+    // operations while setting up the UI.
+    $slowprogress = new \core\progress\display_if_slow(get_string('preparingui', 'backup'));
+    $renderer = $PAGE->get_renderer('core', 'backup');
+    $backup = new backup_ui($bc);
+
+    if ($backup->get_stage() == backup_ui::STAGE_SCHEMA && !$previous) {
+        // After schema stage, we are probably going to get to the confirmation stage,
+        // The confirmation stage has 2 sets of progress, so this is needed to prevent
+        // it showing 2 progress bars.
+        $twobars = true;
+        $slowprogress->start_progress('', 2);
+    } else {
+        $twobars = false;
+    }
+    $backup->get_controller()->set_progress($slowprogress);
+    $backup->process();
+
+    if ($backup->enforce_changed_dependencies()) {
+        debugging('Your settings have been altered due to unmet dependencies', DEBUG_DEVELOPER);
     }
 
-    // Hide the progress display and first backup step bar (the 'finished' step will show next).
-    echo html_writer::end_div();
-    echo html_writer::script('document.getElementById("executionprogress").style.display = "none";');
-} else {
-    $backup->save_controller();
-}
+    $loghtml = '';
+    if ($backup->get_stage() == backup_ui::STAGE_FINAL) {
 
-// Displaying UI can require progress reporting, so do it here before outputting
-// the backup stage bar (as part of the existing progress bar, if required).
-$ui = $backup->display($renderer);
-if ($twobars) {
-    $slowprogress->end_progress();
-}
+        if ($backupmode != backup::MODE_ASYNC) {
+            // Synchronous backup handling.
 
-echo $renderer->progress_bar($backup->get_progress_bar());
+            // Display an extra backup step bar so that we can show the 'processing' step first.
+            echo html_writer::start_div('', array('id' => 'executionprogress'));
+            echo $renderer->progress_bar($backup->get_progress_bar());
+            $backup->get_controller()->set_progress(new \core\progress\display());
 
-echo $ui;
-$backup->destroy();
-unset($backup);
+            // Prepare logger and add to end of chain.
+            $logger = new core_backup_html_logger($CFG->debugdeveloper ? backup::LOG_DEBUG : backup::LOG_INFO);
+            $backup->get_controller()->add_logger($logger);
 
-// Display log data if there was any.
-if ($loghtml != '') {
-    echo $renderer->log_display($loghtml);
+            // Carry out actual backup.
+            $backup->execute();
+
+            // Backup controller gets saved/loaded so the logger object changes and we
+            // have to retrieve it.
+            $logger = $backup->get_controller()->get_logger();
+            while (!is_a($logger, 'core_backup_html_logger')) {
+                $logger = $logger->get_next();
+            }
+
+            // Get HTML from logger.
+            if ($CFG->debugdisplay) {
+                $loghtml = $logger->get_html();
+            }
+
+            // Hide the progress display and first backup step bar (the 'finished' step will show next).
+            echo html_writer::end_div();
+            echo html_writer::script('document.getElementById("executionprogress").style.display = "none";');
+        } else {
+            // Async backup handling.
+            $backup->get_controller()->finish_ui();
+
+            echo html_writer::start_div('', array('id' => 'executionprogress'));
+            echo $renderer->progress_bar($backup->get_progress_bar());
+            echo html_writer::end_div();
+
+            // Create adhoc task for backup.
+            $asynctask = new \core\task\asynchronous_backup_task();
+            $asynctask->set_blocking(false);
+            $asynctask->set_custom_data(array('backupid' => $backupid));
+            \core\task\manager::queue_adhoc_task($asynctask);
+
+            // Add ajax progress bar and initiate ajax via a template.
+            $restoreurl = new moodle_url('/backup/restorefile.php', array('contextid' => $contextid));
+            $progresssetup = array(
+                    'backupid' => $backupid,
+                    'contextid' => $contextid,
+                    'courseurl' => $courseurl->out(),
+                    'restoreurl' => $restoreurl->out(),
+                    'headingident' => 'backup'
+            );
+            echo $renderer->render_from_template('core/async_backup_status', $progresssetup);
+        }
+
+    } else {
+        $backup->save_controller();
+    }
+
+    if ($backup->get_stage() != backup_ui::STAGE_FINAL) {
+
+        // Displaying UI can require progress reporting, so do it here before outputting
+        // the backup stage bar (as part of the existing progress bar, if required).
+        $ui = $backup->display($renderer);
+        if ($twobars) {
+            $slowprogress->end_progress();
+        }
+
+        echo $renderer->progress_bar($backup->get_progress_bar());
+        echo $ui;
+
+        // Display log data if there was any.
+        if ($loghtml != '' && $backupmode != backup::MODE_ASYNC) {
+            echo $renderer->log_display($loghtml);
+        }
+    }
+
+    $backup->destroy();
+    unset($backup);
+
+} else { // User has a pending async operation.
+    echo $OUTPUT->notification(get_string('pendingasyncerror', 'backup'), 'error');
+    echo $OUTPUT->container(get_string('pendingasyncdetail', 'backup'));
+    echo $OUTPUT->continue_button($courseurl);
 }
 
 echo $OUTPUT->footer();

backup/controller/backup_controller.class.php

@@ -58,7 +58,11 @@ class backup_controller extends base_controller {
     protected $plan;   // Backup execution plan
     protected $includefiles; // Whether this backup includes files or not.
 
-    protected $execution;     // inmediate/delayed
+    /**
+     * Immediate/delayed execution type.
+     * @var integer
+     */
+    protected $execution;
     protected $executiontime; // epoch time when we want the backup to be executed (requires cron to run)
 
     protected $destination; // Destination chain object (fs_moodle, fs_os, db, email...)
@@ -85,11 +89,17 @@ class backup_controller extends base_controller {
         $this->userid = $userid;
 
         // Apply some defaults
-        $this->execution = backup::EXECUTION_INMEDIATE;
         $this->operation = backup::OPERATION_BACKUP;
         $this->executiontime = 0;
         $this->checksum = '';
 
+        // Set execution based on backup mode.
+        if ($mode == backup::MODE_ASYNC) {
+            $this->execution = backup::EXECUTION_DELAYED;
+        } else {
+            $this->execution = backup::EXECUTION_INMEDIATE;
+        }
+
         // Apply current backup version and release if necessary
         backup_controller_dbops::apply_version_and_release();
 
@@ -112,7 +122,7 @@ class backup_controller extends base_controller {
         // display progress must set it.
         $this->progress = new \core\progress\none();
 
-        // Instantiate the output_controller singleton and active it if interactive and inmediate
+        // Instantiate the output_controller singleton and active it if interactive and immediate.
         $oc = output_controller::get_instance();
         if ($this->interactive == backup::INTERACTIVE_YES && $this->execution == backup::EXECUTION_INMEDIATE) {
             $oc->set_active(true);
@@ -182,7 +192,8 @@ class backup_controller extends base_controller {
         // TODO: Check it's a correct status.
         $this->status = $status;
         // Ensure that, once set to backup::STATUS_AWAITING, controller is stored in DB.
-        if ($status == backup::STATUS_AWAITING) {
+        // Also save if executing so we can better track progress.
+        if ($status == backup::STATUS_AWAITING || $status == backup::STATUS_EXECUTING) {
             $this->save_controller();
             $tbc = self::load_controller($this->backupid);
             $this->logger = $tbc->logger; // wakeup loggers
@@ -192,14 +203,18 @@ class backup_controller extends base_controller {
             // If the operation has ended without error (backup::STATUS_FINISHED_OK)
             // proceed by cleaning the object from database. MDL-29262.
             $this->save_controller(false, true);
+        } else if ($status == backup::STATUS_FINISHED_ERR) {
+            // If the operation has ended with an error save the controller
+            // preserving the object in the database. We may want it for debugging.
+            $this->save_controller();
         }
     }
 
     public function set_execution($execution, $executiontime = 0) {
         $this->log('setting controller execution', backup::LOG_DEBUG);
-        // TODO: Check valid execution mode
-        // TODO: Check time in future
-        // TODO: Check time = 0 if inmediate
+        // TODO: Check valid execution mode.
+        // TODO: Check time in future.
+        // TODO: Check time = 0 if immediate.
         $this->execution = $execution;
         $this->executiontime = $executiontime;
 
@@ -333,8 +348,8 @@ class backup_controller extends base_controller {
      * @param bool $cleanobj to decide if the object itself must be cleaned (true) or no (false)
      */
     public function save_controller($includeobj = true, $cleanobj = false) {
-        // Going to save controller to persistent storage, calculate checksum for later checks and save it
-        // TODO: flag the controller as NA. Any operation on it should be forbidden util loaded back
+        // Going to save controller to persistent storage, calculate checksum for later checks and save it.
+        // TODO: flag the controller as NA. Any operation on it should be forbidden until loaded back.
         $this->log('saving controller to db', backup::LOG_DEBUG);
         if ($includeobj ) {  // Only calculate checksum if we are going to include the object.
             $this->checksum = $this->calculate_checksum();
@@ -399,6 +414,7 @@ class backup_controller extends base_controller {
         $this->log("setting file inclusion to {$this->includefiles}", backup::LOG_DEBUG);
         return $this->includefiles;
     }
+
 }
 
 /*

backup/controller/restore_controller.class.php

@@ -53,7 +53,11 @@ class restore_controller extends base_controller {
     /** @var restore_plan */
     protected $plan;   // Restore execution plan
 
-    protected $execution;     // inmediate/delayed
+    /**
+     * Immediate/delayed execution type.
+     * @var integer
+     */
+    protected $execution;
     protected $executiontime; // epoch time when we want the restore to be executed (requires cron to run)
 
     protected $checksum; // Cache @checksumable results for lighter @is_checksum_correct() uses
@@ -88,7 +92,6 @@ class restore_controller extends base_controller {
         // Apply some defaults
         $this->type = '';
         $this->format = backup::FORMAT_UNKNOWN;
-        $this->execution = backup::EXECUTION_INMEDIATE;
         $this->operation = backup::OPERATION_RESTORE;
         $this->executiontime = 0;
         $this->samesite = false;
@@ -110,6 +113,13 @@ class restore_controller extends base_controller {
         // Default logger chain (based on interactive/execution)
         $this->logger = backup_factory::get_logger_chain($this->interactive, $this->execution, $this->restoreid);
 
+        // Set execution based on backup mode.
+        if ($mode == backup::MODE_ASYNC) {
+            $this->execution = backup::EXECUTION_DELAYED;
+        } else {
+            $this->execution = backup::EXECUTION_INMEDIATE;
+        }
+
         // By default there is no progress reporter unless you specify one so it
         // can be used during loading of the plan.
         if ($progress) {
@@ -119,7 +129,7 @@ class restore_controller extends base_controller {
         }
         $this->progress->start_progress('Constructing restore_controller');
 
-        // Instantiate the output_controller singleton and active it if interactive and inmediate
+        // Instantiate the output_controller singleton and active it if interactive and immediate.
         $oc = output_controller::get_instance();
         if ($this->interactive == backup::INTERACTIVE_YES && $this->execution == backup::EXECUTION_INMEDIATE) {
             $oc->set_active(true);
@@ -198,7 +208,8 @@ class restore_controller extends base_controller {
         // TODO: Check it's a correct status.
         $this->status = $status;
         // Ensure that, once set to backup::STATUS_AWAITING | STATUS_NEED_PRECHECK, controller is stored in DB.
-        if ($status == backup::STATUS_AWAITING || $status == backup::STATUS_NEED_PRECHECK) {
+        // Also save if executing so we can better track progress.
+        if ($status == backup::STATUS_AWAITING || $status == backup::STATUS_NEED_PRECHECK || $status == backup::STATUS_EXECUTING) {
             $this->save_controller();
             $tbc = self::load_controller($this->restoreid);
             $this->logger = $tbc->logger; // wakeup loggers
@@ -208,14 +219,18 @@ class restore_controller extends base_controller {
             // If the operation has ended without error (backup::STATUS_FINISHED_OK)
             // proceed by cleaning the object from database. MDL-29262.
             $this->save_controller(false, true);
+        } else if ($status == backup::STATUS_FINISHED_ERR) {
+            // If the operation has ended with an error save the controller
+            // preserving the object in the database. We may want it for debugging.
+            $this->save_controller();
         }
     }
 
     public function set_execution($execution, $executiontime = 0) {
         $this->log('setting controller execution', backup::LOG_DEBUG);
-        // TODO: Check valid execution mode
-        // TODO: Check time in future
-        // TODO: Check time = 0 if inmediate
+        // TODO: Check valid execution mode.
+        // TODO: Check time in future.
+        // TODO: Check time = 0 if immediate.
         $this->execution = $execution;
         $this->executiontime = $executiontime;
 

backup/externallib.php

@@ -0,0 +1,240 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+
+/**
+ * External backup API.
+ *
+ * @package    core_backup
+ * @category   external
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die;
+
+require_once("$CFG->libdir/externallib.php");
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+
+/**
+ * Backup external functions.
+ *
+ * @package    core_backup
+ * @category   external
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ * @since Moodle 3.7
+ */
+class core_backup_external extends external_api {
+
+    /**
+     * Returns description of method parameters
+     *
+     * @return external_function_parameters
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_progress_parameters() {
+        return new external_function_parameters(
+            array(
+                'backupids' => new external_multiple_structure(
+                        new external_value(PARAM_ALPHANUM, 'Backup id to get progress for', VALUE_REQUIRED, null, NULL_ALLOWED),
+                        'Backup id to get progress for', VALUE_REQUIRED
+                 ),
+                'contextid' => new external_value(PARAM_INT, 'Context id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED),
+            )
+        );
+    }
+
+    /**
+     * Get asynchronous backup progress.
+     *
+     * @param string $backupids The ids of the backup to get progress for.
+     * @param int $contextid The context the backup relates to.
+     * @return array $results The array of results.
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_progress($backupids, $contextid) {
+        global $CFG;
+        require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+        require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php');
+
+        // Parameter validation.
+        self::validate_parameters(
+                self::get_async_backup_progress_parameters(),
+                array(
+                    'backupids' => $backupids,
+                    'contextid' => $contextid
+                )
+        );
+
+        // Context validation.
+        list($context, $course, $cm) = get_context_info_array($contextid);
+        self::validate_context($context);
+
+        if ($cm) {
+            require_capability('moodle/backup:backupactivity', $context);
+        } else {
+            require_capability('moodle/backup:backupcourse', $context);
+            $instanceid = $course->id;
+        }
+
+        $results = array();
+        foreach ($backupids as $backupid) {
+            $results[] = backup_controller_dbops::get_progress($backupid);
+        }
+
+        return $results;
+    }
+
+    /**
+     * Returns description of method result value
+     *
+     * @return external_description
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_progress_returns() {
+        return new external_multiple_structure(
+            new external_single_structure(
+                array(
+                    'status'   => new external_value(PARAM_INT, 'Backup Status'),
+                    'progress' => new external_value(PARAM_FLOAT, 'Backup progress'),
+                    'backupid' => new external_value(PARAM_ALPHANUM, 'Backup id'),
+                    'operation' => new external_value(PARAM_ALPHANUM, 'operation type'),
+                ), 'Backup completion status'
+          ), 'Backup data'
+        );
+    }
+
+    /**
+     * Returns description of method parameters
+     *
+     * @return external_function_parameters
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_links_backup_parameters() {
+        return new external_function_parameters(
+                array(
+                    'filename' => new external_value(PARAM_FILE, 'Backup filename', VALUE_REQUIRED, null, NULL_NOT_ALLOWED),
+                    'contextid' => new external_value(PARAM_INT, 'Context id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED),
+                )
+         );
+    }
+
+    /**
+     * Get the data to be used when generating the table row for an asynchronous backup,
+     * the table row updates via ajax when backup is complete.
+     *
+     * @param string $filename The file name of the backup file.
+     * @param int $contextid The context the backup relates to.
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_links_backup($filename, $contextid) {
+        // Parameter validation.
+        self::validate_parameters(
+                self::get_async_backup_links_backup_parameters(),
+                    array(
+                        'filename' => $filename,
+                        'contextid' => $contextid
+                    )
+                );
+
+        // Context validation.
+        list($context, $course, $cm) = get_context_info_array($contextid);
+        self::validate_context($context);
+        require_capability('moodle/backup:backupcourse', $context);
+
+        if ($cm) {
+            $filearea = 'activity';
+        } else {
+            $filearea = 'course';
+        }
+
+        $results = \async_helper::get_backup_file_info($filename, $filearea, $contextid);
+
+        return $results;
+    }
+
+    /**
+     * Returns description of method result value.
+     *
+     * @return external_description
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_links_backup_returns() {
+        return new external_single_structure(
+            array(
+               'filesize'   => new external_value(PARAM_TEXT, 'Backup file size'),
+               'fileurl' => new external_value(PARAM_URL, 'Backup file URL'),
+               'restoreurl' => new external_value(PARAM_URL, 'Backup restore URL'),
+        ), 'Table row data.');
+    }
+    /**
+     * Returns description of method parameters
+     *
+     * @return external_function_parameters
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_links_restore_parameters() {
+        return new external_function_parameters(
+                array(
+                    'backupid' => new external_value(PARAM_ALPHANUMEXT, 'Backup id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED),
+                    'contextid' => new external_value(PARAM_INT, 'Context id', VALUE_REQUIRED, null, NULL_NOT_ALLOWED),
+                )
+        );
+    }
+
+    /**
+     * Get the data to be used when generating the table row for an asynchronous restore,
+     * the table row updates via ajax when restore is complete.
+     *
+     * @param string $backupid The id of the backup record.
+     * @param int $contextid The context the restore relates to.
+     * @return array $results The array of results.
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_links_restore($backupid, $contextid) {
+        // Parameter validation.
+        self::validate_parameters(
+                self::get_async_backup_links_restore_parameters(),
+                    array(
+                        'backupid' => $backupid,
+                        'contextid' => $contextid
+                    )
+                );
+
+        // Context validation.
+        $context = context::instance_by_id($contextid);
+        self::validate_context($context);
+        require_capability('moodle/restore:restorecourse', $context);
+
+        $results = \async_helper::get_restore_url($backupid);
+
+        return $results;
+    }
+
+    /**
+     * Returns description of method result value.
+     *
+     * @return external_description
+     * @since Moodle 3.7
+     */
+    public static function get_async_backup_links_restore_returns() {
+        return new external_single_structure(
+                array(
+                    'restoreurl' => new external_value(PARAM_URL, 'Restore url'),
+                ), 'Table row data.');
+    }
+}

backup/restore.php

@@ -28,10 +28,20 @@ define('NO_OUTPUT_BUFFERING', true);
 require_once('../config.php');
 require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php');
 
+// Restore of large courses requires extra memory. Use the amount configured
+// in admin settings.
+raise_memory_limit(MEMORY_EXTRA);
+
 $contextid   = required_param('contextid', PARAM_INT);
 $stage       = optional_param('stage', restore_ui::STAGE_CONFIRM, PARAM_INT);
 $cancel      = optional_param('cancel', '', PARAM_ALPHA);
 
+// Determine if we are performing realtime for asynchronous backups.
+$backupmode = backup::MODE_GENERAL;
+if (async_helper::is_async_enabled()) {
+    $backupmode = backup::MODE_ASYNC;
+}
+
 list($context, $course, $cm) = get_context_info_array($contextid);
 
 navigation_node::override_active_url(new moodle_url('/backup/restorefile.php', array('contextid'=>$contextid)));
@@ -70,10 +80,6 @@ $slowprogress->start_progress('', 10);
 // This progress section counts for loading the restore controller.
 $slowprogress->start_progress('', 1, 1);
 
-// Restore of large courses requires extra memory. Use the amount configured
-// in admin settings.
-raise_memory_limit(MEMORY_EXTRA);
-
 if ($stage & restore_ui::STAGE_CONFIRM + restore_ui::STAGE_DESTINATION) {
     $restore = restore_ui::engage_independent_stage($stage, $contextid);
 } else {
@@ -83,7 +89,7 @@ if ($stage & restore_ui::STAGE_CONFIRM + restore_ui::STAGE_DESTINATION) {
         $restore = restore_ui::engage_independent_stage($stage/2, $contextid);
         if ($restore->process()) {
             $rc = new restore_controller($restore->get_filepath(), $restore->get_course_id(), backup::INTERACTIVE_YES,
-                                backup::MODE_GENERAL, $USER->id, $restore->get_target());
+                    $backupmode, $USER->id, $restore->get_target());
         }
     }
     if ($rc) {
@@ -120,7 +126,7 @@ if (!$restore->is_independent()) {
     // Use a temporary (disappearing) progress bar to show the precheck progress if any.
     $precheckprogress = new \core\progress\display_if_slow(get_string('preparingdata', 'backup'));
     $restore->get_controller()->set_progress($precheckprogress);
-    if ($restore->get_stage() == restore_ui::STAGE_PROCESS && !$restore->requires_substage()) {
+    if ($restore->get_stage() == restore_ui::STAGE_PROCESS && !$restore->requires_substage() && $backupmode != backup::MODE_ASYNC) {
         try {
             // Div used to hide the 'progress' step once the page gets onto 'finished'.
             echo html_writer::start_div('', array('id' => 'executionprogress'));
@@ -150,7 +156,35 @@ if (!$restore->is_independent()) {
 }
 
 echo $renderer->progress_bar($restore->get_progress_bar());
-echo $restore->display($renderer);
+
+if ($restore->get_stage() != restore_ui::STAGE_PROCESS) {
+    echo $restore->display($renderer);
+} else if ($restore->get_stage() == restore_ui::STAGE_PROCESS && $restore->requires_substage()) {
+    echo $restore->display($renderer);
+} else if ($restore->get_stage() == restore_ui::STAGE_PROCESS
+        && !$restore->requires_substage()
+        && $backupmode == backup::MODE_ASYNC) {
+    // Asynchronous restore.
+    // Create adhoc task for restore.
+    $restoreid = $restore->get_restoreid();
+    $asynctask = new \core\task\asynchronous_restore_task();
+    $asynctask->set_blocking(false);
+    $asynctask->set_custom_data(array('backupid' => $restoreid));
+    \core\task\manager::queue_adhoc_task($asynctask);
+
+    // Add ajax progress bar and initiate ajax via a template.
+    $courseurl = new moodle_url('/course/view.php', array('id' => $course->id));
+    $restoreurl = new moodle_url('/backup/restorefile.php', array('contextid' => $contextid));
+    $progresssetup = array(
+            'backupid' => $restoreid,
+            'contextid' => $contextid,
+            'courseurl' => $courseurl->out(),
+            'restoreurl' => $restoreurl->out()
+    );
+    echo $renderer->render_from_template('core/async_backup_status', $progresssetup);
+
+}
+
 $restore->destroy();
 unset($restore);
 

backup/restorefile.php

@@ -114,6 +114,7 @@ $PAGE->set_context($context);
 $PAGE->set_title(get_string('course') . ': ' . $coursefullname);
 $PAGE->set_heading($heading);
 $PAGE->set_pagelayout('admin');
+$PAGE->requires->js_call_amd('core_backup/async_backup', 'asyncBackupAllStatus', array($context->id));
 
 $form = new course_restore_form(null, array('contextid'=>$contextid));
 $data = $form->get_data();
@@ -128,8 +129,6 @@ if ($data && has_capability('moodle/restore:uploadfile', $context)) {
     die;
 }
 
-
-
 echo $OUTPUT->header();
 
 // require uploadfile cap to use file picker
@@ -196,4 +195,13 @@ if (!empty($automatedbackups)) {
     echo $OUTPUT->container_end();
 }
 
+// In progress course restores.
+if (async_helper::is_async_enabled()) {
+    echo $OUTPUT->heading_with_help(get_string('asyncrestoreinprogress', 'backup'), 'asyncrestoreinprogress', 'backup');
+    echo $OUTPUT->container_start();
+    $renderer = $PAGE->get_renderer('core', 'backup');
+    echo $renderer->restore_progress_viewer($USER->id, $context);
+    echo $OUTPUT->container_end();
+}
+
 echo $OUTPUT->footer();

backup/tests/async_backup_test.php

@@ -0,0 +1,118 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Asyncronhous backup tests.
+ *
+ * @package    core_backup
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php');
+require_once($CFG->libdir . '/completionlib.php');
+
+/**
+ * Asyncronhous backup tests.
+ *
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class core_backup_async_backup_testcase extends \core_privacy\tests\provider_testcase {
+
+    /**
+     * Tests the asynchronous backup.
+     */
+    public function test_async_backup() {
+        global $DB, $CFG, $USER;
+
+        $this->resetAfterTest(true);
+        $this->setAdminUser();
+        $CFG->enableavailability = true;
+        $CFG->enablecompletion = true;
+
+        // Create a course with some availability data set.
+        $generator = $this->getDataGenerator();
+        $course = $generator->create_course(
+                array('format' => 'topics', 'numsections' => 3,
+                        'enablecompletion' => COMPLETION_ENABLED),
+                array('createsections' => true));
+        $forum = $generator->create_module('forum', array(
+                'course' => $course->id));
+        $forum2 = $generator->create_module('forum', array(
+                'course' => $course->id, 'completion' => COMPLETION_TRACKING_MANUAL));
+
+        // We need a grade, easiest is to add an assignment.
+        $assignrow = $generator->create_module('assign', array(
+                'course' => $course->id));
+        $assign = new assign(context_module::instance($assignrow->cmid), false, false);
+        $item = $assign->get_grade_item();
+
+        // Make a test grouping as well.
+        $grouping = $generator->create_grouping(array('courseid' => $course->id,
+                'name' => 'Grouping!'));
+
+        $availability = '{"op":"|","show":false,"c":[' .
+                '{"type":"completion","cm":' . $forum2->cmid .',"e":1},' .
+                '{"type":"grade","id":' . $item->id . ',"min":4,"max":94},' .
+                '{"type":"grouping","id":' . $grouping->id . '}' .
+                ']}';
+        $DB->set_field('course_modules', 'availability', $availability, array(
+                'id' => $forum->cmid));
+        $DB->set_field('course_sections', 'availability', $availability, array(
+                'course' => $course->id, 'section' => 1));
+
+        // Start backup process.
+
+        // Make the backup controller for an async backup.
+        $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE,
+                backup::INTERACTIVE_YES, backup::MODE_ASYNC, $USER->id);
+        $bc->finish_ui();
+        $backupid = $bc->get_backupid();
+
+        $prebackuprec = $DB->get_record('backup_controllers', array('backupid' => $backupid));
+
+        // Check the initial backup controller was created correctly.
+        $this->assertEquals(backup::STATUS_AWAITING, $prebackuprec->status);
+        $this->assertEquals(2, $prebackuprec->execution);
+
+        // Create the adhoc task.
+        $asynctask = new \core\task\asynchronous_backup_task();
+        $asynctask->set_blocking(false);
+        $asynctask->set_custom_data(array('backupid' => $backupid));
+        \core\task\manager::queue_adhoc_task($asynctask);
+
+        // We are expecting trace output during this test.
+        $this->expectOutputRegex("/$backupid/");
+
+        // Execute adhoc task.
+        $now = time();
+        $task = \core\task\manager::get_next_adhoc_task($now);
+        $this->assertInstanceOf('\\core\\task\\asynchronous_backup_task', $task);
+        $task->execute();
+        \core\task\manager::adhoc_task_complete($task);
+
+        $postbackuprec = $DB->get_record('backup_controllers', array('backupid' => $backupid));
+
+        // Check backup was created successfully.
+        $this->assertEquals(backup::STATUS_FINISHED_OK, $postbackuprec->status);
+        $this->assertEquals(1.0, $postbackuprec->progress);
+    }
+}

backup/tests/async_restore_test.php

@@ -0,0 +1,140 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Asyncronhous restore tests.
+ *
+ * @package    core_backup
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php');
+require_once($CFG->libdir . '/completionlib.php');
+
+/**
+ * Asyncronhous restore tests.
+ *
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class core_backup_async_restore_testcase extends \core_privacy\tests\provider_testcase {
+
+    /**
+     * Tests the asynchronous backup.
+     */
+    public function test_async_restore() {
+        global $DB, $CFG, $USER;
+
+        $this->resetAfterTest(true);
+        $this->setAdminUser();
+        $CFG->enableavailability = true;
+        $CFG->enablecompletion = true;
+
+        // Create a course with some availability data set.
+        $generator = $this->getDataGenerator();
+        $course = $generator->create_course(
+                array('format' => 'topics', 'numsections' => 3,
+                        'enablecompletion' => COMPLETION_ENABLED),
+                array('createsections' => true));
+        $forum = $generator->create_module('forum', array(
+                'course' => $course->id));
+        $forum2 = $generator->create_module('forum', array(
+                'course' => $course->id, 'completion' => COMPLETION_TRACKING_MANUAL));
+
+        // We need a grade, easiest is to add an assignment.
+        $assignrow = $generator->create_module('assign', array(
+                'course' => $course->id));
+        $assign = new assign(context_module::instance($assignrow->cmid), false, false);
+        $item = $assign->get_grade_item();
+
+        // Make a test grouping as well.
+        $grouping = $generator->create_grouping(array('courseid' => $course->id,
+                'name' => 'Grouping!'));
+
+        $availability = '{"op":"|","show":false,"c":[' .
+                '{"type":"completion","cm":' . $forum2->cmid .',"e":1},' .
+                '{"type":"grade","id":' . $item->id . ',"min":4,"max":94},' .
+                '{"type":"grouping","id":' . $grouping->id . '}' .
+                ']}';
+        $DB->set_field('course_modules', 'availability', $availability, array(
+                'id' => $forum->cmid));
+        $DB->set_field('course_sections', 'availability', $availability, array(
+                'course' => $course->id, 'section' => 1));
+
+        // Backup the course.
+        $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE,
+                backup::INTERACTIVE_YES, backup::MODE_GENERAL, $USER->id);
+        $bc->finish_ui();
+        $backupid = $bc->get_backupid();
+        $bc->execute_plan();
+        $bc->destroy();
+
+        // Get the backup file.
+        $coursecontext = context_course::instance($course->id);
+        $fs = get_file_storage();
+        $files = $fs->get_area_files($coursecontext->id, 'backup', 'course', false, 'id ASC');
+        $backupfile = reset($files);
+
+        // Extract backup file.
+        $backupdir = "restore_" . uniqid();
+        $path = $CFG->tempdir . DIRECTORY_SEPARATOR . "backup" . DIRECTORY_SEPARATOR . $backupdir;
+
+        $fp = get_file_packer('application/vnd.moodle.backup');
+        $fp->extract_to_pathname($backupfile, $path);
+
+        // Create restore controller.
+        $newcourseid = restore_dbops::create_new_course(
+                $course->fullname, $course->shortname . '_2', $course->category);
+        $rc = new restore_controller($backupdir, $newcourseid,
+                backup::INTERACTIVE_NO, backup::MODE_ASYNC, $USER->id,
+                backup::TARGET_NEW_COURSE);
+
+        $this->assertTrue($rc->execute_precheck());
+        $restoreid = $rc->get_restoreid();
+
+        $prerestorerec = $DB->get_record('backup_controllers', array('backupid' => $restoreid));
+        $prerestorerec->controller = '';
+
+        $rc->destroy();
+
+        // Create the adhoc task.
+        $asynctask = new \core\task\asynchronous_restore_task();
+        $asynctask->set_blocking(false);
+        $asynctask->set_custom_data(array('backupid' => $restoreid));
+        \core\task\manager::queue_adhoc_task($asynctask);
+
+        // We are expecting trace output during this test.
+        $this->expectOutputRegex("/$restoreid/");
+
+        // Execute adhoc task.
+        $now = time();
+        $task = \core\task\manager::get_next_adhoc_task($now);
+        $this->assertInstanceOf('\\core\\task\\asynchronous_restore_task', $task);
+        $task->execute();
+        \core\task\manager::adhoc_task_complete($task);
+
+        $postrestorerec = $DB->get_record('backup_controllers', array('backupid' => $restoreid));
+
+        // Check backup was created successfully.
+        $this->assertEquals(backup::STATUS_FINISHED_OK, $postrestorerec->status);
+        $this->assertEquals(1.0, $postrestorerec->progress);
+    }
+}

backup/util/dbops/backup_controller_dbops.class.php

@@ -641,4 +641,31 @@ abstract class backup_controller_dbops extends backup_dbops {
             }
         }
     }
+
+    /**
+     * Get the progress details of a backup operation.
+     * Get backup records directly from database, if the backup has successfully completed
+     * there will be no controller object to load.
+     *
+     * @param string $backupid The backup id to query.
+     * @return array $progress The backup progress details.
+     */
+    public static function get_progress($backupid) {
+        global $DB;
+
+        $progress = array();
+        $backuprecord = $DB->get_record(
+            'backup_controllers',
+            array('backupid' => $backupid),
+            'status, progress, operation',
+            MUST_EXIST);
+
+        $status = $backuprecord->status;
+        $progress = $backuprecord->progress;
+        $operation = $backuprecord->operation;
+
+        $progress = array('status' => $status, 'progress' => $progress, 'backupid' => $backupid, 'operation' => $operation);
+
+        return $progress;
+    }
 }

backup/util/helper/async_helper.class.php

@@ -0,0 +1,367 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Helper functions for asynchronous backups and restores.
+ *
+ * @package    core
+ * @copyright  2019 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once($CFG->dirroot . '/user/lib.php');
+
+/**
+ * Helper functions for asynchronous backups and restores.
+ *
+ * @package     core
+ * @copyright   2019 Matt Porritt <[email protected]>
+ * @license     http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class async_helper  {
+
+    /**
+     * @var string $type The type of async operation.
+     */
+    protected $type = 'backup';
+
+    /**
+     * @var string $backupid The id of the backup or restore.
+     */
+    protected $backupid;
+
+    /**
+     * @var object $user The user who created the backup record.
+     */
+    protected $user;
+
+    /**
+     * @var object $backuprec The backup controller record from the database.
+     */
+    protected $backuprec;
+
+    /**
+     * Class constructor.
+     *
+     * @param string $type The type of async operation.
+     * @param string $id The id of the backup or restore.
+     */
+    public function __construct($type, $id) {
+        $this->type = $type;
+        $this->backupid = $id;
+        $this->backuprec = $this->get_backup_record($id);
+        $this->user = $this->get_user();
+    }
+
+    /**
+     * Given a backup id return a the record from the database.
+     * We use this method rather than 'load_controller' as the controller may
+     * not exist if this backup/restore has completed.
+     *
+     * @param int $id The backup id to get.
+     * @return object $backuprec The backup controller record.
+     */
+    private function get_backup_record($id) {
+        global $DB;
+
+        $backuprec = $DB->get_record('backup_controllers', array('backupid' => $id), '*', MUST_EXIST);
+
+        return $backuprec;
+    }
+
+    /**
+     * Given a user id return a user object.
+     *
+     * @return object $user The limited user record.
+     */
+    private function get_user() {
+        $userid = $this->backuprec->userid;
+        $user = core_user::get_user($userid, '*', MUST_EXIST);
+
+        return $user;
+    }
+
+    /**
+     * Callback for preg_replace_callback.
+     * Replaces message placeholders with real values.
+     *
+     * @param array $matches The match array from from preg_replace_callback.
+     * @return string $match The replaced string.
+     */
+    private function lookup_message_variables($matches) {
+        $options = array(
+                'operation' => $this->type,
+                'backupid' => $this->backupid,
+                'user_username' => $this->user->username,
+                'user_email' => $this->user->email,
+                'user_firstname' => $this->user->firstname,
+                'user_lastname' => $this->user->lastname,
+                'link' => $this->get_resource_link(),
+        );
+
+        $match = $options[$matches[1]] ?? $matches[1];
+
+        return $match;
+    }
+
+    /**
+     * Get the link to the resource that is being backuped or restored.
+     *
+     * @return moodle_url $url The link to the resource.
+     */
+    private function get_resource_link() {
+        // Get activity context only for backups.
+        if ($this->backuprec->type == 'activity' && $this->type == 'backup') {
+            $context = context_module::instance($this->backuprec->itemid);
+        } else { // Course or Section which have the same context getter.
+            $context = context_course::instance($this->backuprec->itemid);
+        }
+
+        // Generate link based on operation type.
+        if ($this->type == 'backup') {
+            // For backups simply generate link to restore file area UI.
+            $url = new moodle_url('/backup/restorefile.php', array('contextid' => $context->id));
+        } else {
+            // For restore generate link to the item itself.
+            $url = $context->get_url();
+        }
+
+        return $url;
+    }
+
+    /**
+     * Sends a confirmation message for an aynchronous process.
+     *
+     * @return int $messageid The id of the sent message.
+     */
+    public function send_message() {
+        global $USER;
+
+        $subjectraw = get_config('backup', 'backup_async_message_subject');
+        $subjecttext = preg_replace_callback(
+                '/\{([-_A-Za-z0-9]+)\}/u',
+                array('async_helper', 'lookup_message_variables'),
+                $subjectraw);
+
+        $messageraw = get_config('backup', 'backup_async_message');
+        $messagehtml = preg_replace_callback(
+                '/\{([-_A-Za-z0-9]+)\}/u',
+                array('async_helper', 'lookup_message_variables'),
+                $messageraw);
+        $messagetext = html_to_text($messagehtml);
+
+        $message = new \core\message\message();
+        $message->component = 'moodle';
+        $message->name = 'asyncbackupnotification';
+        $message->userfrom          = $USER;
+        $message->userto            = $this->user;
+        $message->subject           = $subjecttext;
+        $message->fullmessage       = $messagetext;
+        $message->fullmessageformat = FORMAT_HTML;
+        $message->fullmessagehtml   = $messagehtml;
+        $message->smallmessage      = '';
+        $message->notification      = '1';
+
+        $messageid = message_send($message);
+
+        return $messageid;
+    }
+
+    /**
+     * Check if asynchronous backup and restore mode is
+     * enabled at system level.
+     *
+     * @return bool $async True if async mode enabled false otherwise.
+     */
+    static public function is_async_enabled() {
+        global $CFG;
+
+        $async = false;
+        if (!empty($CFG->enableasyncbackup)) {
+            $async = true;
+        }
+
+        return $async;
+    }
+
+    /**
+     * Check if there is a pending async operation for given details.
+     *
+     * @param int $id The item id to check in the backup record.
+     * @param string $type The type of operation: course, activity or section.
+     * @param string $operation Operation backup or restore.
+     * @return boolean $asyncpedning Is there a pending async operation.
+     */
+    public static function is_async_pending($id, $type, $operation) {
+        global $DB, $USER;
+        $asyncpending = false;
+
+        // Only check for pending async operations if async mode is enabled.
+        if (self::is_async_enabled()) {
+            $select = 'userid = ? AND itemid = ? AND type = ? AND operation = ? AND execution = ? AND status < ? AND status > ?';
+            $params = array(
+                $USER->id,
+                $id,
+                $type,
+                $operation,
+                backup::EXECUTION_DELAYED,
+                backup::STATUS_FINISHED_ERR,
+                backup::STATUS_NEED_PRECHECK
+            );
+            $asyncpending = $DB->record_exists_select('backup_controllers', $select, $params);
+        }
+        return $asyncpending;
+    }
+
+    /**
+     * Get the size, url and restore url for a backup file.
+     *
+     * @param string $filename The name of the file to get info for.
+     * @param string $filearea The file area for the file.
+     * @param int $contextid The context ID of the file.
+     * @return array $results The result array containing the size, url and restore url of the file.
+     */
+    public static function get_backup_file_info($filename, $filearea, $contextid) {
+        $fs = get_file_storage();
+        $file = $fs->get_file($contextid, 'backup', $filearea, 0, '/', $filename);
+        $filesize = display_size ($file->get_filesize());
+        $fileurl = moodle_url::make_pluginfile_url(
+            $file->get_contextid(),
+            $file->get_component(),
+            $file->get_filearea(),
+            null,
+            $file->get_filepath(),
+            $file->get_filename(),
+            true
+            );
+
+        $params = array();
+        $params['action'] = 'choosebackupfile';
+        $params['filename'] = $file->get_filename();
+        $params['filepath'] = $file->get_filepath();
+        $params['component'] = $file->get_component();
+        $params['filearea'] = $file->get_filearea();
+        $params['filecontextid'] = $file->get_contextid();
+        $params['contextid'] = $contextid;
+        $params['itemid'] = $file->get_itemid();
+        $restoreurl = new moodle_url('/backup/restorefile.php', $params);
+        $filesize = display_size ($file->get_filesize());
+
+        $results = array(
+            'filesize' => $filesize,
+            'fileurl' => $fileurl->out(false),
+            'restoreurl' => $restoreurl->out(false));
+
+        return $results;
+    }
+
+    /**
+     * Get the url of a restored backup item based on the backup ID.
+     *
+     * @param string $backupid The backup ID to get the restore location url.
+     * @return array $urlarray The restored item URL as an array.
+     */
+    public static function get_restore_url($backupid) {
+        global $DB;
+
+        $backupitemid = $DB->get_field('backup_controllers', 'itemid', array('backupid' => $backupid), MUST_EXIST);
+        $newcontext = context_course::instance($backupitemid);
+
+        $restoreurl = $newcontext->get_url()->out();
+        $urlarray = array('restoreurl' => $restoreurl);
+
+        return $urlarray;
+    }
+
+    /**
+     * Get markup for in progress async backups,
+     * to use in backup table UI.
+     *
+     * @param \core_backup_renderer $renderer The backup renderer object.
+     * @param integer $instanceid The context id to get backup data for.
+     * @return array $tabledata the rows of table data.
+     */
+    public static function get_async_backups($renderer, $instanceid) {
+        global $DB;
+
+        $tabledata = array();
+
+        // Get relevant backup ids based on context instance id.
+        $select = 'itemid = ? AND execution = ? AND status < ? AND status > ?';
+        $params = array($instanceid, backup::EXECUTION_DELAYED, backup::STATUS_FINISHED_ERR, backup::STATUS_NEED_PRECHECK);
+        $backups = $DB->get_records_select('backup_controllers', $select, $params, 'timecreated DESC', 'id, backupid, timecreated');
+
+        foreach ($backups as $backup) {
+            $bc = \backup_controller::load_controller($backup->backupid);  // Get the backup controller.
+            $filename = $bc->get_plan()->get_setting('filename')->get_value();
+            $timecreated = $backup->timecreated;
+            $status = $renderer->get_status_display($bc->get_status(), $bc->get_backupid());
+
+            $tablerow = array($filename, userdate($timecreated), '-', '-', '-', $status);
+            $tabledata[] = $tablerow;
+        }
+
+        return $tabledata;
+    }
+
+    /**
+     * Get the course name of the resource being restored.
+     *
+     * @param \context $context The Moodle context for the restores.
+     * @return string $coursename The full name of the course.
+     */
+    public static function get_restore_name(\context $context) {
+        global $DB;
+        $instanceid = $context->instanceid;
+
+        if ($context->contextlevel == CONTEXT_MODULE) {
+            // For modules get the course name and module name.
+            $cm = get_coursemodule_from_id('', $context->instanceid, 0, false, MUST_EXIST);
+            $coursename = $DB->get_field('course', 'fullname', array('id' => $cm->course));
+            $itemname = $coursename . ' - ' . $cm->name;
+        } else {
+            $itemname = $DB->get_field('course', 'fullname', array('id' => $context->instanceid));
+
+        }
+
+        return $itemname;
+    }
+
+    /**
+     * Get all the current in progress async restores for a user.
+     *
+     * @param int $userid Moodle user id.
+     * @return array $restores List of current restores in progress.
+     */
+    public static function get_async_restores($userid) {
+        global $DB;
+
+        $select = 'userid = ? AND execution = ? AND status < ? AND status > ? AND operation = ?';
+        $params = array($userid, backup::EXECUTION_DELAYED, backup::STATUS_FINISHED_ERR, backup::STATUS_NEED_PRECHECK, 'restore');
+        $restores = $DB->get_records_select(
+            'backup_controllers',
+            $select,
+            $params,
+            'timecreated DESC',
+            'id, backupid, status, itemid, timecreated');
+
+            return $restores;
+    }
+
+}
+

backup/util/helper/tests/async_helper_test.php

@@ -0,0 +1,148 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Asyncronhous helper tests.
+ *
+ * @package    core_backup
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+global $CFG;
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php');
+
+/**
+ * Asyncronhous helper tests.
+ *
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class core_backup_async_helper_testcase extends \core_privacy\tests\provider_testcase {
+
+    /**
+     * Tests sending message for asynchronous backup.
+     */
+    public function test_send_message() {
+        global $DB, $USER;
+        $this->preventResetByRollback();
+        $this->resetAfterTest(true);
+        $this->setAdminUser();
+
+        set_config('backup_async_message_users', '1', 'backup');
+        set_config('backup_async_message_subject', 'Moodle {operation} completed sucessfully', 'backup');
+        set_config('backup_async_message',
+                'Dear {user_firstname} {user_lastname}, <br/> Your {operation} (ID: {backupid}) has completed successfully!',
+                'backup');
+        set_config('allowedemaildomains', 'example.com');
+
+        $generator = $this->getDataGenerator();
+        $course = $generator->create_course();  // Create a course with some availability data set.
+        $user2 = $generator->create_user(array('firstname' => 'test', 'lastname' => 'human', 'maildisplay' => 1));
+        $generator->enrol_user($user2->id, $course->id, 'editingteacher');
+
+        $DB->set_field_select('message_processors', 'enabled', 0, "name <> 'email'");
+        set_user_preference('message_provider_moodle_asyncbackupnotification', 'email', $user2);
+
+        // Make the backup controller for an async backup.
+        $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE,
+                backup::INTERACTIVE_YES, backup::MODE_ASYNC, $user2->id);
+        $bc->finish_ui();
+        $backupid = $bc->get_backupid();
+        $bc->destroy();
+
+        $sink = $this->redirectEmails();
+
+        // Send message.
+        $asynchelper = new async_helper('backup', $backupid);
+        $messageid = $asynchelper->send_message();
+
+        $emails = $sink->get_messages();
+        $this->assertCount(1, $emails);
+        $email = reset($emails);
+
+        $this->assertSame($USER->email, $email->from);
+        $this->assertSame($user2->email, $email->to);
+        $this->assertSame('Moodle backup completed sucessfully', $email->subject);
+        $this->assertNotEmpty($email->header);
+        $this->assertNotEmpty($email->body);
+        $this->assertRegExp("/$backupid/", $email->body);
+        $this->assertThat($email->body, $this->logicalNot($this->stringContains('{')));
+        $this->assertGreaterThan(0, $messageid);
+        $sink->clear();
+    }
+
+    /**
+     * Tests getting the asynchronous backup table items.
+     */
+    public function test_get_async_backups() {
+        global $DB, $CFG, $USER, $PAGE;
+
+        $this->resetAfterTest(true);
+        $this->setAdminUser();
+        $CFG->enableavailability = true;
+        $CFG->enablecompletion = true;
+
+        // Create a course with some availability data set.
+        $generator = $this->getDataGenerator();
+        $course = $generator->create_course(
+            array('format' => 'topics', 'numsections' => 3,
+                'enablecompletion' => COMPLETION_ENABLED),
+            array('createsections' => true));
+        $forum = $generator->create_module('forum', array(
+            'course' => $course->id));
+        $forum2 = $generator->create_module('forum', array(
+            'course' => $course->id, 'completion' => COMPLETION_TRACKING_MANUAL));
+
+        // We need a grade, easiest is to add an assignment.
+        $assignrow = $generator->create_module('assign', array(
+            'course' => $course->id));
+        $assign = new assign(context_module::instance($assignrow->cmid), false, false);
+        $item = $assign->get_grade_item();
+
+        // Make a test grouping as well.
+        $grouping = $generator->create_grouping(array('courseid' => $course->id,
+            'name' => 'Grouping!'));
+
+        $availability = '{"op":"|","show":false,"c":[' .
+            '{"type":"completion","cm":' . $forum2->cmid .',"e":1},' .
+            '{"type":"grade","id":' . $item->id . ',"min":4,"max":94},' .
+            '{"type":"grouping","id":' . $grouping->id . '}' .
+            ']}';
+        $DB->set_field('course_modules', 'availability', $availability, array(
+            'id' => $forum->cmid));
+        $DB->set_field('course_sections', 'availability', $availability, array(
+            'course' => $course->id, 'section' => 1));
+
+        // Make the backup controller for an async backup.
+        $bc = new backup_controller(backup::TYPE_1COURSE, $course->id, backup::FORMAT_MOODLE,
+            backup::INTERACTIVE_YES, backup::MODE_ASYNC, $USER->id);
+        $bc->finish_ui();
+        $bc->destroy();
+        unset($bc);
+
+        $coursecontext = context_course::instance($course->id);
+        $renderer = $PAGE->get_renderer('core', 'backup');
+
+        $result = \async_helper::get_async_backups($renderer, $coursecontext->instanceid);
+
+        $this->assertEquals(1, count($result));
+        $this->assertEquals('backup.mbz', $result[0][0]);
+    }
+}

backup/util/includes/backup_includes.php

@@ -54,6 +54,7 @@ require_once($CFG->dirroot . '/backup/util/structure/backup_nested_element.class
 require_once($CFG->dirroot . '/backup/util/structure/backup_optigroup.class.php');
 require_once($CFG->dirroot . '/backup/util/structure/backup_optigroup_element.class.php');
 require_once($CFG->dirroot . '/backup/util/structure/backup_structure_processor.class.php');
+require_once($CFG->dirroot . '/backup/util/helper/async_helper.class.php');
 require_once($CFG->dirroot . '/backup/util/helper/backup_helper.class.php');
 require_once($CFG->dirroot . '/backup/util/helper/backup_general_helper.class.php');
 require_once($CFG->dirroot . '/backup/util/helper/backup_null_iterator.class.php');

backup/util/includes/restore_includes.php

@@ -34,6 +34,7 @@ require_once($CFG->dirroot . '/backup/util/interfaces/executable.class.php');
 require_once($CFG->dirroot . '/backup/util/interfaces/processable.class.php');
 require_once($CFG->dirroot . '/backup/backup.class.php');
 require_once($CFG->dirroot . '/backup/util/structure/restore_path_element.class.php');
+require_once($CFG->dirroot . '/backup/util/helper/async_helper.class.php');
 require_once($CFG->dirroot . '/backup/util/helper/backup_anonymizer_helper.class.php');
 require_once($CFG->dirroot . '/backup/util/helper/backup_file_manager.class.php');
 require_once($CFG->dirroot . '/backup/util/helper/restore_prechecks_helper.class.php');

backup/util/ui/amd/build/async_backup.min.js

@@ -0,0 +1 @@
+define(["jquery","core/ajax","core/str","core/notification","core/templates"],function(a,b,c,d,e){function f(b,c){var d=Math.round(c)+"%",e=a("#"+b+"_bar"),f=c.toFixed(2)+"%";e.attr("aria-valuenow",d),e.css("width",d),e.text(f)}function g(c){var f=a("#"+c+"_bar").parent().parent(),g=f.parent(),h=f.siblings(),i=h[1],j=a(i).text(),k=h[0],l=a(k).text();b.call([{methodname:"core_backup_get_async_backup_links_backup",args:{filename:l,contextid:n}}])[0].done(function(a){var b={filename:l,time:j,size:a.filesize,fileurl:a.fileurl,restoreurl:a.restoreurl};e.render("core/async_backup_progress_row",b).then(function(a,b){e.replaceNodeContents(g,a,b)}).fail(function(){d.exception(new Error("Failed to load table row"))})})}function h(c){var f=a("#"+c+"_bar").parent().parent(),g=f.parent(),h=f.siblings(),i=h[0],j=h[1],k=a(j).text();b.call([{methodname:"core_backup_get_async_backup_links_restore",args:{backupid:c,contextid:n}}])[0].done(function(b){var c=a(i).text(),f={resourcename:c,restoreurl:b.restoreurl,time:k};e.render("core/async_restore_progress_row",f).then(function(a,b){e.replaceNodeContents(g,a,b)}).fail(function(){d.exception(new Error("Failed to load table row"))})})}function i(e){var g,h=100*e.progress,i=a("#"+m+"_bar"),j=a("#"+m+"_status"),k=a("#"+m+"_detail"),l=a("#"+m+"_button");if(e.status==s){i.addClass("bg-success"),f(m,h);var r="async"+p+"processing";c.get_string(r,"backup").then(function(a){return j.text(a),a})["catch"](function(){d.exception(new Error("Failed to load string: backup "+r))})}else if(e.status==t){i.addClass("bg-danger"),i.removeClass("bg-success"),f(m,100);var v="async"+p+"error",w="async"+p+"errordetail";g=[{key:v,component:"backup"},{key:w,component:"backup"}],c.get_strings(g).then(function(a){return j.text(a[0]),k.text(a[1]),a})["catch"](function(){d.exception(new Error("Failed to load string"))}),a(".backup_progress").children("span").removeClass("backup_stage_current"),a(".backup_progress").children("span").last().addClass("backup_stage_current"),clearInterval(q)}else if(e.status==u){i.addClass("bg-success"),f(m,100);var x="async"+p+"complete";if(c.get_string(x,"backup").then(function(a){return j.text(a),a})["catch"](function(){d.exception(new Error("Failed to load string: backup "+x))}),"restore"==p)b.call([{methodname:"core_backup_get_async_backup_links_restore",args:{backupid:m,contextid:n}}])[0].done(function(a){var b="async"+p+"completedetail",e="async"+p+"completebutton",f=[{key:b,component:"backup",param:a.restoreurl},{key:e,component:"backup"}];c.get_strings(f).then(function(b){return k.html(b[0]),l.text(b[1]),l.attr("href",a.restoreurl),b})["catch"](function(){d.exception(new Error("Failed to load string"))})});else{var y="async"+p+"completedetail",z="async"+p+"completebutton";g=[{key:y,component:"backup",param:o},{key:z,component:"backup"}],c.get_strings(g).then(function(a){return k.html(a[0]),l.text(a[1]),l.attr("href",o),a})["catch"](function(){d.exception(new Error("Failed to load string"))})}a(".backup_progress").children("span").removeClass("backup_stage_current"),a(".backup_progress").children("span").last().addClass("backup_stage_current"),clearInterval(q)}}function j(b){b.forEach(function(b){var c=100*b.progress,d=b.backupid,e=a("#"+d+"_bar"),i=b.operation;b.status==s?(e.addClass("bg-success"),f(d,c)):b.status==t?(e.addClass("bg-danger"),e.addClass("complete"),a("#"+d+"_bar").removeClass("bg-success"),f(d,100)):b.status==u&&(e.addClass("bg-success"),e.addClass("complete"),f(d,100),"backup"==i?g(d):h(d))})}function k(){b.call([{methodname:"core_backup_get_async_backup_progress",args:{backupids:[m],contextid:n}}])[0].done(function(a){i(a[0])})}function l(){var c=[],d=a(".progress").find(".progress-bar").not(".complete");d.each(function(){c.push(this.id.substring(0,32))}),c.length>0?b.call([{methodname:"core_backup_get_async_backup_progress",args:{backupids:c,contextid:n}}])[0].done(function(a){j(a)}):clearInterval(r)}var m,n,o,p,q,r,s=800,t=900,u=1e3,v={},w=5e3;return v.asyncBackupAllStatus=function(a){n=a,r=setInterval(l,w)},v.asyncBackupStatus=function(b,c,d,e){m=b,n=c,o=d,p="backup"==e?"backup":"restore",a(".backup_progress").children("a").removeAttr("href"),q=setInterval(k,w)},v});

backup/util/ui/amd/src/async_backup.js

@@ -0,0 +1,424 @@
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * This module updates the UI during an asynchronous
+ * backup or restore process.
+ *
+ * @module     backup/util/async_backup
+ * @package    core
+ * @copyright  2018 Matt Porritt <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ * @since      3.7
+ */
+define(['jquery', 'core/ajax', 'core/str', 'core/notification', 'core/templates'],
+        function($, ajax, Str, notification, Templates) {
+
+    /**
+     * Module level constants.
+     *
+     * Using var instead of const as ES6 isn't fully supported yet.
+     */
+    var STATUS_EXECUTING = 800;
+    var STATUS_FINISHED_ERR = 900;
+    var STATUS_FINISHED_OK = 1000;
+
+    /**
+     * Module level variables.
+     */
+    var Asyncbackup = {};
+    var checkdelay = 5000; //  How often we check for progress updates.
+    var backupid; //  The backup id to get the progress for.
+    var contextid; //  The course this backup progress is for.
+    var restoreurl; //  The URL to view course restores.
+    var typeid; //  The type of operation backup or restore.
+    var backupintervalid; //  The id of the setInterval function.
+    var allbackupintervalid; //  The id of the setInterval function.
+
+    /**
+     * Helper function to update UI components.
+     *
+     * @param {string} backupid The id to match elements on.
+     * @param {number} percentage The completion percentage to apply.
+     */
+    function updateElement(backupid, percentage) {
+        var percentagewidth = Math.round(percentage) + '%';
+        var elementbar = $('#' + backupid + '_bar');
+        var percentagetext = percentage.toFixed(2) + '%';
+
+        // Set progress bar percentage indicators
+        elementbar.attr('aria-valuenow', percentagewidth);
+        elementbar.css('width', percentagewidth);
+        elementbar.text(percentagetext);
+    }
+
+    /**
+     * Update backup table row when an async backup completes.
+     *
+     * @param {string} backupid The id to match elements on.
+     */
+    function updateBackupTableRow(backupid) {
+        var statuscell = $('#' + backupid + '_bar').parent().parent();
+        var tablerow = statuscell.parent();
+        var cellsiblings = statuscell.siblings();
+        var timecell = cellsiblings[1];
+        var timevalue = $(timecell).text();
+        var filenamecell = cellsiblings[0];
+        var filename = $(filenamecell).text();
+
+        ajax.call([{
+            // Get the table data via webservice.
+            methodname: 'core_backup_get_async_backup_links_backup',
+            args: {
+                'filename': filename,
+                'contextid': contextid
+            },
+        }])[0].done(function(response) {
+            // We have the data now update the UI.
+            var context = {
+                    filename: filename,
+                    time: timevalue,
+                    size: response.filesize,
+                    fileurl: response.fileurl,
+                    restoreurl: response.restoreurl
+                    };
+
+            Templates.render('core/async_backup_progress_row', context).then(function(html, js) {
+                Templates.replaceNodeContents(tablerow, html, js);
+                return;
+            }).fail(function() {
+                notification.exception(new Error('Failed to load table row'));
+                return;
+            });
+        });
+    }
+
+    /**
+     * Update restore table row when an async restore completes.
+     *
+     * @param {string} backupid The id to match elements on.
+     */
+    function updateRestoreTableRow(backupid) {
+        var statuscell = $('#' + backupid + '_bar').parent().parent();
+        var tablerow = statuscell.parent();
+        var cellsiblings = statuscell.siblings();
+        var coursecell = cellsiblings[0];
+        var timecell = cellsiblings[1];
+        var timevalue = $(timecell).text();
+
+        ajax.call([{
+            // Get the table data via webservice.
+            methodname: 'core_backup_get_async_backup_links_restore',
+            args: {
+                'backupid': backupid,
+                'contextid': contextid
+            },
+        }])[0].done(function(response) {
+         // We have the data now update the UI.
+            var resourcename = $(coursecell).text();
+            var context = {
+                    resourcename: resourcename,
+                    restoreurl: response.restoreurl,
+                    time: timevalue
+                    };
+
+            Templates.render('core/async_restore_progress_row', context).then(function(html, js) {
+                Templates.replaceNodeContents(tablerow, html, js);
+                return;
+            }).fail(function() {
+                notification.exception(new Error('Failed to load table row'));
+                return;
+            });
+        });
+    }
+
+    /**
+     * Update the Moodle user interface with the progress of
+     * the backup process.
+     *
+     * @param {object} progress The progress and status of the process.
+     */
+    function updateProgress(progress) {
+        var percentage = progress.progress * 100;
+        var elementbar = $('#' + backupid + '_bar');
+        var elementstatus = $('#' + backupid + '_status');
+        var elementdetail = $('#' + backupid + '_detail');
+        var elementbutton = $('#' + backupid + '_button');
+        var stringRequests;
+
+        if (progress.status == STATUS_EXECUTING) {
+            // Process is in progress.
+            // Add in progress class color to bar
+            elementbar.addClass('bg-success');
+
+            updateElement(backupid, percentage);
+
+            // Change heading
+            var strProcessing = 'async' + typeid + 'processing';
+            Str.get_string(strProcessing, 'backup').then(function(title) {
+                elementstatus.text(title);
+                return title;
+            }).catch(function() {
+                notification.exception(new Error('Failed to load string: backup ' + strProcessing));
+            });
+
+        } else if (progress.status == STATUS_FINISHED_ERR) {
+            // Process completed with error.
+
+            // Add in fail class color to bar
+            elementbar.addClass('bg-danger');
+
+            // Remove in progress class color to bar
+            elementbar.removeClass('bg-success');
+
+            updateElement(backupid, 100);
+
+            // Change heading and text
+            var strStatus = 'async' + typeid + 'error';
+            var strStatusDetail = 'async' + typeid + 'errordetail';
+            stringRequests = [
+                {key: strStatus, component: 'backup'},
+                {key: strStatusDetail, component: 'backup'}
+            ];
+            Str.get_strings(stringRequests).then(function(strings) {
+                elementstatus.text(strings[0]);
+                elementdetail.text(strings[1]);
+
+                return strings;
+            })
+            .catch(function() {
+                notification.exception(new Error('Failed to load string'));
+                return;
+            });
+
+            $('.backup_progress').children('span').removeClass('backup_stage_current');
+            $('.backup_progress').children('span').last().addClass('backup_stage_current');
+
+            // Stop checking when we either have an error or a completion.
+            clearInterval(backupintervalid);
+
+        } else if (progress.status == STATUS_FINISHED_OK) {
+            // Process completed successfully.
+
+            // Add in progress class color to bar
+            elementbar.addClass('bg-success');
+
+            updateElement(backupid, 100);
+
+            // Change heading and text
+            var strComplete = 'async' + typeid + 'complete';
+            Str.get_string(strComplete, 'backup').then(function(title) {
+                elementstatus.text(title);
+                return title;
+            }).catch(function() {
+                notification.exception(new Error('Failed to load string: backup ' + strComplete));
+            });
+
+            if (typeid == 'restore') {
+                ajax.call([{
+                    // Get the table data via webservice.
+                    methodname: 'core_backup_get_async_backup_links_restore',
+                    args: {
+                        'backupid': backupid,
+                        'contextid': contextid
+                    },
+                }])[0].done(function(response) {
+                    var strDetail = 'async' + typeid + 'completedetail';
+                    var strButton = 'async' + typeid + 'completebutton';
+                    var stringRequests = [
+                        {key: strDetail, component: 'backup', param: response.restoreurl},
+                        {key: strButton, component: 'backup'}
+                    ];
+                    Str.get_strings(stringRequests).then(function(strings) {
+                        elementdetail.html(strings[0]);
+                        elementbutton.text(strings[1]);
+                        elementbutton.attr('href', response.restoreurl);
+
+                        return strings;
+                    })
+                    .catch(function() {
+                        notification.exception(new Error('Failed to load string'));
+                        return;
+                    });
+
+                });
+            } else {
+                var strDetail = 'async' + typeid + 'completedetail';
+                var strButton = 'async' + typeid + 'completebutton';
+                stringRequests = [
+                    {key: strDetail, component: 'backup', param: restoreurl},
+                    {key: strButton, component: 'backup'}
+                ];
+                Str.get_strings(stringRequests).then(function(strings) {
+                    elementdetail.html(strings[0]);
+                    elementbutton.text(strings[1]);
+                    elementbutton.attr('href', restoreurl);
+
+                    return strings;
+                })
+                .catch(function() {
+                    notification.exception(new Error('Failed to load string'));
+                    return;
+                });
+
+            }
+
+            $('.backup_progress').children('span').removeClass('backup_stage_current');
+            $('.backup_progress').children('span').last().addClass('backup_stage_current');
+
+            // Stop checking when we either have an error or a completion.
+            clearInterval(backupintervalid);
+        }
+    }
+
+    /**
+     * Update the Moodle user interface with the progress of
+     * all the pending processes.
+     *
+     * @param {object} progress The progress and status of the process.
+     */
+    function updateProgressAll(progress) {
+        progress.forEach(function(element) {
+            var percentage = element.progress * 100;
+            var backupid = element.backupid;
+            var elementbar = $('#' + backupid + '_bar');
+            var type = element.operation;
+
+            if (element.status == STATUS_EXECUTING) {
+                // Process is in element.
+
+                // Add in element class color to bar
+                elementbar.addClass('bg-success');
+
+                updateElement(backupid, percentage);
+
+            } else if (element.status == STATUS_FINISHED_ERR) {
+                // Process completed with error.
+
+                // Add in fail class color to bar
+                elementbar.addClass('bg-danger');
+                elementbar.addClass('complete');
+
+                // Remove in element class color to bar
+                $('#' + backupid + '_bar').removeClass('bg-success');
+
+                updateElement(backupid, 100);
+
+            } else if (element.status == STATUS_FINISHED_OK) {
+                // Process completed successfully.
+
+                // Add in element class color to bar
+                elementbar.addClass('bg-success');
+                elementbar.addClass('complete');
+
+                updateElement(backupid, 100);
+
+                // We have a successful backup. Update the UI with download and file details.
+                if (type == 'backup') {
+                    updateBackupTableRow(backupid);
+                } else {
+                    updateRestoreTableRow(backupid);
+                }
+
+            }
+
+        });
+    }
+
+    /**
+     * Get the progress of the backup process via ajax.
+     */
+    function getBackupProgress() {
+        ajax.call([{
+            // Get the backup progress via webservice.
+            methodname: 'core_backup_get_async_backup_progress',
+            args: {
+                'backupids': [backupid],
+                'contextid': contextid
+            },
+        }])[0].done(function(response) {
+            // We have the progress now update the UI.
+            updateProgress(response[0]);
+        });
+    }
+
+    /**
+     * Get the progress of all backup processes via ajax.
+     */
+    function getAllBackupProgress() {
+        var backupids = [];
+        var progressbars = $('.progress').find('.progress-bar').not('.complete');
+
+        progressbars.each(function() {
+            backupids.push((this.id).substring(0, 32));
+        });
+
+        if (backupids.length > 0) {
+            ajax.call([{
+                // Get the backup progress via webservice.
+                methodname: 'core_backup_get_async_backup_progress',
+                args: {
+                    'backupids': backupids,
+                    'contextid': contextid
+                },
+            }])[0].done(function(response) {
+                updateProgressAll(response);
+            });
+        } else {
+            clearInterval(allbackupintervalid); // No more progress bars to update, stop checking.
+        }
+    }
+
+    /**
+     * Get status updates for all backups.
+     *
+     * @public
+     * @param {number} context The context id.
+     */
+    Asyncbackup.asyncBackupAllStatus = function(context) {
+        contextid = context;
+        allbackupintervalid = setInterval(getAllBackupProgress, checkdelay);
+    };
+
+    /**
+     * Get status updates for backup.
+     *
+     * @public
+     * @param {string} backup The backup record id.
+     * @param {number} context The context id.
+     * @param {string} restore The restore link.
+     * @param {string} type The operation type (backup or restore).
+     */
+    Asyncbackup.asyncBackupStatus = function(backup, context, restore, type) {
+        backupid = backup;
+        contextid = context;
+        restoreurl = restore;
+
+        if (type == 'backup') {
+            typeid = 'backup';
+        } else {
+            typeid = 'restore';
+        }
+
+        // Remove the links from the progress bar, no going back now.
+        $('.backup_progress').children('a').removeAttr('href');
+
+        //  Periodically check for progress updates and update the UI as required.
+        backupintervalid = setInterval(getBackupProgress, checkdelay);
+
+      };
+
+      return Asyncbackup;
+});

backup/util/ui/renderer.php

@@ -22,6 +22,13 @@
  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
+defined('MOODLE_INTERNAL') || die;
+
+global $CFG;
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
+require_once($CFG->dirroot . '/backup/util/includes/restore_includes.php');
+require_once($CFG->dirroot . '/backup/moodle2/backup_plan_builder.class.php');
+
 /**
  * The primary renderer for the backup.
  *
@@ -64,9 +71,8 @@ class core_backup_renderer extends plugin_renderer_base {
      * @return string HTML content that shows the log
      */
     public function log_display($loghtml) {
-        global $OUTPUT;
         $out = html_writer::start_div('backup_log');
-        $out .= $OUTPUT->heading(get_string('backuplog', 'backup'));
+        $out .= $this->output->heading(get_string('backuplog', 'backup'));
         $out .= html_writer::start_div('backup_log_contents');
         $out .= $loghtml;
         $out .= html_writer::end_div();
@@ -533,6 +539,32 @@ class core_backup_renderer extends plugin_renderer_base {
         return $this->render($files);
     }
 
+    /**
+     * Generate the status indicator markup for display in the
+     * backup restore file area UI.
+     *
+     * @param int $statuscode The status code of the backup.
+     * @param string $backupid The backup record id.
+     * @return string|boolean $status The status indicator for the operation.
+     */
+    public function get_status_display($statuscode, $backupid) {
+        if ($statuscode == backup::STATUS_AWAITING || $statuscode == backup::STATUS_EXECUTING) {  // Inprogress.
+            $progresssetup = array(
+                    'backupid' => $backupid,
+                    'width' => '100'
+            );
+            $status = $this->render_from_template('core/async_backup_progress', $progresssetup);
+        } else if ($statuscode == backup::STATUS_FINISHED_ERR) { // Error.
+            $icon = $this->output->render(new \pix_icon('i/delete', get_string('failed', 'backup')));
+            $status = \html_writer::span($icon, 'action-icon');
+        } else if ($statuscode == backup::STATUS_FINISHED_OK) { // Complete.
+            $icon = $this->output->render(new \pix_icon('i/checked', get_string('successful', 'backup')));
+            $status = \html_writer::span($icon, 'action-icon');
+        }
+
+        return $status;
+    }
+
     /**
      * Displays a backup files viewer
      *
@@ -544,12 +576,35 @@ class core_backup_renderer extends plugin_renderer_base {
         global $CFG;
         $files = $viewer->files;
 
+        $async = async_helper::is_async_enabled();
+
+        $tablehead = array(
+                get_string('filename', 'backup'),
+                get_string('time'),
+                get_string('size'),
+                get_string('download'),
+                get_string('restore'));
+        if ($async) {
+            $tablehead[] = get_string('status', 'backup');
+        }
+
         $table = new html_table();
         $table->attributes['class'] = 'backup-files-table generaltable';
-        $table->head = array(get_string('filename', 'backup'), get_string('time'), get_string('size'), get_string('download'), get_string('restore'));
+        $table->head = $tablehead;
         $table->width = '100%';
         $table->data = array();
 
+        // First add in progress asynchronous backups.
+        // Only if asynchronous backups are enabled.
+        // Also only render async status in correct area. Courese OR activity (not both).
+        if ($async
+                && (($viewer->filearea == 'course' && $viewer->currentcontext->contextlevel == CONTEXT_COURSE)
+                || ($viewer->filearea == 'activity' && $viewer->currentcontext->contextlevel == CONTEXT_MODULE))
+                ) {
+                    $table->data = \async_helper::get_async_backups($this, $viewer->currentcontext->instanceid);
+        }
+
+        // Add completed backups.
         foreach ($files as $file) {
             if ($file->is_directory()) {
                 continue;
@@ -585,13 +640,18 @@ class core_backup_renderer extends plugin_renderer_base {
                     $downloadlink = '';
                 }
             }
-            $table->data[] = array(
+            $tabledata = array(
                 $file->get_filename(),
-                userdate($file->get_timemodified()),
-                display_size($file->get_filesize()),
+                userdate ($file->get_timemodified()),
+                display_size ($file->get_filesize()),
                 $downloadlink,
-                $restorelink,
-                );
+                $restorelink
+            );
+            if ($async) {
+                $tabledata[] = $this->get_status_display(backup::STATUS_FINISHED_OK, null);
+            }
+
+            $table->data[] = $tabledata;
         }
 
         $html = html_writer::table($table);
@@ -853,6 +913,42 @@ class core_backup_renderer extends plugin_renderer_base {
         $output .= html_writer::end_tag('div');
         return $output;
     }
+
+    /**
+     * Get markup to render table for all of a users async
+     * in progress restores.
+     *
+     * @param int $userid The Moodle user id.
+     * @param \context $context The Moodle context for these restores.
+     * @return string $html The table HTML.
+     */
+    public function restore_progress_viewer ($userid, $context) {
+        $tablehead = array(get_string('course'), get_string('time'), get_string('status', 'backup'));
+
+        $table = new html_table();
+        $table->attributes['class'] = 'backup-files-table generaltable';
+        $table->head = $tablehead;
+        $tabledata = array();
+
+        // Get all in progress async restores for this user.
+        $restores = \async_helper::get_async_restores($userid);
+
+        // For each backup get, new item name, time restore created and progress.
+        foreach ($restores as $restore) {
+
+            $restorename = \async_helper::get_restore_name($context);
+            $timecreated = $restore->timecreated;
+            $status = $this->get_status_display($restore->status, $restore->backupid);
+
+            $tablerow = array($restorename, userdate($timecreated), $status);
+            $tabledata[] = $tablerow;
+        }
+
+        $table->data = $tabledata;
+        $html = html_writer::table($table);
+
+        return $html;
+    }
 }
 
 /**

badges/cron.php

@@ -1,158 +0,0 @@
-<?php
-// This file is part of Moodle - http://moodle.org/
-//
-// Moodle is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// Moodle is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
-
-/**
- * Cron job for reviewing and aggregating badge award criteria
- *
- * @package    core
- * @subpackage badges
- * @copyright  2012 onwards Totara Learning Solutions Ltd {@link http://www.totaralms.com/}
- * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
- * @author     Yuliya Bozhko <[email protected]>
- */
-
-defined('MOODLE_INTERNAL') || die();
-require_once($CFG->libdir . '/badgeslib.php');
-
-function badge_cron() {
-    global $CFG;
-
-    if (!empty($CFG->enablebadges)) {
-        badge_review_cron();
-        badge_message_cron();
-    }
-}
-
-/**
- * Reviews criteria and awards badges
- *
- * First find all badges that can be earned, then reviews each badge.
- * (Not sure how efficient this is timewise).
- */
-function badge_review_cron() {
-    global $DB, $CFG;
-    $total = 0;
-
-    $courseparams = array();
-    if (empty($CFG->badges_allowcoursebadges)) {
-        $coursesql = '';
-    } else {
-        $coursesql = ' OR EXISTS (SELECT id FROM {course} WHERE visible = :visible AND startdate < :current) ';
-        $courseparams = array('visible' => true, 'current' => time());
-    }
-
-    $sql = 'SELECT id
-                FROM {badge}
-                WHERE (status = :active OR status = :activelocked)
-                    AND (type = :site ' . $coursesql . ')';
-    $badgeparams = array(
-                    'active' => BADGE_STATUS_ACTIVE,
-                    'activelocked' => BADGE_STATUS_ACTIVE_LOCKED,
-                    'site' => BADGE_TYPE_SITE
-                    );
-    $params = array_merge($badgeparams, $courseparams);
-    $badges = $DB->get_fieldset_sql($sql, $params);
-
-    mtrace('Started reviewing available badges.');
-    foreach ($badges as $bid) {
-        $badge = new badge($bid);
-
-        if ($badge->has_criteria()) {
-            if (debugging()) {
-                mtrace('Processing badge "' . $badge->name . '"...');
-            }
-
-            $issued = $badge->review_all_criteria();
-
-            if (debugging()) {
-                mtrace('...badge was issued to ' . $issued . ' users.');
-            }
-            $total += $issued;
-        }
-    }
-
-    mtrace('Badges were issued ' . $total . ' time(s).');
-}
-
-/**
- * Sends out scheduled messages to badge creators
- *
- */
-function badge_message_cron() {
-    global $DB;
-
-    mtrace('Sending scheduled badge notifications.');
-
-    $scheduled = $DB->get_records_select('badge', 'notification > ? AND (status != ?) AND nextcron < ?',
-                            array(BADGE_MESSAGE_ALWAYS, BADGE_STATUS_ARCHIVED, time()),
-                            'notification ASC', 'id, name, notification, usercreated as creator, timecreated');
-
-    foreach ($scheduled as $sch) {
-        // Send messages.
-        badge_assemble_notification($sch);
-
-        // Update next cron value.
-        $nextcron = badges_calculate_message_schedule($sch->notification);
-        $DB->set_field('badge', 'nextcron', $nextcron, array('id' => $sch->id));
-    }
-}
-
-/**
- * Creates single message for all notification and sends it out
- *
- * @param object $badge A badge which is notified about.
- */
-function badge_assemble_notification(stdClass $badge) {
-    global $DB;
-
-    $userfrom = core_user::get_noreply_user();
-    $userfrom->maildisplay = true;
-
-    if ($msgs = $DB->get_records_select('badge_issued', 'issuernotified IS NULL AND badgeid = ?', array($badge->id))) {
-        // Get badge creator.
-        $creator = $DB->get_record('user', array('id' => $badge->creator), '*', MUST_EXIST);
-        $creatorsubject = get_string('creatorsubject', 'badges', $badge->name);
-        $creatormessage = '';
-
-        // Put all messages in one digest.
-        foreach ($msgs as $msg) {
-            $issuedlink = html_writer::link(new moodle_url('/badges/badge.php', array('hash' => $msg->uniquehash)), $badge->name);
-            $recipient = $DB->get_record('user', array('id' => $msg->userid), '*', MUST_EXIST);
-
-            $a = new stdClass();
-            $a->user = fullname($recipient);
-            $a->link = $issuedlink;
-            $creatormessage .= get_string('creatorbody', 'badges', $a);
-            $DB->set_field('badge_issued', 'issuernotified', time(), array('badgeid' => $msg->badgeid, 'userid' => $msg->userid));
-        }
-
-        // Create a message object.
-        $eventdata = new \core\message\message();
-        $eventdata->courseid          = SITEID;
-        $eventdata->component         = 'moodle';
-        $eventdata->name              = 'badgecreatornotice';
-        $eventdata->userfrom          = $userfrom;
-        $eventdata->userto            = $creator;
-        $eventdata->notification      = 1;
-        $eventdata->subject           = $creatorsubject;
-        $eventdata->fullmessage       = format_text_email($creatormessage, FORMAT_HTML);
-        $eventdata->fullmessageformat = FORMAT_PLAIN;
-        $eventdata->fullmessagehtml   = $creatormessage;
-        $eventdata->smallmessage      = $creatorsubject;
-
-        message_send($eventdata);
-    }
-}

calendar/lib.php

@@ -3302,9 +3302,11 @@ function calendar_get_legacy_events($tstart, $tend, $users, $groups, $courses,
  * @param   string  $view The type of calendar to have displayed
  * @param   bool    $includenavigation Whether to include navigation
  * @param   bool    $skipevents Whether to load the events or not
+ * @param   int     $lookahead Overwrites site and users's lookahead setting.
  * @return  array[array, string]
  */
-function calendar_get_view(\calendar_information $calendar, $view, $includenavigation = true, bool $skipevents = false) {
+function calendar_get_view(\calendar_information $calendar, $view, $includenavigation = true, bool $skipevents = false,
+        ?int $lookahead = null) {
     global $PAGE, $CFG;
 
     $renderer = $PAGE->get_renderer('core_calendar');
@@ -3322,12 +3324,14 @@ function calendar_get_view(\calendar_information $calendar, $view, $includenavig
         $date->modify('+1 day');
     } else if ($view === 'upcoming' || $view === 'upcoming_mini') {
         // Number of days in the future that will be used to fetch events.
-        if (isset($CFG->calendar_lookahead)) {
-            $defaultlookahead = intval($CFG->calendar_lookahead);
-        } else {
-            $defaultlookahead = CALENDAR_DEFAULT_UPCOMING_LOOKAHEAD;
+        if (!$lookahead) {
+            if (isset($CFG->calendar_lookahead)) {
+                $defaultlookahead = intval($CFG->calendar_lookahead);
+            } else {
+                $defaultlookahead = CALENDAR_DEFAULT_UPCOMING_LOOKAHEAD;
+            }
+            $lookahead = get_user_preferences('calendar_lookahead', $defaultlookahead);
         }
-        $lookahead = get_user_preferences('calendar_lookahead', $defaultlookahead);
 
         // Maximum number of events to be displayed on upcoming view.
         $defaultmaxevents = CALENDAR_DEFAULT_UPCOMING_MAXEVENTS;

calendar/view.php

@@ -53,6 +53,7 @@ $categoryid = optional_param('category', null, PARAM_INT);
 $courseid = optional_param('course', SITEID, PARAM_INT);
 $view = optional_param('view', 'upcoming', PARAM_ALPHA);
 $time = optional_param('time', 0, PARAM_INT);
+$lookahead = optional_param('lookahead', null, PARAM_INT);
 
 $url = new moodle_url('/calendar/view.php');
 
@@ -124,7 +125,7 @@ echo html_writer::start_tag('div', array('class'=>'heightcontainer'));
 echo $OUTPUT->heading(get_string('calendar', 'calendar'));
 
 
-list($data, $template) = calendar_get_view($calendar, $view);
+list($data, $template) = calendar_get_view($calendar, $view, true, false, $lookahead);
 echo $renderer->render_from_template($template, $data);
 
 echo html_writer::end_tag('div');

course/amd/src/actions.js

@@ -341,30 +341,21 @@ define(['jquery', 'core/ajax', 'core/templates', 'core/notification', 'core/str'
          * @param {String} image new image name ("i/show", "i/hide", etc.)
          * @param {String} stringname new string for the action menu item
          * @param {String} stringcomponent
-         * @param {String} titlestr string for "title" attribute (if different from stringname)
-         * @param {String} titlecomponent
+         * @param {String} titlestr not used
+         * @param {String} titlecomponent not used
          * @param {String} newaction new value for data-action attribute of the link
          * @return {Promise} promise which is resolved when the replacement has completed
          */
         var replaceActionItem = function(actionitem, image, stringname,
                                            stringcomponent, titlestr, titlecomponent, newaction) {
 
-
             var stringRequests = [{key: stringname, component: stringcomponent}];
-            if (titlestr) {
-                stringRequests.push({key: titlestr, component: titlecomponent});
-            }
+            // Do not provide an icon with duplicate, different text to the menu item.
 
             return str.get_strings(stringRequests).then(function(strings) {
                 actionitem.find('span.menu-action-text').html(strings[0]);
-                actionitem.attr('title', strings[0]);
 
-                var title = '';
-                if (titlestr) {
-                    title = strings[1];
-                    actionitem.attr('title', title);
-                }
-                return templates.renderPix(image, 'core', title);
+                return templates.renderPix(image, 'core');
             }).then(function(pixhtml) {
                 actionitem.find('.icon').replaceWith(pixhtml);
                 actionitem.attr('data-action', newaction);

course/classes/analytics/indicator/activities_due.php

@@ -0,0 +1,100 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Activities due indicator.
+ *
+ * @package   core
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core_course\analytics\indicator;
+
+defined('MOODLE_INTERNAL') || die();
+
+require_once($CFG->dirroot . '/calendar/externallib.php');
+
+/**
+ * Activities due indicator.
+ *
+ * @package   core
+ * @copyright 2019 David Monllao {@link http://www.davidmonllao.com}
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class activities_due extends \core_analytics\local\indicator\binary {
+
+    /**
+     * Returns the name.
+     *
+     * If there is a corresponding '_help' string this will be shown as well.
+     *
+     * @return \lang_string
+     */
+    public static function get_name() : \lang_string {
+        return new \lang_string('indicator:activitiesdue');
+    }
+
+    /**
+     * required_sample_data
+     *
+     * @return string[]
+     */
+    public static function required_sample_data() {
+        return array('user');
+    }
+
+    /**
+     * calculate_sample
+     *
+     * @param int $sampleid
+     * @param string $sampleorigin
+     * @param int $starttime
+     * @param int $endtime
+     * @return float
+     */
+    protected function calculate_sample($sampleid, $sampleorigin, $starttime = false, $endtime = false) {
+
+        $actionevents = \core_calendar_external::get_calendar_action_events_by_timesort($starttime, $endtime, 0, 1,
+            true, $sampleid);
+
+        if ($actionevents->events) {
+
+            // We first need to check that at least one of the core_calendar_provide_event_action
+            // callbacks has the $userid param.
+            foreach ($actionevents->events as $event) {
+                $nparams = $this->get_provide_event_action_num_params($event->modulename);
+                if ($nparams > 2) {
+                    return self::get_max_value();
+                }
+            }
+        }
+
+        return self::get_min_value();
+    }
+
+    /**
+     * Returns the number of params declared in core_calendar_provide_event_action's implementation.
+     *
+     * @param  string $modulename The module name
+     * @return int
+     */
+    private function get_provide_event_action_num_params(string $modulename) {
+        $functionname = 'mod_' . $modulename . '_core_calendar_provide_event_action';
+        $reflection = new \ReflectionFunction($functionname);
+        return $reflection->getNumberOfParameters();
+    }
+}

course/delete.php

@@ -24,6 +24,7 @@
 
 require_once(__DIR__ . '/../config.php');
 require_once($CFG->dirroot . '/course/lib.php');
+require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 
 $id = required_param('id', PARAM_INT); // Course ID.
 $delete = optional_param('delete', '', PARAM_ALPHANUM); // Confirmation hash.
@@ -74,16 +75,26 @@ if ($delete === md5($course->timemodified)) {
 }
 
 $strdeletecheck = get_string("deletecheck", "", $courseshortname);
-$strdeletecoursecheck = get_string("deletecoursecheck");
-$message = "{$strdeletecoursecheck}<br /><br />{$coursefullname} ({$courseshortname})";
-
-$continueurl = new moodle_url('/course/delete.php', array('id' => $course->id, 'delete' => md5($course->timemodified)));
-$continuebutton = new single_button($continueurl, get_string('delete'), 'post');
 
 $PAGE->navbar->add($strdeletecheck);
 $PAGE->set_title("$SITE->shortname: $strdeletecheck");
 $PAGE->set_heading($SITE->fullname);
 echo $OUTPUT->header();
-echo $OUTPUT->confirm($message, $continuebutton, $categoryurl);
+
+// Only let user delete this course if there is not an async backup in progress.
+if (!async_helper::is_async_pending($id, 'course', 'backup')) {
+    $strdeletecoursecheck = get_string("deletecoursecheck");
+    $message = "{$strdeletecoursecheck}<br /><br />{$coursefullname} ({$courseshortname})";
+
+    $continueurl = new moodle_url('/course/delete.php', array('id' => $course->id, 'delete' => md5($course->timemodified)));
+    $continuebutton = new single_button($continueurl, get_string('delete'), 'post');
+    echo $OUTPUT->confirm($message, $continuebutton, $categoryurl);
+} else {
+    // Async backup is pending, don't let user delete course.
+    echo $OUTPUT->notification(get_string('pendingasyncerror', 'backup'), 'error');
+    echo $OUTPUT->container(get_string('pendingasyncdeletedetail', 'backup'));
+    echo $OUTPUT->continue_button($categoryurl);
+}
+
 echo $OUTPUT->footer();
 exit;

course/format/renderer.php

@@ -117,10 +117,9 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
                 $name = empty($value['name']) ? '' : $value['name'];
                 $attr = empty($value['attr']) ? array() : $value['attr'];
                 $class = empty($value['pixattr']['class']) ? '' : $value['pixattr']['class'];
-                $alt = empty($value['pixattr']['alt']) ? '' : $value['pixattr']['alt'];
                 $al = new action_menu_link_secondary(
                     new moodle_url($url),
-                    new pix_icon($icon, $alt, null, array('class' => "smallicon " . $class)),
+                    new pix_icon($icon, '', null, array('class' => "smallicon " . $class)),
                     $name,
                     $attr
                 );
@@ -328,8 +327,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
                 'url'   => new moodle_url('/course/editsection.php', array('id' => $section->id, 'sr' => $sectionreturn)),
                 'icon' => 'i/settings',
                 'name' => $streditsection,
-                'pixattr' => array('class' => '', 'alt' => $streditsection),
-                'attr' => array('class' => 'icon edit', 'title' => $streditsection));
+                'pixattr' => array('class' => ''),
+                'attr' => array('class' => 'icon edit'));
         }
 
         if ($section->section) {
@@ -343,8 +342,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
                             'url' => $url,
                             'icon' => 'i/hide',
                             'name' => $strhidefromothers,
-                            'pixattr' => array('class' => '', 'alt' => $strhidefromothers),
-                            'attr' => array('class' => 'icon editing_showhide', 'title' => $strhidefromothers,
+                            'pixattr' => array('class' => ''),
+                            'attr' => array('class' => 'icon editing_showhide',
                                 'data-sectionreturn' => $sectionreturn, 'data-action' => 'hide'));
                     } else {
                         $strshowfromothers = get_string('showfromothers', 'format_'.$course->format);
@@ -353,8 +352,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
                             'url' => $url,
                             'icon' => 'i/show',
                             'name' => $strshowfromothers,
-                            'pixattr' => array('class' => '', 'alt' => $strshowfromothers),
-                            'attr' => array('class' => 'icon editing_showhide', 'title' => $strshowfromothers,
+                            'pixattr' => array('class' => ''),
+                            'attr' => array('class' => 'icon editing_showhide',
                                 'data-sectionreturn' => $sectionreturn, 'data-action' => 'show'));
                     }
                 }
@@ -370,8 +369,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
                                 'url' => $url,
                                 'icon' => 'i/up',
                                 'name' => $strmoveup,
-                                'pixattr' => array('class' => '', 'alt' => $strmoveup),
-                                'attr' => array('class' => 'icon moveup', 'title' => $strmoveup));
+                                'pixattr' => array('class' => ''),
+                                'attr' => array('class' => 'icon moveup'));
                         }
 
                         $url = clone($baseurl);
@@ -383,8 +382,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
                                 'url' => $url,
                                 'icon' => 'i/down',
                                 'name' => $strmovedown,
-                                'pixattr' => array('class' => '', 'alt' => $strmovedown),
-                                'attr' => array('class' => 'icon movedown', 'title' => $strmovedown));
+                                'pixattr' => array('class' => ''),
+                                'attr' => array('class' => 'icon movedown'));
                         }
                     }
                 }
@@ -405,8 +404,8 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
                     'url' => $url,
                     'icon' => 'i/delete',
                     'name' => $strdelete,
-                    'pixattr' => array('class' => '', 'alt' => $strdelete),
-                    'attr' => array('class' => 'icon editing_delete', 'title' => $strdelete));
+                    'pixattr' => array('class' => ''),
+                    'attr' => array('class' => 'icon editing_delete'));
             }
         }
 
@@ -1009,7 +1008,7 @@ abstract class format_section_renderer_base extends plugin_renderer_base {
             if ($sectionreturn !== null) {
                 $url->param('sectionreturn', $sectionreturn);
             }
-            $icon = $this->output->pix_icon('t/add', $straddsections);
+            $icon = $this->output->pix_icon('t/add', '');
             $newsections = $maxsections - $lastsection;
             echo html_writer::link($url, $icon . $straddsections,
                 array('class' => 'add-sections', 'data-add-sections' => $straddsections, 'new-sections' => $newsections));

course/format/topics/renderer.php

@@ -123,21 +123,19 @@ class format_topics_renderer extends format_section_renderer_base {
         if ($section->section && has_capability('moodle/course:setcurrentsection', $coursecontext)) {
             if ($course->marker == $section->section) {  // Show the "light globe" on/off.
                 $url->param('marker', 0);
-                $markedthistopic = get_string('markedthistopic');
                 $highlightoff = get_string('highlightoff');
                 $controls['highlight'] = array('url' => $url, "icon" => 'i/marked',
                                                'name' => $highlightoff,
-                                               'pixattr' => array('class' => '', 'alt' => $markedthistopic),
-                                               'attr' => array('class' => 'editing_highlight', 'title' => $markedthistopic,
+                                               'pixattr' => array('class' => ''),
+                                               'attr' => array('class' => 'editing_highlight',
                                                    'data-action' => 'removemarker'));
             } else {
                 $url->param('marker', $section->section);
-                $markthistopic = get_string('markthistopic');
                 $highlight = get_string('highlight');
                 $controls['highlight'] = array('url' => $url, "icon" => 'i/marker',
                                                'name' => $highlight,
-                                               'pixattr' => array('class' => '', 'alt' => $markthistopic),
-                                               'attr' => array('class' => 'editing_highlight', 'title' => $markthistopic,
+                                               'pixattr' => array('class' => ''),
+                                               'attr' => array('class' => 'editing_highlight',
                                                    'data-action' => 'setmarker'));
             }
         }

course/lib.php

@@ -1891,7 +1891,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
     if ($hasmanageactivities) {
         $actions['update'] = new action_menu_link_secondary(
             new moodle_url($baseurl, array('update' => $mod->id)),
-            new pix_icon('t/edit', $str->editsettings, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+            new pix_icon('t/edit', '', 'moodle', array('class' => 'iconsmall')),
             $str->editsettings,
             array('class' => 'editing_update', 'data-action' => 'update')
         );
@@ -1917,7 +1917,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
         }
         $actions['moveright'] = new action_menu_link_secondary(
             new moodle_url($baseurl, array('id' => $mod->id, 'indent' => '1')),
-            new pix_icon($rightarrow, $str->moveright, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+            new pix_icon($rightarrow, '', 'moodle', array('class' => 'iconsmall')),
             $str->moveright,
             array('class' => 'editing_moveright ' . $enabledclass, 'data-action' => 'moveright',
                 'data-keepopen' => true, 'data-sectionreturn' => $sr)
@@ -1930,7 +1930,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
         }
         $actions['moveleft'] = new action_menu_link_secondary(
             new moodle_url($baseurl, array('id' => $mod->id, 'indent' => '-1')),
-            new pix_icon($leftarrow, $str->moveleft, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+            new pix_icon($leftarrow, '', 'moodle', array('class' => 'iconsmall')),
             $str->moveleft,
             array('class' => 'editing_moveleft ' . $enabledclass, 'data-action' => 'moveleft',
                 'data-keepopen' => true, 'data-sectionreturn' => $sr)
@@ -1953,7 +1953,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
         if ($displayedoncoursepage) {
             $actions['hide'] = new action_menu_link_secondary(
                 new moodle_url($baseurl, array('hide' => $mod->id)),
-                new pix_icon('t/hide', $str->modhide, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+                new pix_icon('t/hide', '', 'moodle', array('class' => 'iconsmall')),
                 $str->modhide,
                 array('class' => 'editing_hide', 'data-action' => 'hide')
             );
@@ -1961,7 +1961,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
             // Offer to "show" only if the section is visible.
             $actions['show'] = new action_menu_link_secondary(
                 new moodle_url($baseurl, array('show' => $mod->id)),
-                new pix_icon('t/show', $str->modshow, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+                new pix_icon('t/show', '', 'moodle', array('class' => 'iconsmall')),
                 $str->modshow,
                 array('class' => 'editing_show', 'data-action' => 'show')
             );
@@ -1971,7 +1971,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
             // When making the "stealth" module unavailable we perform the same action as hiding the visible module.
             $actions['hide'] = new action_menu_link_secondary(
                 new moodle_url($baseurl, array('hide' => $mod->id)),
-                new pix_icon('t/unblock', $str->makeunavailable, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+                new pix_icon('t/unblock', '', 'moodle', array('class' => 'iconsmall')),
                 $str->makeunavailable,
                 array('class' => 'editing_makeunavailable', 'data-action' => 'hide', 'data-sectionreturn' => $sr)
             );
@@ -1982,7 +1982,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
             $action = $sectionvisible ? 'stealth' : 'show';
             $actions[$action] = new action_menu_link_secondary(
                 new moodle_url($baseurl, array($action => $mod->id)),
-                new pix_icon('t/block', $str->makeavailable, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+                new pix_icon('t/block', '', 'moodle', array('class' => 'iconsmall')),
                 $str->makeavailable,
                 array('class' => 'editing_makeavailable', 'data-action' => $action, 'data-sectionreturn' => $sr)
             );
@@ -1995,7 +1995,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
             course_allowed_module($mod->get_course(), $mod->modname)) {
         $actions['duplicate'] = new action_menu_link_secondary(
             new moodle_url($baseurl, array('duplicate' => $mod->id)),
-            new pix_icon('t/copy', $str->duplicate, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+            new pix_icon('t/copy', '', 'moodle', array('class' => 'iconsmall')),
             $str->duplicate,
             array('class' => 'editing_duplicate', 'data-action' => 'duplicate', 'data-sectionreturn' => $sr)
         );
@@ -2026,7 +2026,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
 
             $actions[$actionname] = new action_menu_link_primary(
                 new moodle_url($baseurl, array('id' => $mod->id, 'groupmode' => $nextgroupmode)),
-                new pix_icon($groupimage, $grouptitle, 'moodle', array('class' => 'iconsmall')),
+                new pix_icon($groupimage, '', 'moodle', array('class' => 'iconsmall')),
                 $grouptitle,
                 array('class' => 'editing_'. $actionname, 'data-action' => $nextactionname,
                     'aria-live' => 'assertive', 'data-sectionreturn' => $sr)
@@ -2040,7 +2040,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
     if (has_capability('moodle/role:assign', $modcontext)){
         $actions['assign'] = new action_menu_link_secondary(
             new moodle_url('/admin/roles/assign.php', array('contextid' => $modcontext->id)),
-            new pix_icon('t/assignroles', $str->assign, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+            new pix_icon('t/assignroles', '', 'moodle', array('class' => 'iconsmall')),
             $str->assign,
             array('class' => 'editing_assign', 'data-action' => 'assignroles', 'data-sectionreturn' => $sr)
         );
@@ -2050,7 +2050,7 @@ function course_get_cm_edit_actions(cm_info $mod, $indent = -1, $sr = null) {
     if ($hasmanageactivities) {
         $actions['delete'] = new action_menu_link_secondary(
             new moodle_url($baseurl, array('delete' => $mod->id)),
-            new pix_icon('t/delete', $str->delete, 'moodle', array('class' => 'iconsmall', 'title' => '')),
+            new pix_icon('t/delete', '', 'moodle', array('class' => 'iconsmall')),
             $str->delete,
             array('class' => 'editing_delete', 'data-action' => 'delete', 'data-sectionreturn' => $sr)
         );

course/tests/behat/behat_course.php

@@ -306,7 +306,7 @@ class behat_course extends behat_base {
 
         // Click on highlight topic link.
         $this->execute('behat_general::i_click_on_in_the',
-            array(get_string('markthistopic'), "link", $this->escape($xpath), "xpath_element")
+            array(get_string('highlight'), "link", $this->escape($xpath), "xpath_element")
         );
     }
 
@@ -328,7 +328,7 @@ class behat_course extends behat_base {
 
         // Click on un-highlight topic link.
         $this->execute('behat_general::i_click_on_in_the',
-            array(get_string('markedthistopic'), "link", $this->escape($xpath), "xpath_element")
+            array(get_string('highlightoff'), "link", $this->escape($xpath), "xpath_element")
         );
     }
 
@@ -446,7 +446,7 @@ class behat_course extends behat_base {
         $xpath = $this->section_exists($sectionnumber);
 
         // The important checking, we can not check the img.
-        $this->execute('behat_general::should_exist_in_the', ['This topic is highlighted as the current topic', 'icon', $xpath, 'xpath_element']);
+        $this->execute('behat_general::should_exist_in_the', ['Remove highlight', 'link', $xpath, 'xpath_element']);
     }
 
     /**
@@ -777,11 +777,10 @@ class behat_course extends behat_base {
         // Ensure the destination is valid.
         $sectionxpath = $this->section_exists($sectionnumber);
 
-        $activitynode = $this->get_activity_element('Move', 'icon', $activityname);
-
         // JS enabled.
         if ($this->running_javascript()) {
 
+            $activitynode = $this->get_activity_element('Move', 'icon', $activityname);
             $destinationxpath = $sectionxpath . "/descendant::ul[contains(concat(' ', normalize-space(@class), ' '), ' yui3-dd-drop ')]";
 
             $this->execute("behat_general::i_drag_and_i_drop_it_in",
@@ -1186,9 +1185,9 @@ class behat_course extends behat_base {
 
         // Checking the show button alt text and show icon.
         $showtext = get_string('showfromothers', $courseformat);
-        $linkxpath = $xpath . "/descendant::a[@title=" . behat_context_helper::escape($showtext) . "]";
+        $linkxpath = $xpath . "//a[*[contains(text(), " . behat_context_helper::escape($showtext) . ")]]";
 
-        $exception = new ElementNotFoundException($this->getSession(), 'Show section link ');
+        $exception = new ElementNotFoundException($this->getSession(), 'Show section link');
 
         // Returing the link so both Non-JS and JS browsers can interact with it.
         return $this->find('xpath', $linkxpath, $exception);

course/view.php

@@ -5,6 +5,7 @@
     require_once('../config.php');
     require_once('lib.php');
     require_once($CFG->libdir.'/completionlib.php');
+    require_once($CFG->dirroot . '/backup/util/includes/backup_includes.php');
 
     $id          = optional_param('id', 0, PARAM_INT);
     $name        = optional_param('name', '', PARAM_TEXT);
@@ -243,6 +244,10 @@
     $PAGE->set_heading($course->fullname);
     echo $OUTPUT->header();
 
+if ($USER->editing == 1 && async_helper::is_async_pending($id, 'course', 'backup')) {
+    echo $OUTPUT->notification(get_string('pendingasyncedit', 'backup'), 'warning');
+}
+
     if ($completion->is_enabled()) {
         // This value tracks whether there has been a dynamic change to the page.
         // It is used so that if a user does this - (a) set some tickmarks, (b)

dataformat/pdf/classes/privacy/provider.php

@@ -0,0 +1,46 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Privacy provider implementation for dataformat_pdf.
+ *
+ * @package    dataformat_pdf
+ * @copyright  2019 Shamim Rezaie <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace dataformat_pdf\privacy;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Privacy provider implementation for dataformat_pdf.
+ *
+ * @copyright  2019 Shamim Rezaie <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class provider implements \core_privacy\local\metadata\null_provider {
+
+    /**
+     * Get the language string identifier with the component's language
+     * file to explain why this plugin stores no data.
+     *
+     * @return  string
+     */
+    public static function get_reason() : string {
+        return 'privacy:metadata';
+    }
+}

dataformat/pdf/classes/writer.php

@@ -0,0 +1,139 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * pdf data format writer
+ *
+ * @package    dataformat_pdf
+ * @copyright  2019 Shamim Rezaie <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace dataformat_pdf;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * pdf data format writer
+ *
+ * @package    dataformat_pdf
+ * @copyright  2019 Shamim Rezaie <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class writer extends \core\dataformat\base {
+
+    public $mimetype = "application/pdf";
+
+    public $extension = ".pdf";
+
+    /**
+     * @var \pdf The pdf object that is used to generate the pdf file.
+     */
+    protected $pdf;
+
+    /**
+     * @var float Each column's width in the current sheet.
+     */
+    protected $colwidth;
+
+    /**
+     * @var string[] Title of columns in the current sheet.
+     */
+    protected $columns;
+
+    /**
+     * writer constructor.
+     */
+    public function __construct() {
+        global $CFG;
+        require_once($CFG->libdir . '/pdflib.php');
+
+        $this->pdf = new \pdf();
+        $this->pdf->setPrintHeader(false);
+        $this->pdf->SetFooterMargin(PDF_MARGIN_FOOTER);
+
+        // Set background color for headings.
+        $this->pdf->SetFillColor(238, 238, 238);
+    }
+
+    public function send_http_headers() {
+    }
+
+    public function start_output() {
+        $this->pdf->AddPage('L');
+    }
+
+    public function start_sheet($columns) {
+        $margins = $this->pdf->getMargins();
+        $pagewidth = $this->pdf->getPageWidth() - $margins['left'] - $margins['right'];
+
+        $this->colwidth = $pagewidth / count($columns);
+        $this->columns = $columns;
+
+        $this->print_heading();
+    }
+
+    public function write_record($record, $rownum) {
+        $rowheight = 0;
+
+        foreach ($record as $cell) {
+            $rowheight = max($rowheight, $this->pdf->getStringHeight($this->colwidth, $cell, false, true, '', 1));
+        }
+
+        $margins = $this->pdf->getMargins();
+        if ($this->pdf->GetY() + $rowheight + $margins['bottom'] > $this->pdf->getPageHeight()) {
+            $this->pdf->AddPage('L');
+            $this->print_heading();
+        }
+
+        $total = count($record);
+        $counter = 1;
+        foreach ($record as $cell) {
+            $nextposition = ($counter == $total) ? 1 : 0;
+            $this->pdf->Multicell($this->colwidth, $rowheight, $cell, 1, 'L', false, $nextposition);
+            $counter++;
+        }
+    }
+
+    public function close_output() {
+        $filename = $this->filename . $this->get_extension();
+
+        $this->pdf->Output($filename, 'D');
+    }
+
+    /**
+     * Prints the heading row.
+     */
+    private function print_heading() {
+        $fontfamily = $this->pdf->getFontFamily();
+        $fontstyle = $this->pdf->getFontStyle();
+        $this->pdf->SetFont($fontfamily, 'B');
+        $rowheight = 0;
+        foreach ($this->columns as $columns) {
+            $rowheight = max($rowheight, $this->pdf->getStringHeight($this->colwidth, $columns, false, true, '', 1));
+        }
+
+        $total = count($this->columns);
+        $counter = 1;
+        foreach ($this->columns as $columns) {
+            $nextposition = ($counter == $total) ? 1 : 0;
+            $this->pdf->Multicell($this->colwidth, $rowheight, $columns, 1, 'C', true, $nextposition);
+            $counter++;
+        }
+
+        $this->pdf->SetFont($fontfamily, $fontstyle);
+    }
+}

dataformat/pdf/lang/en/dataformat_pdf.php

@@ -0,0 +1,28 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * pdf dataformat lang strings.
+ *
+ * @package    dataformat_pdf
+ * @copyright  2019 Shamim Rezaie <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+$string['dataformat'] = 'Portable Document Format (.pdf)';
+$string['privacy:metadata'] = 'The PDF data format plugin does not store any personal data.';
+$string['shortname'] = 'PDF';
+

dataformat/pdf/version.php

@@ -0,0 +1,29 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Data activity filter version information
+ *
+ * @package    dataformat_pdf
+ * @copyright  2019 Shamim Rezaie <[email protected]>
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+defined('MOODLE_INTERNAL') || die();
+
+$plugin->version   = 2019040100;
+$plugin->requires  = 2018112800;  // Requires this Moodle version.
+$plugin->component = 'dataformat_pdf';

lang/en/admin.php

@@ -1216,6 +1216,7 @@ $string['taskanalyticscleanup'] = 'Analytics cleanup';
 $string['taskautomatedbackup'] = 'Automated backups';
 $string['taskbackupcleanup'] = 'Clean backup tables and logs';
 $string['taskbadgescron'] = 'Award badges';
+$string['taskbadgesmessagecron'] = 'Background processing for sending badges notifications';
 $string['taskblogcron'] = 'Sync external blogs';
 $string['taskcachecleanup'] = 'Remove expired cache entries';
 $string['taskcachecron'] = 'Background processing for caches';

lang/en/analytics.php

@@ -67,9 +67,10 @@ $string['errorunknownaction'] = 'Unknown action';
 $string['eventpredictionactionstarted'] = 'Prediction process started';
 $string['eventinsightsviewed'] = 'Insights viewed';
 $string['fixedack'] = 'Acknowledged';
-$string['insightmessagesubject'] = 'New insight for "{$a->contextname}": {$a->insightname}';
-$string['insightinfomessage'] = 'The system generated some insights for you: {$a}';
-$string['insightinfomessagehtml'] = 'The system generated some insights for you: <a href="{$a}">{$a}</a>.';
+$string['insightmessagesubject'] = 'New insight for "{$a}"';
+$string['insightinfomessage'] = 'The system generated an insight for you: {$a}';
+$string['insightinfomessagehtml'] = 'The system generated an insight for you.';
+$string['insightinfomessageaction'] = '{$a->text}: {$a->url}';
 $string['invalidtimesplitting'] = 'Model with ID {$a} needs a time-splitting method before it can be used to train.';
 $string['invalidanalysablefortimesplitting'] = 'It cannot be analysed using {$a} time-splitting method.';
 $string['nocourses'] = 'No courses to analyse';
@@ -122,5 +123,9 @@ $string['privacy:metadata:analytics:predictionactions:timecreated'] = 'When the
 $string['processingsitecontents'] = 'Processing site contents';
 $string['successfullyanalysed'] = 'Successfully analysed';
 $string['timesplittingmethod'] = 'Time-splitting method';
-$string['timesplittingmethod_help'] = 'The time-splitting method is what defines when the system will calculate predictions and the portion of activity logs that will be considered for those predictions. E.g. They can divide the course duration in parts and generate a prediction at the end of these parts.';
+$string['timesplittingmethod_help'] = 'The time-splitting method is what defines when the system will calculate predictions and the portion of activity logs that will be considered for those predictions. E.g. they can divide the course duration in parts and generate a prediction at the end of these parts.';
+$string['timesplittingmethod_link'] = 'Time_splitting_methods';
+$string['viewdetails'] = 'View details';
+$string['viewinsight'] = 'View insight';
+$string['viewinsightdetails'] = 'View insight details';
 $string['viewprediction'] = 'View prediction details';

lang/en/backup.php

@@ -22,6 +22,36 @@
  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  */
 
+$string['asyncbackupcomplete'] = 'The backup process has completed';
+$string['asyncbackupcompletebutton'] = 'Continue';
+$string['asyncbackupcompletedetail'] = 'The backup process has completed successfully completed. <br/> You can access the backup in the <a href="{$a}">restore page.</a>';
+$string['asyncbackuperror'] = 'The backup process has failed';
+$string['asyncbackuperrordetail'] = 'The backup process has failed. Please contact your system administrator.';
+$string['asyncbackuppending'] = 'The backup process is pending';
+$string['asyncbackupprocessing'] = 'The backup is in progress';
+$string['asyncbadexecution'] = 'Bad backup controller execution, is {$a} should be 2';
+$string['asynccheckprogress'] = ' You can check the progress at anytime at the <a href="{$a}">restore page.</a>';
+$string['asyncgeneralsettings'] = 'Asynchronous backup/restore general settings';
+$string['asyncemailenable'] = 'Enable message notifications';
+$string['asyncemailenabledetail'] = 'When enabled users will receive a message when an asynchronous restore/backup completes';
+$string['asyncmessagebody'] = 'Message';
+$string['asyncmessagebodydetail'] = 'Message to send when an asynchronous restore/backup completes';
+$string['asyncmessagebodydefault'] = 'Dear {user_firstname} {user_lastname}, <br/> Your {operation} (ID: {backupid}) has completed successfully! <br/><br/>You can view it here {link}.<br/>Kind Regards,<br/>Your Moodle Administrator.';
+$string['asyncmessagesubject'] = 'Subject';
+$string['asyncmessagesubjectdetail'] = 'Message subject';
+$string['asyncmessagesubjectdefault'] = 'Moodle {operation} completed successfully';
+$string['asyncnowait'] = 'You don\'t need to wait here, the process will continue in the background.';
+$string['asyncprocesspending'] = 'Process pending';
+$string['asyncrestorecomplete'] = 'The restore process has completed';
+$string['asyncrestorecompletebutton'] = 'Continue';
+$string['asyncrestorecompletedetail'] = 'The restore process has completed successfully completed. Clicking continue will take you to the <a href="{$a}">course for the restored item.</a>';
+$string['asyncrestoreerror'] = 'The restore process has failed';
+$string['asyncrestoreerrordetail'] = 'The restore process has failed. Please contact your system administrator.';
+$string['asyncrestorepending'] = 'The restore process is pending';
+$string['asyncrestoreprocessing'] = 'The restore is in progress';
+$string['asyncreturn'] = 'Return to course';
+$string['asyncrestoreinprogress'] = 'Restores in progress';
+$string['asyncrestoreinprogress_help'] = 'Asynchronous course restores that are in progress are shown here.';
 $string['autoactivedisabled'] = 'Disabled';
 $string['autoactiveenabled'] = 'Enabled';
 $string['autoactivemanual'] = 'Manual';
@@ -61,6 +91,7 @@ $string['backupmode30'] = 'Hub';
 $string['backupmode40'] = 'Same site';
 $string['backupmode50'] = 'Automated';
 $string['backupmode60'] = 'Converted';
+$string['backupmode70'] = 'Asynchronous';
 $string['backupsection'] = 'Backup course section: {$a}';
 $string['backupsettings'] = 'Backup settings';
 $string['backupsitedetails'] = 'Site details';
@@ -137,6 +168,8 @@ $string['currentstage2'] = 'Schema settings';
 $string['currentstage4'] = 'Confirmation and review';
 $string['currentstage8'] = 'Perform backup';
 $string['currentstage16'] = 'Complete';
+$string['enableasyncbackup'] = 'Enable asynchronous backups';
+$string['enableasyncbackup_help'] = 'If enabled, all backup and restore operations will be done asynchronously. This does not effect imports and exports. Asynchronous backups and restores allow users to do other operations while a backup or restore is in progress.';
 $string['enterasearch'] = 'Enter a search';
 $string['error_block_for_module_not_found'] = 'Orphan block instance (id: {$a->bid}) for course module (id: {$a->mid}) found. This block will not be backed up';
 $string['error_course_module_not_found'] = 'Orphan course module (id: {$a}) found. This module will not be backed up.';
@@ -149,6 +182,7 @@ $string['errorinvalidformat'] = 'Unknown backup format';
 $string['errorinvalidformatinfo'] = 'The selected file is not a valid Moodle backup file and can\'t be restored.';
 $string['errorrestorefrontpagebackup'] = 'You can only restore front page backups on the front page';
 $string['executionsuccess'] = 'The backup file was successfully created.';
+$string['failed'] = 'Backup failed';
 $string['filename'] = 'Filename';
 $string['filealiasesrestorefailures'] = 'Aliases restore failures';
 $string['filealiasesrestorefailuresinfo'] = 'Some aliases included in the backup file could not be restored. The following list contains their expected location and the source file they were referring to at the original site.';
@@ -203,6 +237,7 @@ $string['importcurrentstage16'] = 'Complete';
 $string['importrootsettings'] = 'Import settings';
 $string['importsettings'] = 'General import settings';
 $string['importsuccess'] = 'Import complete. Click continue to return to the course.';
+$string['inprogress'] = 'Backup in progress';
 $string['includeactivities'] = 'Include:';
 $string['includeditems'] = 'Included items:';
 $string['includesection'] = 'Section {$a}';
@@ -223,6 +258,10 @@ $string['nomatchingcourses'] = 'There are no courses to display';
 $string['norestoreoptions'] = 'There are no categories or existing courses you can restore to.';
 $string['originalwwwroot'] = 'URL of backup';
 $string['overwrite'] = 'Overwrite';
+$string['pendingasyncdetail'] = 'Asynchronous backups only allow a user to have one pending backup for a resource at a time. <br/> Muliple asynchronous backups of the same resource can\'t be queued, as this would likely result in multiple backups with the same content.';
+$string['pendingasyncdeletedetail'] = 'This course has an asynchronous backup pending. <br/> Courses can\'t be deleted until this backup finishes.';
+$string['pendingasyncedit'] = 'There is a pending asynchronous backup for this course. Please do not edit this course until backup is complete.';
+$string['pendingasyncerror'] = 'Backup pending for this resource';
 $string['previousstage'] = 'Previous';
 $string['preparingui'] = 'Preparing to display page';
 $string['preparingdata'] = 'Preparing data';
@@ -323,6 +362,9 @@ $string['skipmodifdays'] = 'Skip courses not modified since';
 $string['skipmodifdayshelp'] = 'Choose to skip courses that have not been modified since a number of days';
 $string['skipmodifprev'] = 'Skip courses not modified since previous backup';
 $string['skipmodifprevhelp'] = 'Choose whether to skip courses that have not been modified since the last automatic backup. This requires logging to be enabled.';
+$string['status'] = 'Status';
+$string['successful'] = 'Backup successful';
+$string['successfulrestore'] = 'Restore successful';
 $string['timetaken'] = 'Time taken';
 $string['title'] = 'Title';
 $string['totalcategorysearchresults'] = 'Total categories: {$a}';

lang/en/course.php

@@ -36,6 +36,7 @@ $string['customfield_visibletoall'] = 'Everyone';
 $string['customfield_visibletoteachers'] = 'Teachers';
 $string['customfieldsettings'] = 'Settings for course custom fields';
 $string['favourite'] = 'Starred course';
+$string['gradetopassnotset'] = 'This course does not have a grade to pass set. It may be set in the grade item of the course (Gradebook setup).';
 $string['privacy:perpage'] = 'The number of courses to show per page.';
 $string['privacy:completionpath'] = 'Course completion';
 $string['privacy:favouritespath'] = 'Course starred information';

lang/en/moodle.php

@@ -1053,6 +1053,8 @@ $string['indicator:accessesafterend'] = 'Course accessed after end date';
 $string['indicator:accessesafterend_help'] = 'This indicator reflects if the student accessed the course after the course end date.';
 $string['indicator:accessesbeforestart'] = 'Course accessed before start date';
 $string['indicator:accessesbeforestart_help'] = 'This indicator reflects if the student accessed the course before the course start date.';
+$string['indicator:activitiesdue'] = 'Activities due';
+$string['indicator:activitiesdue_help'] = 'The user has activities due.';
 $string['indicator:anywrite'] = 'Any write action';
 $string['indicator:anywrite_help'] = 'This indicator represents any write (submit) action taken by the student.';
 $string['indicator:anywriteincourse'] = 'Any write action in the course';
@@ -1188,6 +1190,7 @@ $string['memberincourse'] = 'People in the course';
 $string['messagebody'] = 'Message body';
 $string['messagedselectedusers'] = 'Selected users have been messaged and the recipient list has been reset.';
 $string['messagedselectedusersfailed'] = 'Something went wrong while messaging selected users.  Some may have received the email.';
+$string['messageprovider:asyncbackupnotification'] = 'Asynchronous backup/restore notifications';
 $string['messageprovider:availableupdate'] = 'Available update notifications';
 $string['messageprovider:backup'] = 'Backup notifications';
 $string['messageprovider:badgecreatornotice'] = 'Badge creator notifications';
@@ -1428,6 +1431,7 @@ $string['nosuchemail'] = 'No such email address';
 $string['notavailable'] = 'Not available';
 $string['notavailablecourse'] = '{$a} is not available';
 $string['noteachersyet'] = 'No teachers in this course yet';
+$string['noteachingupcomingcourses'] = 'Upcoming courses have no teachers or students';
 $string['notenrolled'] = '{$a} is not enrolled in this course.';
 $string['notenrolledprofile'] = 'This profile is not available because this user is not enrolled in this course.';
 $string['noteusercannotrolldatesoncontext'] = '<strong>Note:</strong> The ability to roll dates when restoring this backup has been disabled because you lack the required permissions.';
@@ -1947,6 +1951,7 @@ $string['stringsnotset'] = 'The following strings are not defined in {$a}';
 $string['studentnotallowed'] = 'Sorry, but you can not enter this course as \'{$a}\'';
 $string['students'] = 'Students';
 $string['studentsandteachers'] = 'Students and teachers';
+$string['studentsatriskincourse'] = 'Students at risk in {$a} course';
 $string['subcategories'] = 'Subcategories';
 $string['subcategory'] = 'Subcategory';
 $string['subcategoryof'] = 'Subcategory of {$a}';
@@ -1976,14 +1981,20 @@ $string['target:coursecompetencies'] = 'Students at risk of not achieving the co
 $string['target:coursecompetencies_help'] = 'This target describes whether a student is at risk of not achieving the competencies assigned to a course. This target considers that all competencies assigned to the course must be achieved by the end of the course.';
 $string['target:coursedropout'] = 'Students at risk of dropping out';
 $string['target:coursedropout_help'] = 'This target describes whether the student is considered at risk of dropping out.';
+$string['target:coursegradetopass'] = 'Students at risk of not getting the minimum grade to pass the course.';
+$string['target:coursegradetopass_help'] = 'This target describes whether the student is at risk of not getting the minimum grade to pass the course.';
 $string['target:noteachingactivity'] = 'No teaching';
 $string['target:noteachingactivity_help'] = 'This target describes whether courses due to start in the coming week will have teaching activity.';
+$string['target:upcomingactivitiesdue'] = 'Upcoming activities due';
+$string['target:upcomingactivitiesdue_help'] = 'This target generates reminders for upcoming activities due.';
 $string['targetlabelstudentcompletionno'] = 'Student who is likely to meet the course completion conditions';
 $string['targetlabelstudentcompletionyes'] = 'Student at risk of not meeting the course completion conditions';
 $string['targetlabelstudentcompetenciesno'] = 'Student who is likely to achieve the competencies assigned to a course';
 $string['targetlabelstudentcompetenciesyes'] = 'Student at risk of not achieving the competencies assigned to a course';
 $string['targetlabelstudentdropoutyes'] = 'Student at risk of dropping out';
 $string['targetlabelstudentdropoutno'] = 'Not at risk';
+$string['targetlabelstudentgradetopassno'] = 'Student who is likely to meet the minimum grade to pass the course.';
+$string['targetlabelstudentgradetopassyes'] = 'Student at risk of not meeting the minimum grade to pass the course.';
 $string['targetlabelteachingyes'] = 'Users with teaching capabilities have access to the course';
 $string['targetlabelteachingno'] = 'No teaching';
 $string['targetrole'] = 'Target role';
@@ -2010,10 +2021,8 @@ $string['timesplitting:quartersaccum'] = 'Quarters accumulative';
 $string['timesplitting:quartersaccum_help'] = 'This time-splitting method divides the course into quarters (4 equal parts), with each prediction being based on the data of all previous quarters.';
 $string['timesplitting:singlerange'] = 'Single range';
 $string['timesplitting:singlerange_help'] = 'This time-splitting method considers the entire course as a single span.';
-$string['timesplitting:weekly'] = 'Weekly';
-$string['timesplitting:weekly_help'] = 'This time-splitting method divides the course into weeks (7 days), with each prediction being based on the data of only the most recent previous week.';
-$string['timesplitting:weeklyaccum'] = 'Weekly accumulative';
-$string['timesplitting:weeklyaccum_help'] = 'This time-splitting method divides the course into weeks (7 days), with each prediction being based on the data of all previous weeks.';
+$string['timesplitting:upcomingweek'] = 'Upcoming week';
+$string['timesplitting:upcomingweek_help'] = 'This time-splitting method generates predictions every week. The indicators calculations will be based on the upcoming week.';
 $string['thanks'] = 'Thanks';
 $string['theme'] = 'Theme';
 $string['themes'] = 'Themes';
@@ -2207,6 +2216,7 @@ $string['years'] = 'years';
 $string['yes'] = 'Yes';
 $string['youareabouttocreatezip'] = 'You are about to create a zip file containing';
 $string['youaregoingtorestorefrom'] = 'You are about to start the restore process for';
+$string['youhaveupcomingactivitiesdue'] = 'You have upcoming activities due';
 $string['youneedtoenrol'] = 'To perform that action you need to enrol in this course.';
 $string['yourlastlogin'] = 'Your last login was';
 $string['yourself'] = 'yourself';

lang/en/role.php

@@ -37,6 +37,7 @@ $string['allowswitch'] = 'Allow role switches';
 $string['allowview'] = 'Allow role to view';
 $string['allsiteusers'] = 'All site users';
 $string['analytics:listinsights'] = 'List insights';
+$string['analytics:listowninsights'] = 'List own insights';
 $string['analytics:managemodels'] = 'Manage models';
 $string['archetype'] = 'Role archetype';
 $string['archetype_help'] = 'The role archetype determines the permissions when a role is reset to default. It also determines any new permissions for the role when the site is upgraded.';

lib/amd/build/icon_system_fontawesome.min.js

@@ -1 +1 @@
-define(["core/icon_system","jquery","core/ajax","core/mustache","core/localstorage","core/url"],function(a,b,c,d,e,f){var g=null,h=null,i=function(){a.apply(this,arguments)};return i.prototype=Object.create(a.prototype),i.prototype.init=function(){if(g)return b.when(this);var a=e.get("core/iconmap-fontawesome");return a&&(a=JSON.parse(a)),a?(g=a,b.when(this)):(null===h&&(h=c.call([{methodname:"core_output_load_fontawesome_icon_map",args:[]}],!0,!1)[0]),h.then(function(a){return g={},b.each(a,function(a,b){g[b.component+"/"+b.pix]=b.to}),e.set("core/iconmap-fontawesome",JSON.stringify(g)),this}.bind(this)))},i.prototype.renderIcon=function(a,b,c,e){var h=g[b+"/"+a],i=!1;if("undefined"==typeof h){var j=f.imageUrl(a,b);i={attributes:[{name:"src",value:j},{name:"alt",value:c},{name:"title",value:c}]}}var k={key:h,title:c,alt:c,unmappedIcon:i},l=d.render(e,k);return l.trim()},i.prototype.getTemplateName=function(){return"core/pix_icon_fontawesome"},i});
+define(["core/icon_system","jquery","core/ajax","core/mustache","core/localstorage","core/url"],function(a,b,c,d,e,f){var g=null,h=null,i=function(){a.apply(this,arguments)};return i.prototype=Object.create(a.prototype),i.prototype.init=function(){if(g)return b.when(this);var a=e.get("core/iconmap-fontawesome");return a&&(a=JSON.parse(a)),a?(g=a,b.when(this)):(null===h&&(h=c.call([{methodname:"core_output_load_fontawesome_icon_map",args:[]}],!0,!1)[0]),h.then(function(a){return g={},b.each(a,function(a,b){g[b.component+"/"+b.pix]=b.to}),e.set("core/iconmap-fontawesome",JSON.stringify(g)),this}.bind(this)))},i.prototype.renderIcon=function(a,b,c,e){var h=g[b+"/"+a],i=!1;if("undefined"==typeof h){var j=f.imageUrl(a,b);i={attributes:[{name:"src",value:j},{name:"alt",value:c},{name:"title",value:c}]}}var k={key:h,title:c,alt:c,unmappedIcon:i};"undefined"!=typeof c&&""!==c||(k["aria-hidden"]=!0);var l=d.render(e,k);return l.trim()},i.prototype.getTemplateName=function(){return"core/pix_icon_fontawesome"},i});

lib/amd/build/icon_system_standard.min.js

@@ -1 +1 @@
-define(["core/icon_system","core/url","core/mustache"],function(a,b,c){var d=function(){a.apply(this,arguments)};return d.prototype=Object.create(a.prototype),d.prototype.renderIcon=function(a,d,e,f){var g=b.imageUrl(a,d),h={attributes:[{name:"src",value:g},{name:"alt",value:e},{name:"title",value:e}]},i=c.render(f,h);return i.trim()},d.prototype.getTemplateName=function(){return"core/pix_icon"},d});
+define(["core/icon_system","core/url","core/mustache"],function(a,b,c){var d=function(){a.apply(this,arguments)};return d.prototype=Object.create(a.prototype),d.prototype.renderIcon=function(a,d,e,f){var g=b.imageUrl(a,d),h={attributes:[{name:"src",value:g},{name:"alt",value:e},{name:"title",value:e}]};"undefined"!=typeof e&&""!=e||h.attributes.push({name:"aria-hidden",value:"true"});var i=c.render(f,h);return i.trim()},d.prototype.getTemplateName=function(){return"core/pix_icon"},d});

lib/amd/src/icon_system_fontawesome.js

@@ -105,6 +105,10 @@ define(['core/icon_system', 'jquery', 'core/ajax', 'core/mustache', 'core/locals
             unmappedIcon: unmappedIcon
         };
 
+        if (typeof title === "undefined" || title === '') {
+            context['aria-hidden'] = true;
+        }
+
         var result = Mustache.render(template, context);
         return result.trim();
     };

lib/amd/src/icon_system_standard.js

@@ -52,6 +52,10 @@ define(['core/icon_system', 'core/url', 'core/mustache'],
                 {name: 'title', value: title}
             ]
         };
+        if (typeof title === "undefined" || title == "") {
+            templatecontext.attributes.push({name: 'aria-hidden', value: 'true'});
+        }
+
         var result = Mustache.render(template, templatecontext);
         return result.trim();
     };

lib/badgeslib.php

@@ -1611,3 +1611,50 @@ function badge_award_criteria_competency_has_records_for_competencies($competenc
 
     return $DB->record_exists_sql($sql, $params);
 }
+
+/**
+ * Creates single message for all notification and sends it out
+ *
+ * @param object $badge A badge which is notified about.
+ */
+function badge_assemble_notification(stdClass $badge) {
+    global $DB;
+
+    $userfrom = core_user::get_noreply_user();
+    $userfrom->maildisplay = true;
+
+    if ($msgs = $DB->get_records_select('badge_issued', 'issuernotified IS NULL AND badgeid = ?', array($badge->id))) {
+        // Get badge creator.
+        $creator = $DB->get_record('user', array('id' => $badge->creator), '*', MUST_EXIST);
+        $creatorsubject = get_string('creatorsubject', 'badges', $badge->name);
+        $creatormessage = '';
+
+        // Put all messages in one digest.
+        foreach ($msgs as $msg) {
+            $issuedlink = html_writer::link(new moodle_url('/badges/badge.php', array('hash' => $msg->uniquehash)), $badge->name);
+            $recipient = $DB->get_record('user', array('id' => $msg->userid), '*', MUST_EXIST);
+
+            $a = new stdClass();
+            $a->user = fullname($recipient);
+            $a->link = $issuedlink;
+            $creatormessage .= get_string('creatorbody', 'badges', $a);
+            $DB->set_field('badge_issued', 'issuernotified', time(), array('badgeid' => $msg->badgeid, 'userid' => $msg->userid));
+        }
+
+        // Create a message object.
+        $eventdata = new \core\message\message();
+        $eventdata->courseid          = SITEID;
+        $eventdata->component         = 'moodle';
+        $eventdata->name              = 'badgecreatornotice';
+        $eventdata->userfrom          = $userfrom;
+        $eventdata->userto            = $creator;
+        $eventdata->notification      = 1;
+        $eventdata->subject           = $creatorsubject;
+        $eventdata->fullmessage       = format_text_email($creatormessage, FORMAT_HTML);
+        $eventdata->fullmessageformat = FORMAT_PLAIN;
+        $eventdata->fullmessagehtml   = $creatormessage;
+        $eventdata->smallmessage      = $creatorsubject;
+
+        message_send($eventdata);
+    }
+}

lib/behat/classes/partial_named_selector.php

@@ -89,6 +89,7 @@ class behat_partial_named_selector extends \Behat\Mink\Selector\PartialNamedSele
         'file' => 'file',
         'filemanager' => 'filemanager',
         'group_message' => 'group_message',
+        'group_message_conversation' => 'group_message_conversation',
         'group_message_header' => 'group_message_header',
         'group_message_member' => 'group_message_member',
         'group_message_tab' => 'group_message_tab',
@@ -154,7 +155,10 @@ XPATH
         , 'group_message' => <<<XPATH
         .//*[@data-conversation-id]//img[contains(@alt, %locator%)]/..
 XPATH
-        , 'group_message_header' => <<<XPATH
+        , 'group_message_conversation' => <<<XPATH
+            .//*[@data-region='message-drawer' and contains(., %locator%)]//div[@data-region='content-message-container']
+XPATH
+    , 'group_message_header' => <<<XPATH
         .//*[@data-region='message-drawer']//div[@data-region='header-container']//*[text()[contains(., %locator%)]]
 XPATH
     , 'group_message_member' => <<<XPATH