From ff656baeab5db885ddac82fbb36871ef589ce29a Mon Sep 17 00:00:00 2001 From: David Monllao Date: Tue, 23 May 2017 17:41:12 +0800 Subject: [PATCH] MDL-58859 analytics: Add tests to core Part of MDL-57791 epic. --- analytics/tests/course_activities_test.php | 221 ++++++++++++++ analytics/tests/course_test.php | 200 ++++++++++++ .../fixtures/test_indicator_fullname.php | 30 ++ .../tests/fixtures/test_indicator_max.php | 7 + .../tests/fixtures/test_indicator_min.php | 7 + .../tests/fixtures/test_indicator_random.php | 10 + .../tests/fixtures/test_target_shortname.php | 59 ++++ analytics/tests/model_test.php | 151 +++++++++ analytics/tests/prediction_test.php | 287 ++++++++++++++++++ lib/tests/component_test.php | 2 +- 10 files changed, 973 insertions(+), 1 deletion(-) create mode 100644 analytics/tests/course_activities_test.php create mode 100644 analytics/tests/course_test.php create mode 100644 analytics/tests/fixtures/test_indicator_fullname.php create mode 100644 analytics/tests/fixtures/test_indicator_max.php create mode 100644 analytics/tests/fixtures/test_indicator_min.php create mode 100644 analytics/tests/fixtures/test_indicator_random.php create mode 100644 analytics/tests/fixtures/test_target_shortname.php create mode 100644 analytics/tests/model_test.php create mode 100644 analytics/tests/prediction_test.php diff --git a/analytics/tests/course_activities_test.php b/analytics/tests/course_activities_test.php new file mode 100644 index 00000000000..2d70b1b9e68 --- /dev/null +++ b/analytics/tests/course_activities_test.php @@ -0,0 +1,221 @@ +. + +/** + * Unit tests for course activities. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Unit tests for course activities + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_analytics_course_activities_testcase extends advanced_testcase { + + public function availability_levels() { + return array( + 'activity' => array('activity'), + 'section' => array('section'), + ); + } + + /** + * test_get_activities_with_availability + * + * @dataProvider availability_levels + * @param string $availabilitylevel + * @return void + */ + public function test_get_activities_with_availability($availabilitylevel) { + + list($course, $stu1) = $this->setup_course(); + + // forum1 is ignored as section 0 does not count. + $forum = $this->getDataGenerator()->create_module('forum', array('course' => $course->id)); + + $courseman = new \core_analytics\course($course); + + $modinfo = get_fast_modinfo($course, $stu1->id); + $cm = $modinfo->get_cm($forum->cmid); + + if ($availabilitylevel === 'activity') { + $availabilityinfo = new \core_availability\info_module($cm); + } else if ($availabilitylevel === 'section') { + $availabilityinfo = new \core_availability\info_section($cm->get_modinfo()->get_section_info($cm->sectionnum)); + } else { + $this->fail('Unsupported availability level'); + } + + $fromtime = strtotime('2015-10-22 00:00:00 GMT'); + $untiltime = strtotime('2015-10-24 00:00:00 GMT'); + $structure = (object)array('op' => '|', 'show' => true, 'c' => array( + (object)array('type' => 'date', 'd' => '<', 't' => $untiltime), + (object)array('type' => 'date', 'd' => '>=', 't' => $fromtime) + )); + + $method = new ReflectionMethod($availabilityinfo, 'set_in_database'); + $method->setAccessible(true); + $method->invoke($availabilityinfo, json_encode($structure)); + + $this->setUser($stu1); + + // Reset modinfo we also want coursemodinfo cache definition to be cleared. + get_fast_modinfo($course, $stu1->id, true); + rebuild_course_cache($course->id, true); + + $modinfo = get_fast_modinfo($course, $stu1->id); + + $cm = $modinfo->get_cm($forum->cmid); + + // Condition from after provided end time. + $this->assertCount(0, $courseman->get_activities('forum', strtotime('2015-10-20 00:00:00 GMT'), strtotime('2015-10-21 00:00:00 GMT'), $stu1)); + + // Condition until before provided start time + $this->assertCount(0, $courseman->get_activities('forum', strtotime('2015-10-25 00:00:00 GMT'), strtotime('2015-10-26 00:00:00 GMT'), $stu1)); + + // Condition until after provided end time. + $this->assertCount(0, $courseman->get_activities('forum', strtotime('2015-10-22 00:00:00 GMT'), strtotime('2015-10-23 00:00:00 GMT'), $stu1)); + + // Condition until after provided start time and before provided end time. + $this->assertCount(1, $courseman->get_activities('forum', strtotime('2015-10-22 00:00:00 GMT'), strtotime('2015-10-25 00:00:00 GMT'), $stu1)); + } + + public function test_get_activities_with_weeks() { + + $startdate = gmmktime('0', '0', '0', 10, 24, 2015); + $record = array( + 'format' => 'weeks', + 'numsections' => 4, + 'startdate' => $startdate, + ); + + list($course, $stu1) = $this->setup_course($record); + + // forum1 is ignored as section 0 does not count. + $forum1 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 0)); + $forum2 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 1)); + $forum3 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 2)); + $forum4 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 4)); + $forum5 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 4)); + + $courseman = new \core_analytics\course($course); + + $this->setUser($stu1); + + $first = $startdate; + $second = $startdate + WEEKSECS; + $third = $startdate + (WEEKSECS * 2); + $forth = $startdate + (WEEKSECS * 3); + $this->assertCount(1, $courseman->get_activities('forum', $first, $first + WEEKSECS, $stu1)); + $this->assertCount(1, $courseman->get_activities('forum', $second, $second + WEEKSECS, $stu1)); + $this->assertCount(0, $courseman->get_activities('forum', $third, $third + WEEKSECS, $stu1)); + $this->assertCount(2, $courseman->get_activities('forum', $forth, $forth + WEEKSECS, $stu1)); + } + + public function test_get_activities_by_section() { + + // This makes debugging easier, sorry WA's +8 :) + $this->setTimezone('UTC'); + + // 1 year. + $startdate = gmmktime('0', '0', '0', 10, 24, 2015); + $enddate = gmmktime('0', '0', '0', 10, 24, 2016); + $numsections = 12; + $record = array( + 'format' => 'topics', + 'numsections' => $numsections, + 'startdate' => $startdate, + 'enddate' => $enddate + ); + + list($course, $stu1) = $this->setup_course($record); + + // forum1 is ignored as section 0 does not count. + $forum1 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 0)); + $forum2 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 1)); + $forum3 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 4)); + $forum4 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 8)); + $forum5 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 10)); + $forum6 = $this->getDataGenerator()->create_module('forum', array('course' => $course->id), + array('section' => 12)); + + $courseman = new \core_analytics\course($course); + + $this->setUser($stu1); + + // Split the course in quarters. + $duration = ($enddate - $startdate) / 4; + $first = $startdate; + $second = $startdate + $duration; + $third = $startdate + ($duration * 2); + $forth = $startdate + ($duration * 3); + $this->assertCount(1, $courseman->get_activities('forum', $first, $first + $duration, $stu1)); + $this->assertCount(1, $courseman->get_activities('forum', $second, $second + $duration, $stu1)); + $this->assertCount(1, $courseman->get_activities('forum', $third, $third + $duration, $stu1)); + $this->assertCount(2, $courseman->get_activities('forum', $forth, $forth + $duration, $stu1)); + + // Split the course in as many parts as sections. + $duration = ($enddate - $startdate) / $numsections; + for($i = 1; $i <= $numsections; $i++) { + // -1 because section 1 start represents the course start. + $timeranges[$i] = $startdate + ($duration * ($i - 1)); + } + $this->assertCount(1, $courseman->get_activities('forum', $timeranges[1], $timeranges[1] + $duration, $stu1)); + $this->assertCount(1, $courseman->get_activities('forum', $timeranges[4], $timeranges[4] + $duration, $stu1)); + $this->assertCount(1, $courseman->get_activities('forum', $timeranges[8], $timeranges[8] + $duration, $stu1)); + $this->assertCount(1, $courseman->get_activities('forum', $timeranges[10], $timeranges[10] + $duration, $stu1)); + $this->assertCount(1, $courseman->get_activities('forum', $timeranges[12], $timeranges[12] + $duration, $stu1)); + + // Nothing here. + $this->assertCount(0, $courseman->get_activities('forum', $timeranges[2], $timeranges[2] + $duration, $stu1)); + $this->assertCount(0, $courseman->get_activities('forum', $timeranges[3], $timeranges[3] + $duration, $stu1)); + } + + protected function setup_course($courserecord = null) { + global $CFG; + + $this->resetAfterTest(true); + + $this->setAdminUser(); + + $CFG->enableavailability = true; + + $course = $this->getDataGenerator()->create_course($courserecord); + $stu1 = $this->getDataGenerator()->create_user(); + $this->getDataGenerator()->enrol_user($stu1->id, $course->id, 'student'); + + return array($course, $stu1); + } + +} diff --git a/analytics/tests/course_test.php b/analytics/tests/course_test.php new file mode 100644 index 00000000000..4b93de5953a --- /dev/null +++ b/analytics/tests/course_test.php @@ -0,0 +1,200 @@ +. + +/** + * Unit tests for course. + * + * @package core_analytics + * @copyright 2016 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * Unit tests for course. + * + * @package core_analytics + * @copyright 2016 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_analytics_course_testcase extends advanced_testcase { + + public function setUp() { + global $DB; + + $this->course = $this->getDataGenerator()->create_course(); + $this->stu1 = $this->getDataGenerator()->create_user(); + $this->stu2 = $this->getDataGenerator()->create_user(); + $this->both = $this->getDataGenerator()->create_user(); + $this->editingteacher = $this->getDataGenerator()->create_user(); + $this->teacher = $this->getDataGenerator()->create_user(); + + $this->studentroleid = $DB->get_field('role', 'id', array('shortname' => 'student')); + $this->editingteacherroleid = $DB->get_field('role', 'id', array('shortname' => 'editingteacher')); + $this->teacherroleid = $DB->get_field('role', 'id', array('shortname' => 'teacher')); + + + $this->getDataGenerator()->enrol_user($this->stu1->id, $this->course->id, $this->studentroleid); + $this->getDataGenerator()->enrol_user($this->stu2->id, $this->course->id, $this->studentroleid); + $this->getDataGenerator()->enrol_user($this->both->id, $this->course->id, $this->studentroleid); + $this->getDataGenerator()->enrol_user($this->both->id, $this->course->id, $this->editingteacherroleid); + $this->getDataGenerator()->enrol_user($this->editingteacher->id, $this->course->id, $this->editingteacherroleid); + $this->getDataGenerator()->enrol_user($this->teacher->id, $this->course->id, $this->teacherroleid); + + + set_config('studentroles', $this->studentroleid, 'analytics'); + set_config('teacherroles', $this->editingteacherroleid . ',' . $this->teacherroleid, 'analytics'); + } + + /** + * Users tests. + */ + public function test_users() { + global $DB; + + $this->resetAfterTest(true); + + $courseman = new \core_analytics\course($this->course->id); + $this->assertCount(3, $courseman->get_user_ids(array($this->studentroleid))); + $this->assertCount(2, $courseman->get_user_ids(array($this->editingteacherroleid))); + $this->assertCount(1, $courseman->get_user_ids(array($this->teacherroleid))); + + // Distinct is applied + $this->assertCount(3, $courseman->get_user_ids(array($this->editingteacherroleid, $this->teacherroleid))); + $this->assertCount(4, $courseman->get_user_ids(array($this->editingteacherroleid, $this->studentroleid))); + } + + /** + * Course validation tests. + * + * @return void + */ + public function test_course_validation() { + global $DB; + + $this->resetAfterTest(true); + + $courseman = new \core_analytics\course($this->course->id); + $this->assertFalse($courseman->was_started()); + $this->assertFalse($courseman->is_finished()); + $this->assertFalse($courseman->is_valid()); + + // Nothing should change when assigning as teacher. + for ($i = 0; $i < 10; $i++) { + $user = $this->getDataGenerator()->create_user(); + $this->getDataGenerator()->enrol_user($user->id, $this->course->id, $this->teacherroleid); + } + $courseman = new \core_analytics\course($this->course->id); + $this->assertFalse($courseman->is_valid()); + + // More students now. + for ($i = 0; $i < 10; $i++) { + $user = $this->getDataGenerator()->create_user(); + $this->getDataGenerator()->enrol_user($user->id, $this->course->id, $this->studentroleid); + } + $courseman = new \core_analytics\course($this->course->id); + $this->assertFalse($courseman->is_valid()); + + // Valid start date unknown end date. + $this->course->startdate = gmmktime('0', '0', '0', 10, 24, 2015); + $DB->update_record('course', $this->course); + $courseman = new \core_analytics\course($this->course->id); + $this->assertTrue($courseman->was_started()); + $this->assertFalse($courseman->is_finished()); + $this->assertFalse($courseman->is_valid()); + + // Valid start and end date. + $this->course->enddate = gmmktime('0', '0', '0', 8, 27, 2016); + $DB->update_record('course', $this->course); + $courseman = new \core_analytics\course($this->course->id); + $this->assertTrue($courseman->was_started()); + $this->assertTrue($courseman->is_finished()); + $this->assertTrue($courseman->is_valid()); + + // Valid start and ongoing course. + $this->course->enddate = gmmktime('0', '0', '0', 8, 27, 2286); + $DB->update_record('course', $this->course); + $courseman = new \core_analytics\course($this->course->id); + $this->assertTrue($courseman->was_started()); + $this->assertFalse($courseman->is_finished()); + $this->assertFalse($courseman->is_valid()); + } + + /** + * Get the minimum time that is considered valid according to guess_end logic. + * + * @param int $time + * @return int + */ + protected function time_greater_than($time) { + return $time - (WEEKSECS * 2); + } + + /** + * Get the maximum time that is considered valid according to guess_end logic. + * + * @param int $time + * @return int + */ + protected function time_less_than($time) { + return $time + (WEEKSECS * 2); + } + + /** + * Generate a log. + * + * @param int $time + * @param int $userid + * @param int $courseid + * @return void + */ + protected function generate_log($time, $userid = false, $courseid = false) { + global $DB; + + if (empty($userid)) { + $userid = $this->stu1->id; + } + if (empty($courseid)) { + $courseid = $this->course->id; + } + + $context = context_course::instance($courseid); + $obj = (object)[ + 'eventname' => '\\core\\event\\course_viewed', + 'component' => 'core', + 'action' => 'viewed', + 'target' => 'course', + 'objecttable' => 'course', + 'objectid' => $courseid, + 'crud' => 'r', + 'edulevel' => \core\event\base::LEVEL_PARTICIPATING, + 'contextid' => $context->id, + 'contextlevel' => $context->contextlevel, + 'contextinstanceid' => $context->instanceid, + 'userid' => $userid, + 'courseid' => $courseid, + 'relateduserid' => null, + 'anonymous' => 0, + 'other' => null, + 'timecreated' => $time, + 'origin' => 'web', + ]; + $DB->insert_record('logstore_standard_log', $obj); + } + +} + diff --git a/analytics/tests/fixtures/test_indicator_fullname.php b/analytics/tests/fixtures/test_indicator_fullname.php new file mode 100644 index 00000000000..7d52ad7f013 --- /dev/null +++ b/analytics/tests/fixtures/test_indicator_fullname.php @@ -0,0 +1,30 @@ +retrieve('course', $sampleid); + + $firstchar = substr($course->fullname, 0, 1); + if ($firstchar === 'a') { + return self::MIN_VALUE; + } else if ($firstchar === 'b') { + return -0.2; + } else if ($firstchar === 'c') { + return 0.2; + } else { + return self::MAX_VALUE; + } + } + +} diff --git a/analytics/tests/fixtures/test_indicator_max.php b/analytics/tests/fixtures/test_indicator_max.php new file mode 100644 index 00000000000..752e843775a --- /dev/null +++ b/analytics/tests/fixtures/test_indicator_max.php @@ -0,0 +1,7 @@ +get_record('course', array('id' => $sampleid)); + + if ($sample->visible == 0) { + // We skip not-visible courses as a way to emulate the training data / prediction data difference. + // In normal circumstances targets will return null when they receive a sample that can not be + // processed, that same sample may be used for prediction. + // We can not do this in is_valid_analysable because the analysable there is the site not the course. + return null; + } + + $firstchar = substr($sample->shortname, 0, 1); + if ($firstchar === 'a') { + return 1; + } else { + return 0; + } + } +} diff --git a/analytics/tests/model_test.php b/analytics/tests/model_test.php new file mode 100644 index 00000000000..88f8543ab4f --- /dev/null +++ b/analytics/tests/model_test.php @@ -0,0 +1,151 @@ +. + +/** + * Unit tests for the model. + * + * @package analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once(__DIR__ . '/fixtures/test_indicator_max.php'); +require_once(__DIR__ . '/fixtures/test_indicator_min.php'); +require_once(__DIR__ . '/fixtures/test_indicator_fullname.php'); +require_once(__DIR__ . '/fixtures/test_target_shortname.php'); + +/** + * Unit tests for the model. + * + * @package analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class analytics_model_testcase extends advanced_testcase { + + public function setUp() { + + $target = \core_analytics\manager::get_target('test_target_shortname'); + $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_fullname'); + foreach ($indicators as $key => $indicator) { + $indicators[$key] = \core_analytics\manager::get_indicator($indicator); + } + + $this->model = testable_model::create($target, $indicators); + $this->modelobj = $this->model->get_model_obj(); + } + + public function test_enable() { + $this->resetAfterTest(true); + + $this->assertEquals(0, $this->model->get_model_obj()->enabled); + $this->assertEquals(0, $this->model->get_model_obj()->trained); + $this->assertEquals('', $this->model->get_model_obj()->timesplitting); + + $this->model->enable('\\core_analytics\\local\\time_splitting\\quarters'); + $this->assertEquals(1, $this->model->get_model_obj()->enabled); + $this->assertEquals(0, $this->model->get_model_obj()->trained); + $this->assertEquals('\\core_analytics\\local\\time_splitting\\quarters', $this->model->get_model_obj()->timesplitting); + } + + public function test_create() { + $this->resetAfterTest(true); + + $target = \core_analytics\manager::get_target('\tool_models\local\target\course_dropout'); + $indicators = array( + \core_analytics\manager::get_indicator('\core_analytics\local\indicator\any_write_action'), + \core_analytics\manager::get_indicator('\core_analytics\local\indicator\read_actions') + ); + $model = \core_analytics\model::create($target, $indicators); + $this->assertInstanceOf('\core_analytics\model', $model); + } + + public function test_model_manager() { + $this->resetAfterTest(true); + + $this->assertCount(3, $this->model->get_indicators()); + $this->assertInstanceOf('\core_analytics\local\target\binary', $this->model->get_target()); + + // Using evaluation as the model is not yet enabled. + $this->model->init_analyser(array('evaluation' => true)); + $this->assertInstanceOf('\core_analytics\local\analyser\base', $this->model->get_analyser()); + + $this->model->enable('\core_analytics\local\time_splitting\quarters'); + $this->assertInstanceOf('\core_analytics\local\analyser\courses', $this->model->get_analyser()); + } + + public function test_output_dir() { + $this->resetAfterTest(true); + + $dir = make_request_directory(); + set_config('modeloutputdir', $dir, 'analytics'); + + $modeldir = $dir . DIRECTORY_SEPARATOR . $this->modelobj->id . DIRECTORY_SEPARATOR . $this->modelobj->version; + $this->assertEquals($modeldir, $this->model->get_output_dir()); + $this->assertEquals($modeldir . DIRECTORY_SEPARATOR . 'asd', $this->model->get_output_dir(array('asd'))); + } + + public function test_unique_id() { + global $DB; + + $this->resetAfterTest(true); + + $originaluniqueid = $this->model->get_unique_id(); + + // Same id across instances. + $this->model = new testable_model($this->modelobj); + $this->assertEquals($originaluniqueid, $this->model->get_unique_id()); + + // We will restore it later. + $originalversion = $this->modelobj->version; + + // Generates a different id if timemodified changes. + $this->modelobj->version = $this->modelobj->version + 10; + $DB->update_record('analytics_models', $this->modelobj); + $this->model = new testable_model($this->modelobj); + $this->assertNotEquals($originaluniqueid, $this->model->get_unique_id()); + + // Restore original timemodified to continue testing. + $this->modelobj->version = $originalversion; + $DB->update_record('analytics_models', $this->modelobj); + // Same when updating through an action that changes the model. + $this->model = new testable_model($this->modelobj); + + $this->model->mark_as_trained(); + $this->assertEquals($originaluniqueid, $this->model->get_unique_id()); + + $this->model->enable(); + $this->assertEquals($originaluniqueid, $this->model->get_unique_id()); + + // Wait 1 sec so the timestamp changes. + sleep(1); + $this->model->enable('\core_analytics\local\time_splitting\quarters'); + $this->assertNotEquals($originaluniqueid, $this->model->get_unique_id()); + + } +} + +class testable_model extends \core_analytics\model { + public function get_output_dir($subdirs = array()) { + return parent::get_output_dir($subdirs); + } + + public function init_analyser($options = array()) { + return parent::init_analyser($options); + } +} diff --git a/analytics/tests/prediction_test.php b/analytics/tests/prediction_test.php new file mode 100644 index 00000000000..426989e4a84 --- /dev/null +++ b/analytics/tests/prediction_test.php @@ -0,0 +1,287 @@ +. + +/** + * Unit tests for evaluation, training and prediction. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once(__DIR__ . '/fixtures/test_indicator_max.php'); +require_once(__DIR__ . '/fixtures/test_indicator_min.php'); +require_once(__DIR__ . '/fixtures/test_indicator_fullname.php'); +require_once(__DIR__ . '/fixtures/test_indicator_random.php'); +require_once(__DIR__ . '/fixtures/test_target_shortname.php'); + +/** + * Unit tests for evaluation, training and prediction. + * + * @package core_analytics + * @copyright 2017 David Monllaó {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_analytics_prediction_testcase extends advanced_testcase { + + /** + * @dataProvider provider_training_and_prediction + * @param string $timesplittingid + * @param int $npredictedranges + * @return void + */ + public function test_training_and_prediction($timesplittingid, $npredictedranges, $predictionsprocessorclass) { + global $DB; + + $ncourses = 10; + + $this->resetAfterTest(true); + + // Generate training data. + $params = array( + 'startdate' => mktime(0, 0, 0, 10, 24, 2015), + 'enddate' => mktime(0, 0, 0, 2, 24, 2016), + ); + for ($i = 0; $i < $ncourses; $i++) { + $name = 'a' . random_string(10); + $courseparams = array('shortname' => $name, 'fullname' => $name) + $params; + $this->getDataGenerator()->create_course($courseparams); + } + for ($i = 0; $i < $ncourses; $i++) { + $name = 'b' . random_string(10); + $courseparams = array('shortname' => $name, 'fullname' => $name) + $params; + $this->getDataGenerator()->create_course($courseparams); + } + + // We repeat the test for all prediction processors. + $predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false); + if ($predictionsprocessor->is_ready() !== true) { + $this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.'); + } + + set_config('predictionsprocessor', $predictionsprocessorclass, 'analytics'); + + $model = $this->add_perfect_model(); + $model->enable($timesplittingid); + + // No samples trained yet. + $this->assertEquals(0, $DB->count_records('analytics_train_samples', array('modelid' => $model->get_id()))); + + $results = $model->train(); + $this->assertEquals(1, $model->get_model_obj()->enabled); + $this->assertEquals(1, $model->get_model_obj()->trained); + + // 1 training file was created. + $trainedsamples = $DB->get_records('analytics_train_samples', array('modelid' => $model->get_id())); + $this->assertEquals(1, count($trainedsamples)); + $samples = json_decode(reset($trainedsamples)->sampleids, true); + $this->assertEquals($ncourses * 2, count($samples)); + $this->assertEquals(1, $DB->count_records('analytics_used_files', + array('modelid' => $model->get_id(), 'action' => 'trained'))); + + // Now we create 2 hidden courses (they should not be used for training by the target). + $courseparams = $params + array('shortname' => 'aaaaaa', 'fullname' => 'aaaaaa', 'visible' => 0); + $course1 = $this->getDataGenerator()->create_course($courseparams); + $courseparams = $params + array('shortname' => 'bbbbbb', 'fullname' => 'bbbbbb', 'visible' => 0); + $course2 = $this->getDataGenerator()->create_course($courseparams); + + // No more files should be created as the 2 new courses should be skipped by the target (not ready for training). + $results = $model->train(); + $trainedsamples = $DB->get_records('analytics_train_samples', array('modelid' => $model->get_id())); + $this->assertEquals(1, count($trainedsamples)); + $this->assertEquals(1, $DB->count_records('analytics_used_files', + array('modelid' => $model->get_id(), 'action' => 'trained'))); + + // They will not be skipped for prediction though. + $result = $model->predict(); + + // $course1 predictions should be 1 == 'a', $course2 predictions should be 0 == 'b'. + $correct = array($course1->id => 1, $course2->id => 0); + foreach ($result->predictions as $sampleprediction) { + list($uniquesampleid, $prediction) = $sampleprediction; + list($uniquesampleid, $rangeindex) = $model->get_time_splitting()->infer_sample_info($uniquesampleid); + + // The range index is not important here, both ranges prediction will be the same. + $this->assertEquals($correct[$uniquesampleid], $prediction); + } + + // 2 ranges will be predicted. + $trainedsamples = $DB->get_records('analytics_predict_ranges', array('modelid' => $model->get_id())); + $this->assertEquals($npredictedranges, count($trainedsamples)); + $this->assertEquals(1, $DB->count_records('analytics_used_files', + array('modelid' => $model->get_id(), 'action' => 'predicted'))); + // 2 predictions for each range. + $this->assertEquals(2 * $npredictedranges, $DB->count_records('analytics_predictions', array('modelid' => $model->get_id()))); + + // No new generated files nor records as there are no new courses available. + $model->predict(); + $trainedsamples = $DB->get_records('analytics_predict_ranges', array('modelid' => $model->get_id())); + $this->assertEquals($npredictedranges, count($trainedsamples)); + $this->assertEquals(1, $DB->count_records('analytics_used_files', + array('modelid' => $model->get_id(), 'action' => 'predicted'))); + $this->assertEquals(2 * $npredictedranges, $DB->count_records('analytics_predictions', array('modelid' => $model->get_id()))); + } + + public function provider_training_and_prediction() { + $cases = array( + 'no_splitting' => array('\core_analytics\local\time_splitting\no_splitting', 1), + 'quarters' => array('\core_analytics\local\time_splitting\quarters', 4) + ); + + // We need to test all system prediction processors. + return $this->add_prediction_processors($cases); + } + + + /** + * Basic test to check that prediction processors work as expected. + * + * @dataProvider provider_test_evaluation + */ + public function test_evaluation($modelquality, $ncourses, $expected, $predictionsprocessorclass) { + $this->resetAfterTest(true); + + $sometimesplittings = '\core_analytics\local\time_splitting\weekly,' . + '\core_analytics\local\time_splitting\single_range,' . + '\core_analytics\local\time_splitting\quarters'; + set_config('timesplittings', $sometimesplittings, 'analytics'); + + if ($modelquality === 'perfect') { + $model = $this->add_perfect_model(); + } else if ($modelquality === 'random') { + $model = $this->add_random_model(); + } else { + throw new \coding_exception('Only perfect and random accepted as $modelquality values'); + } + + + // Generate training data. + $params = array( + 'startdate' => mktime(0, 0, 0, 10, 24, 2015), + 'enddate' => mktime(0, 0, 0, 2, 24, 2016), + ); + for ($i = 0; $i < $ncourses; $i++) { + $name = 'a' . random_string(10); + $params = array('shortname' => $name, 'fullname' => $name) + $params; + $this->getDataGenerator()->create_course($params); + } + for ($i = 0; $i < $ncourses; $i++) { + $name = 'b' . random_string(10); + $params = array('shortname' => $name, 'fullname' => $name) + $params; + $this->getDataGenerator()->create_course($params); + } + + // We repeat the test for all prediction processors. + $predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false); + if ($predictionsprocessor->is_ready() !== true) { + $this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.'); + } + + set_config('predictionsprocessor', $predictionsprocessorclass, 'analytics'); + + $results = $model->evaluate(); + + // We check that the returned status includes at least $expectedcode code. + foreach ($results as $timesplitting => $result) { + $message = 'The returned status code ' . $result->status . ' should include ' . $expected[$timesplitting]; + $this->assertEquals($expected[$timesplitting], $result->status & $expected[$timesplitting], $message); + } + } + + public function provider_test_evaluation() { + + $cases = array( + 'bad-and-no-enough-data' => array( + 'modelquality' => 'random', + 'ncourses' => 5, + 'expectedresults' => array( + // The course duration is too much to be processed by in weekly basis. + '\core_analytics\local\time_splitting\weekly' => \core_analytics\model::NO_DATASET, + // 10 samples is not enough to process anything. + '\core_analytics\local\time_splitting\single_range' => \core_analytics\model::EVALUATE_NOT_ENOUGH_DATA + \core_analytics\model::EVALUATE_LOW_SCORE, + '\core_analytics\local\time_splitting\quarters' => \core_analytics\model::EVALUATE_NOT_ENOUGH_DATA + \core_analytics\model::EVALUATE_LOW_SCORE, + ) + ), + 'bad' => array( + 'modelquality' => 'random', + 'ncourses' => 50, + 'expectedresults' => array( + // The course duration is too much to be processed by in weekly basis. + '\core_analytics\local\time_splitting\weekly' => \core_analytics\model::NO_DATASET, + '\core_analytics\local\time_splitting\single_range' => \core_analytics\model::EVALUATE_LOW_SCORE, + '\core_analytics\local\time_splitting\quarters' => \core_analytics\model::EVALUATE_LOW_SCORE, + ) + ), + 'good' => array( + 'modelquality' => 'perfect', + 'ncourses' => 50, + 'expectedresults' => array( + // The course duration is too much to be processed by in weekly basis. + '\core_analytics\local\time_splitting\weekly' => \core_analytics\model::NO_DATASET, + '\core_analytics\local\time_splitting\single_range' => \core_analytics\model::OK, + '\core_analytics\local\time_splitting\quarters' => \core_analytics\model::OK, + ) + ) + ); + return $this->add_prediction_processors($cases); + } + + protected function add_random_model() { + + $target = \core_analytics\manager::get_target('test_target_shortname'); + $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_random'); + foreach ($indicators as $key => $indicator) { + $indicators[$key] = \core_analytics\manager::get_indicator($indicator); + } + + $model = \core_analytics\model::create($target, $indicators); + + // To load db defaults as well. + return new \core_analytics\model($model->get_id()); + } + + protected function add_perfect_model() { + + $target = \core_analytics\manager::get_target('test_target_shortname'); + $indicators = array('test_indicator_max', 'test_indicator_min', 'test_indicator_fullname'); + foreach ($indicators as $key => $indicator) { + $indicators[$key] = \core_analytics\manager::get_indicator($indicator); + } + + $model = \core_analytics\model::create($target, $indicators); + + // To load db defaults as well. + return new \core_analytics\model($model->get_id()); + } + + protected function add_prediction_processors($cases) { + + $return = array(); + + // We need to test all system prediction processors. + $predictionprocessors = \core_analytics\manager::get_all_prediction_processors(); + foreach ($predictionprocessors as $classfullname => $unused) { + foreach ($cases as $key => $case) { + $newkey = $key . '-' . $classfullname; + $return[$newkey] = $case + array('predictionsprocessorclass' => $classfullname); + } + } + + return $return; + } +} diff --git a/lib/tests/component_test.php b/lib/tests/component_test.php index 5ac54351d4c..30852a69b2a 100644 --- a/lib/tests/component_test.php +++ b/lib/tests/component_test.php @@ -34,7 +34,7 @@ class core_component_testcase extends advanced_testcase { // To be changed if number of subsystems increases/decreases, // this is defined here to annoy devs that try to add more without any thinking, // always verify that it does not collide with any existing add-on modules and subplugins!!! - const SUBSYSTEMCOUNT = 66; + const SUBSYSTEMCOUNT = 67; public function setUp() { $psr0namespaces = new ReflectionProperty('core_component', 'psr0namespaces');