L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MDL-59010 analytics: Direct db calls to logging API

Browse Source 
Part of MDL-57791 epic.
This commit is contained in:
David Monllao
2017-07-24 08:36:46 +02:00
parent 58f86c4cc5
commit f67f35f324
12 changed files with 247 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/settings/analytics.php
+19
View File
@@ -41,6 +41,25 @@ if ($hassiteconfig) {
'\mlbackend_php\processor', $predictors)
);
// Log store.
$logmanager = get_log_manager();
$readers = $logmanager->get_readers('core\log\sql_reader');
$options = array();
$defaultreader = false;
foreach ($readers as $plugin => $reader) {
if (!$reader->is_logging()) {
continue;
}
if (!isset($defaultreader)) {
// The top one as default reader.
$defaultreader = $plugin;
}
$options[$plugin] = $reader->get_name();
}
$settings->add(new admin_setting_configselect('analytics/logstore',
new lang_string('analyticslogstore', 'analytics'), new lang_string('analyticslogstore_help', 'analytics'),
$defaultreader, $options));
// Enable/disable time splitting methods.
$alltimesplittings = \core_analytics\manager::get_all_time_splittings();
admin/tool/models/classes/analytics/target/course_dropout.php
+9 -5
View File
@@ -117,15 +117,17 @@ class course_dropout extends \core_analytics\local\target\binary {
}
if ($fortraining) {
// Not a valid target for training if there are not enough course accesses.
$params = array('courseid' => $course->get_id(), 'anonymous' => 0, 'start' => $course->get_start(),
'end' => $course->get_end());
list($studentssql, $studentparams) = $DB->get_in_or_equal($students, SQL_PARAMS_NAMED);
// Using anonymous to use the db index, not filtering by timecreated to speed it up.
$select = 'courseid = :courseid AND anonymous = :anonymous AND timecreated > :start AND timecreated < :end ' .
'AND userid ' . $studentssql;
// Using anonymous to use the db index, not filtering by timecreated to speed it up.
$nlogs = $DB->count_records_select('logstore_standard_log', $select, array_merge($params, $studentparams));
$logstore = \core_analytics\manager::get_analytics_logstore();
$nlogs = $logstore->get_events_select_count($select, array_merge($params, $studentparams));
// At least a minimum of students activity.
$nstudents = count($students);
@@ -179,9 +181,11 @@ class course_dropout extends \core_analytics\local\target\binary {
// No logs during the last quarter of the course.
$courseduration = $course->get_end() - $course->get_start();
$limit = intval($course->get_end() - ($courseduration / 4));
$select = "courseid = :courseid AND userid = :userid AND timecreated > :limit";
$params = array('userid' => $userenrol->userid, 'courseid' => $course->get_id(), 'limit' => $limit);
$sql = "SELECT id FROM {logstore_standard_log} WHERE courseid = :courseid AND userid = :userid AND timecreated > :limit";
if ($DB->record_exists_sql($sql, $params)) {
$logstore = \core_analytics\manager::get_analytics_logstore();
$nlogs = $logstore->get_events_select_count($select, $params);
if ($nlogs == 0) {
return 0;
}
return 1;
analytics/classes/course.php
+23 -17
View File
@@ -27,6 +27,7 @@ defined('MOODLE_INTERNAL') || die();
require_once($CFG->dirroot . '/course/lib.php');
require_once($CFG->dirroot . '/lib/gradelib.php');
require_once($CFG->dirroot . '/lib/enrollib.php');
/**
*
@@ -162,34 +163,38 @@ class course implements \core_analytics\analysable {
return 0;
}
$logstore = \core_analytics\manager::get_analytics_logstore();
// We first try to find current course student logs.
list($filterselect, $filterparams) = $this->course_students_query_filter();
$sql = "SELECT MIN(timecreated) FROM {logstore_standard_log}
WHERE $filterselect
GROUP BY userid";
$firstlogs = $DB->get_fieldset_sql($sql, $filterparams);
$firstlogs = array();
foreach ($this->studentids as $studentid) {
// Grrr, we are limited by logging API, we could do this easily with a
// select min(timecreated) from xx where courseid = yy group by userid.
// Filters based on the premise that more than 90% of people will be using
// standard logstore, which contains a userid, contextlevel, contextinstanceid index.
$select = "userid = :userid AND contextlevel = :contextlevel AND contextinstanceid = :contextinstanceid";
$params = array('userid' => $studentid, 'contextlevel' => CONTEXT_COURSE, 'contextinstanceid' => $this->get_id());
$events = $logstore->get_events_select($select, $params, 'timecreated ASC', 0, 1);
$event = reset($events);
$firstlogs = $event->timecreated;
}
if (empty($firstlogs)) {
// Can't guess if no student accesses.
return 0;
}
sort($firstlogs);
$firstlogsmedian = $this->median($firstlogs);
// Not using enrol API because we may be dealing with databases that used
// 3rd party enrolment plugins that are not available in the database.
// TODO We will need to switch to enrol API once we have enough data.
list($studentssql, $studentparams) = $DB->get_in_or_equal($this->studentids, SQL_PARAMS_NAMED);
$sql = "SELECT ue.* FROM {user_enrolments} ue
JOIN {enrol} e ON e.id = ue.enrolid
WHERE e.courseid = :courseid AND ue.userid $studentssql";
$studentenrolments = $DB->get_records_sql($sql, array('courseid' => $this->course->id) + $studentparams);
$studentenrolments = enrol_get_course_users($this->get_id(), $this->studentids);
if (empty($studentenrolments)) {
return 0;
}
$enrolstart = array();
foreach ($studentenrolments as $studentenrolment) {
// I don't like CASE WHEN :P
$enrolstart[] = ($studentenrolment->timestart) ? $studentenrolment->timestart : $studentenrolment->timecreated;
$enrolstart[] = ($studentenrolment->uetimestart) ? $studentenrolment->uetimestart : $studentenrolment->uetimecreated;
}
sort($enrolstart);
$enrolstartmedian = $this->median($enrolstart);
@@ -358,7 +363,8 @@ class course implements \core_analytics\analysable {
if ($this->ntotallogs === null) {
list($filterselect, $filterparams) = $this->course_students_query_filter();
$this->ntotallogs = $DB->count_records_select('logstore_standard_log', $filterselect, $filterparams);
$logstore = \core_analytics\manager::get_analytics_logstore();
$this->ntotallogs = $logstore->get_events_select_count($filterselect, $filterparams);
}
return $this->ntotallogs;
@@ -582,7 +588,7 @@ class course implements \core_analytics\analysable {
}
/**
* Returns the query and params used to filter {logstore_standard_log} table by this course students.
* Returns the query and params used to filter the logstore by this course students.
*
* @return array
*/
analytics/classes/local/analyser/student_enrolments.php
+44 -9
View File
@@ -71,15 +71,33 @@ class student_enrolments extends by_course {
$studentids = $course->get_students();
$samplesdata = array();
foreach ($enrolments as $user) {
foreach ($enrolments as $userenrolmentid => $user) {
if (empty($studentids[$user->id])) {
// Not a student.
continue;
}
$sampleid = $user->enrolmentid;
unset($user->enrolmentid);
$sampleid = $userenrolmentid;
$samplesdata[$sampleid]['user_enrolments'] = (object)array(
'id' => $user->ueid,
'status' => $user->uestatus,
'enrolid' => $user->ueenrolid,
'userid' => $user->id,
'timestart' => $user->uetimestart,
'timeend' => $user->uetimeend,
'modifierid' => $user->uemodifierid,
'timecreated' => $user->uetimecreated,
'timemodified' => $user->uetimemodified
);
unset($user->ueid);
unset($user->uestatus);
unset($user->ueenrolid);
unset($user->uetimestart);
unset($user->uetimeend);
unset($user->uemodifierid);
unset($user->uetimecreated);
unset($user->uetimemodified);
$samplesdata[$sampleid]['course'] = $course->get_course_data();
$samplesdata[$sampleid]['context'] = $course->get_context();
@@ -96,21 +114,39 @@ class student_enrolments extends by_course {
public function get_samples($sampleids) {
global $DB;
$enrolments = enrol_get_course_users(false, false, array(), $sampleids);
// Some course enrolments.
list($enrolsql, $params) = $DB->get_in_or_equal($sampleids, SQL_PARAMS_NAMED);
// Although we load all the course users data in memory anyway, using recordsets we will
// not use the double of the memory required by the end of the iteration.
$sql = "SELECT ue.id AS enrolmentid, u.* FROM {user_enrolments} ue
JOIN {user} u on ue.userid = u.id
WHERE ue.id $enrolsql";
$enrolments = $DB->get_recordset_sql($sql, $params);
$samplesdata = array();
foreach ($enrolments as $user) {
foreach ($enrolments as $userenrolmentid => $user) {
$sampleid = $user->enrolmentid;
unset($user->enrolmentid);
$sampleid = $userenrolmentid;
$samplesdata[$sampleid]['user_enrolments'] = (object)array(
'id' => $user->ueid,
'status' => $user->uestatus,
'enrolid' => $user->ueenrolid,
'userid' => $user->id,
'timestart' => $user->uetimestart,
'timeend' => $user->uetimeend,
'modifierid' => $user->uemodifierid,
'timecreated' => $user->uetimecreated,
'timemodified' => $user->uetimemodified
);
unset($user->ueid);
unset($user->uestatus);
unset($user->ueenrolid);
unset($user->uetimestart);
unset($user->uetimeend);
unset($user->uemodifierid);
unset($user->uetimecreated);
unset($user->uetimemodified);
// Enrolment samples are grouped by the course they belong to, so all $sampleids belong to the same
// course, $courseid and $coursemodinfo will only query the DB once and cache the course data in memory.
@@ -125,7 +161,6 @@ class student_enrolments extends by_course {
// Fill the cache.
$this->samplecourses[$sampleid] = $coursemodinfo->get_course()->id;
}
$enrolments->close();
$enrolids = array_keys($samplesdata);
return array(array_combine($enrolids, $enrolids), $samplesdata);
analytics/classes/local/indicator/any_access_after_end.php
+7 -1
View File
@@ -55,7 +55,13 @@ class any_access_after_end extends binary {
"timecreated > :end";
$params = array('userid' => $user->id, 'contextlevel' => $context->contextlevel,
'contextinstanceid' => $context->instanceid, 'end' => $course->get_end());
return $DB->record_exists_select('logstore_standard_log', $select, $params) ? self::get_max_value() : self::get_min_value();
$logstore = \core_analytics\manager::get_analytics_logstore();
$nlogs = $logstore->get_events_select_count($select, $params);
if ($nlogs) {
return self::get_max_value();
} else {
return self::get_min_value();
}
}
}
analytics/classes/local/indicator/any_access_before_start.php
+8 -1
View File
@@ -55,6 +55,13 @@ class any_access_before_start extends binary {
"timecreated < :start";
$params = array('userid' => $user->id, 'contextlevel' => $context->contextlevel,
'contextinstanceid' => $context->instanceid, 'start' => $course->get_start());
return $DB->record_exists_select('logstore_standard_log', $select, $params) ? self::get_max_value() : self::get_min_value();
$logstore = \core_analytics\manager::get_analytics_logstore();
$nlogs = $logstore->get_events_select_count($select, $params);
if ($nlogs) {
return self::get_max_value();
} else {
return self::get_min_value();
}
}
}
analytics/classes/local/indicator/any_write_action.php
+7 -1
View File
@@ -61,6 +61,12 @@ class any_write_action extends binary {
"(crud = 'c' OR crud = 'u') AND timecreated > :starttime AND timecreated <= :endtime";
$params = $params + array('contextlevel' => $context->contextlevel,
'contextinstanceid' => $context->instanceid, 'starttime' => $starttime, 'endtime' => $endtime);
return $DB->record_exists_select('logstore_standard_log', $select, $params) ? self::get_max_value() : self::get_min_value();
$logstore = \core_analytics\manager::get_analytics_logstore();
$nlogs = $logstore->get_events_select_count($select, $params);
if ($nlogs) {
return self::get_max_value();
} else {
return self::get_min_value();
}
}
}
analytics/classes/local/indicator/community_of_inquiry_activity.php
+40 -37
View File
@@ -15,7 +15,7 @@
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Community of inquire abstract indicator.
* Community of inquiry abstract indicator.
*
* @package core_analytics
* @copyright 2017 David Monllao {@link http://www.davidmonllao.com}
@@ -116,8 +116,8 @@ abstract class community_of_inquiry_activity extends linear {
}
$it = array($user->id => $this->activitylogs[$contextid][$user->id]);
}
foreach ($it as $logs) {
foreach ($logs as $log) {
foreach ($it as $events) {
foreach ($events as $log) {
if ($log->crud === 'c' || $log->crud === 'u') {
return true;
}
@@ -145,7 +145,7 @@ abstract class community_of_inquiry_activity extends linear {
$it = array($user->id => $this->activitylogs[$contextid][$user->id]);
}
foreach ($this->activitylogs[$contextid] as $userid => $logs) {
foreach ($this->activitylogs[$contextid] as $userid => $events) {
$methodname = 'feedback_' . $action;
if ($this->{$methodname}($cm, $contextid, $userid)) {
return true;
@@ -179,18 +179,18 @@ abstract class community_of_inquiry_activity extends linear {
}
protected function feedback_viewed_events() {
throw new \coding_exception('Activities with a potential cognitive level that include viewing feedback should define ' .
'"feedback_viewed_events" method or should override feedback_viewed method.');
throw new \coding_exception('Activities with a potential cognitive or social level that include viewing feedback ' .
'should define "feedback_viewed_events" method or should override feedback_viewed method.');
}
protected function feedback_replied_events() {
throw new \coding_exception('Activities with a potential cognitive level that include replying to feedback should define ' .
'"feedback_replied_events" method or should override feedback_replied method.');
throw new \coding_exception('Activities with a potential cognitive or social level that include replying to feedback ' .
'should define "feedback_replied_events" method or should override feedback_replied method.');
}
protected function feedback_submitted_events() {
throw new \coding_exception('Activities with a potential cognitive level that include viewing feedback should define ' .
'"feedback_submitted_events" method or should override feedback_submitted method.');
throw new \coding_exception('Activities with a potential cognitive or social level that include viewing feedback ' .
'should define "feedback_submitted_events" method or should override feedback_submitted method.');
}
protected function feedback_post_action(\cm_info $cm, $contextid, $userid, $eventnames, $after = null) {
@@ -307,44 +307,47 @@ abstract class community_of_inquiry_activity extends linear {
// Filter by context to use the db table index.
list($contextsql, $contextparams) = $DB->get_in_or_equal(array_keys($activities), SQL_PARAMS_NAMED);
// Keeping memory usage as low as possible by using recordsets and storing only 1 log
// per contextid-userid-eventname + 1 timestamp for each of this combination records.
$fields = 'eventname, crud, contextid, contextlevel, contextinstanceid, userid, courseid';
$select = "contextid $contextsql AND timecreated > :starttime AND timecreated <= :endtime";
$sql = "SELECT $fields, timecreated " .
"FROM {logstore_standard_log} " .
"WHERE $select " .
"ORDER BY timecreated ASC";
$params = $contextparams + array('starttime' => $starttime, 'endtime' => $endtime);
$logs = $DB->get_recordset_sql($sql, $params);
// Pity that we need to pass through logging readers API when most of the people just uses the standard one.
$logstore = \core_analytics\manager::get_analytics_logstore();
$events = $logstore->get_events_select_iterator($select, $params, 'timecreated ASC', 0, 0);
// Returs the logs organised by contextid, userid and eventname so it is easier to calculate activities data later.
// At the same time we want to keep this array reasonably "not-massive".
$processedlogs = array();
foreach ($logs as $log) {
if (!isset($processedlogs[$log->contextid])) {
$processedlogs[$log->contextid] = array();
$processedevents = array();
foreach ($events as $event) {
if (!isset($processedevents[$event->contextid])) {
$processedevents[$event->contextid] = array();
}
if (!isset($processedlogs[$log->contextid][$log->userid])) {
$processedlogs[$log->contextid][$log->userid] = array();
if (!isset($processedevents[$event->contextid][$event->userid])) {
$processedevents[$event->contextid][$event->userid] = array();
}
// contextid and userid have already been used to index the logs, the next field to index by is eventname:
// contextid and userid have already been used to index the events, the next field to index by is eventname:
// crud is unique per eventname, courseid is the same for all records and we append timecreated.
if (!isset($processedlogs[$log->contextid][$log->userid][$log->eventname])) {
$processedlogs[$log->contextid][$log->userid][$log->eventname] = $log;
if (!isset($processedevents[$event->contextid][$event->userid][$event->eventname])) {
// Remove all data that can change between events of the same type.
$data = (object)$event->get_data();
unset($data->id);
unset($data->anonymous);
unset($data->relateduserid);
unset($data->other);
unset($data->origin);
unset($data->ip);
$processedevents[$event->contextid][$event->userid][$event->eventname] = $data;
// We want timecreated attribute to be an array containing all user access times.
$processedlogs[$log->contextid][$log->userid][$log->eventname]->timecreated = array(intval($log->timecreated));
} else {
// Add the event timecreated.
$processedlogs[$log->contextid][$log->userid][$log->eventname]->timecreated[] = intval($log->timecreated);
$processedevents[$event->contextid][$event->userid][$event->eventname]->timecreated = array();
}
}
$logs->close();
return $processedlogs;
// Add the event timecreated.
$processedevents[$event->contextid][$event->userid][$event->eventname]->timecreated[] = intval($event->timecreated);
}
$events->close();
return $processedevents;
}
/**
@@ -385,7 +388,7 @@ abstract class community_of_inquiry_activity extends linear {
$potentiallevel = $this->get_cognitive_depth_level($cm);
if (!is_int($potentiallevel) || $potentiallevel > 5 || $potentiallevel < 1) {
throw new \coding_exception('Activities\' potential level of engagement possible values go from 1 to 5.');
throw new \coding_exception('Activities\' potential cognitive depth go from 1 to 5.');
}
$scoreperlevel = $scoreperactivity / $potentiallevel;
@@ -472,7 +475,7 @@ abstract class community_of_inquiry_activity extends linear {
$potentiallevel = $this->get_social_breadth_level($cm);
if (!is_int($potentiallevel) || $potentiallevel > 2 || $potentiallevel < 1) {
throw new \coding_exception('Activities\' potential level of engagement possible values go from 1 to 2.');
throw new \coding_exception('Activities\' potential social breadth go from 1 to 2.');
}
$scoreperlevel = $scoreperactivity / $potentiallevel;
// TODO Add support for other levels than 2.
analytics/classes/local/indicator/read_actions.php
+2 -2
View File
@@ -45,7 +45,6 @@ class read_actions extends linear {
}
protected function calculate_sample($sampleid, $sampleorigin, $starttime = false, $endtime = false) {
global $DB;
$select = '';
$params = array();
@@ -61,7 +60,8 @@ class read_actions extends linear {
"crud = 'r' AND timecreated > :starttime AND timecreated <= :endtime";
$params = $params + array('contextlevel' => $context->contextlevel,
'contextinstanceid' => $context->instanceid, 'starttime' => $starttime, 'endtime' => $endtime);
$nrecords = $DB->count_records_select('logstore_standard_log', $select, $params);
$logstore = \core_analytics\manager::get_analytics_logstore();
$nrecords = $logstore->get_events_select_count($select, $params);
// We define a list of ranges to fit $nrecords into it
// # Done absolutely nothing
analytics/classes/manager.php
+26
View File
@@ -248,6 +248,32 @@ class manager {
return false;
}
/**
* get_analytics_logstore
*
* @return \core\log\reader
*/
public static function get_analytics_logstore() {
$readers = get_log_manager()->get_readers('core\log\sql_reader');
$analyticsstore = get_config('analytics', 'logstore');
if (empty($analyticsstore)) {
$logstore = reset($readers);
} else if (!empty($readers[$analyticsstore])) {
$logstore = $readers[$analyticsstore];
} else {
$logstore = reset($readers);
debugging('The selected log store for analytics is not available anymore. Using "' .
$logstore->get_name() . '"', DEBUG_DEVELOPER);
}
if (!$logstore->is_logging()) {
debugging('The selected log store for analytics "' . $logstore->get_name() .
'" is not logging activity logs', DEBUG_DEVELOPER);
}
return $logstore;
}
/**
* Returns the provided element classes in the site.
*
analytics/classes/site.php
+60 -10
View File
@@ -33,29 +33,79 @@ defined('MOODLE_INTERNAL') || die();
*/
class site implements \core_analytics\analysable {
/**
* @var int
*/
protected $start;
/**
* @var int
*/
protected $end;
/**
* Analysable id
*
* @return int
*/
public function get_id() {
return SYSCONTEXTID;
}
/**
* Analysable context.
*
* @return \context
*/
public function get_context() {
return \context_system::instance();
}
/**
* Analysable start timestamp.
*
* @return int
*/
public function get_start() {
global $DB;
$start = $DB->get_record_sql("SELECT MIN(timecreated) AS time FROM {logstore_standard_log}");
if (!empty($start) && !empty($start->time)) {
return $start->time;
if (!empty($this->start)) {
return $this->start;
}
return 0;
$logstore = \core_analytics\manager::get_analytics_logstore();
// Basically a SELECT MIN(timecreated) FROM ...
$events = $logstore->get_events_select("", array(), "timecreated ASC", 0, 1);
if ($events) {
// There should be just 1 event.
$event = reset($events);
$this->start = $event->timecreated;
} else {
$this->start = 0;
}
return $this->start;
}
/**
* Analysable end timestamp.
*
* @return int
*/
public function get_end() {
global $DB;
$end = $DB->get_record_sql("SELECT MAX(timecreated) AS time FROM {logstore_standard_log}");
if (!empty($end) && !empty($end->time)) {
return $end->time;
if (!empty($this->end)) {
return $this->end;
}
return time();
$logstore = \core_analytics\manager::get_analytics_logstore();
// Basically a SELECT MAX(timecreated) FROM ...
$events = $logstore->get_events_select("", array(), "timecreated DESC", 0, 1);
if ($events) {
// There should be just 1 event.
$event = reset($events);
$this->end = $event->timecreated;
} else {
$this->end = time();
}
return $this->end;
}
}
lang/en/analytics.php
+2
View File
@@ -24,6 +24,8 @@
$string['analysablenotused'] = 'Analysable {$a->analysableid} not used: {$a->errors}';
$string['analysablenotvalidfortarget'] = 'Analysable {$a->analysableid} is not valid for this target: {$a->result}';
$string['analyticslogstore'] = 'Log store used for analytics';
$string['analyticslogstore_help'] = 'The log store that will be used by the analytics API to read users\' activity';
$string['analyticssettings'] = 'Analytics settings';
$string['enabledtimesplittings'] = 'Time splitting methods';
$string['enabledtimesplittings_help'] = 'The time splitting method divides the course duration in parts, the predictions engine will run at the end of these parts. It is recommended that you only enable the time splitting methods you could be interested on using; the evaluation process will iterate through all of them so the more time splitting methods to go through the slower the evaluation process will be.';