diff --git a/analytics/classes/local/analyser/base.php b/analytics/classes/local/analyser/base.php index 9bf10c8c7cc..b179dcb40a1 100644 --- a/analytics/classes/local/analyser/base.php +++ b/analytics/classes/local/analyser/base.php @@ -469,6 +469,9 @@ abstract class base { return $result; } + // Add target metadata. + $this->add_target_metadata($data); + // Write all calculated data to a file. $file = $dataset->store($data); @@ -636,4 +639,28 @@ abstract class base { $DB->insert_record('analytics_predict_samples', $predictionrange); } } + + /** + * Adds target metadata to the dataset. + * + * @param array $data + * @return void + */ + protected function add_target_metadata(&$data) { + $data[0][] = 'targetcolumn'; + $data[1][] = $this->analysabletarget->get_id(); + if ($this->analysabletarget->is_linear()) { + $data[0][] = 'targettype'; + $data[1][] = 'linear'; + $data[0][] = 'targetmin'; + $data[1][] = $this->analysabletarget::get_min_value(); + $data[0][] = 'targetmax'; + $data[1][] = $this->analysabletarget::get_max_value(); + } else { + $data[0][] = 'targettype'; + $data[1][] = 'discrete'; + $data[0][] = 'targetclasses'; + $data[1][] = json_encode($this->analysabletarget::get_classes()); + } + } } diff --git a/analytics/classes/local/indicator/binary.php b/analytics/classes/local/indicator/binary.php index 7f4c3460cc3..a7309961d69 100644 --- a/analytics/classes/local/indicator/binary.php +++ b/analytics/classes/local/indicator/binary.php @@ -46,6 +46,23 @@ abstract class binary extends discrete { return array(0); } + /** + * It should always be displayed. + * + * Binary values have no subtypes by default, please overwrite if + * your indicator is adding extra features. + * + * @param float $value + * @param string $subtype + * @return bool + */ + public function should_be_displayed($value, $subtype) { + if ($subtype != false) { + return false; + } + return true; + } + /** * get_display_value * diff --git a/analytics/classes/local/indicator/discrete.php b/analytics/classes/local/indicator/discrete.php index 0b51e0d1c86..8a8aa6699d2 100644 --- a/analytics/classes/local/indicator/discrete.php +++ b/analytics/classes/local/indicator/discrete.php @@ -85,7 +85,7 @@ abstract class discrete extends base { */ public function get_display_value($value, $subtype = false) { - $displayvalue = array_search($subtype, static::get_classes()); + $displayvalue = array_search($subtype, static::get_classes(), false); debugging('Please overwrite \core_analytics\local\indicator\discrete::get_display_value to show something ' . 'different than the default "' . $displayvalue . '"', DEBUG_DEVELOPER); diff --git a/analytics/classes/local/indicator/linear.php b/analytics/classes/local/indicator/linear.php index 691aa8a7590..56902c002b9 100644 --- a/analytics/classes/local/indicator/linear.php +++ b/analytics/classes/local/indicator/linear.php @@ -63,7 +63,7 @@ abstract class linear extends base { } /** - * should_be_displayed + * Show only the main feature. * * @param float $value * @param string $subtype diff --git a/analytics/classes/local/target/base.php b/analytics/classes/local/target/base.php index 466255e1143..0b2af4d0199 100644 --- a/analytics/classes/local/target/base.php +++ b/analytics/classes/local/target/base.php @@ -231,7 +231,7 @@ abstract class base extends \core_analytics\calculable { */ protected function min_prediction_score() { // The default minimum discards predictions with a low score. - return \core_analytics\model::MIN_SCORE; + return \core_analytics\model::PREDICTION_MIN_SCORE; } /** diff --git a/analytics/classes/local/target/binary.php b/analytics/classes/local/target/binary.php index ea04410f3df..f8c3b04dc1d 100644 --- a/analytics/classes/local/target/binary.php +++ b/analytics/classes/local/target/binary.php @@ -78,7 +78,7 @@ abstract class binary extends discrete { throw new \moodle_exception('errorpredictionformat', 'analytics'); } - if (in_array($value, $this->ignored_predicted_classes())) { + if (in_array($value, $this->ignored_predicted_classes(), false)) { // Just in case, if it is ignored the prediction should not even be recorded but if it would, it is ignored now, // which should mean that is it nothing serious. return self::OUTCOME_VERY_POSITIVE; diff --git a/analytics/classes/local/target/discrete.php b/analytics/classes/local/target/discrete.php index 0799b4d45c9..cbd8fe09d76 100644 --- a/analytics/classes/local/target/discrete.php +++ b/analytics/classes/local/target/discrete.php @@ -42,17 +42,18 @@ abstract class discrete extends base { */ public function is_linear() { // Not supported yet. - throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.'); + throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.' . + ' You can write your own and overwrite this method though.'); } /** * Is the provided class one of this target valid classes? * - * @param string $class + * @param mixed $class * @return bool */ protected static function is_a_class($class) { - return (in_array($class, static::get_classes())); + return (in_array($class, static::get_classes(), false)); } /** @@ -99,7 +100,7 @@ abstract class discrete extends base { throw new \moodle_exception('errorpredictionformat', 'analytics'); } - if (in_array($value, $this->ignored_predicted_classes())) { + if (in_array($value, $this->ignored_predicted_classes(), false)) { // Just in case, if it is ignored the prediction should not even be recorded. return self::OUTCOME_OK; } @@ -138,15 +139,16 @@ abstract class discrete extends base { * Returns the predicted classes that will be ignored. * * Better be keen to add more than less classes here, the callback is always able to discard some classes. As an example - * a target with classes 'grade 0-3', 'grade 3-6', 'grade 6-8' and 'grade 8-10' is interested in flagging both 'grade 0-3' - * and 'grade 3-6'. On the other hand, a target like dropout risk with classes 'yes', 'no' may just be interested in 'yes'. + * a target with classes 'grade 0-3', 'grade 3-6', 'grade 6-8' and 'grade 8-10' is interested in flagging both 'grade 6-8' + * and 'grade 8-10' as ignored. On the other hand, a target like dropout risk with classes 'yes', 'no' may just be + * interested in 'yes'. * * @return array List of values that will be ignored (array keys are ignored). */ protected function ignored_predicted_classes() { // Coding exception as this will only be called if this target have non-linear values. - throw new \coding_exception('Overwrite ignored_predicted_classes() and return an array with the classes that triggers ' . - 'the callback'); + throw new \coding_exception('Overwrite ignored_predicted_classes() and return an array with the classes that should not ' . + 'trigger the callback'); } /** @@ -162,10 +164,8 @@ abstract class discrete extends base { return false; } - if (!$this->is_linear()) { - if (in_array($predictedvalue, $this->ignored_predicted_classes())) { - return false; - } + if (in_array($predictedvalue, $this->ignored_predicted_classes())) { + return false; } return true; diff --git a/analytics/classes/local/target/linear.php b/analytics/classes/local/target/linear.php index 8d8e258fe7d..d16ad96075f 100644 --- a/analytics/classes/local/target/linear.php +++ b/analytics/classes/local/target/linear.php @@ -42,7 +42,8 @@ abstract class linear extends base { */ public function is_linear() { // Not supported yet. - throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.'); + throw new \coding_exception('Sorry, this version\'s prediction processors only support targets with binary values.' . + ' You can write your own and overwrite this method though.'); } /** @@ -52,7 +53,7 @@ abstract class linear extends base { * @param string $ignoredsubtype * @return int */ - public function get_calculated_outcome($value, $ignoredsubtype = false) { + public function get_calculation_outcome($value, $ignoredsubtype = false) { // This is very generic, targets will probably be interested in overwriting this. $diff = static::get_max_value() - static::get_min_value(); @@ -67,7 +68,7 @@ abstract class linear extends base { * * @return float */ - protected static function get_max_value() { + public static function get_max_value() { // Coding exception as this will only be called if this target have linear values. throw new \coding_exception('Overwrite get_max_value() and return the target max value'); } @@ -77,11 +78,33 @@ abstract class linear extends base { * * @return float */ - protected static function get_min_value() { + public static function get_min_value() { // Coding exception as this will only be called if this target have linear values. throw new \coding_exception('Overwrite get_min_value() and return the target min value'); } + /** + * Should the model callback be triggered? + * + * @param mixed $predictedvalue + * @param float $predictionscore + * @return bool + */ + public function triggers_callback($predictedvalue, $predictionscore) { + + if (!parent::triggers_callback($predictedvalue, $predictionscore)) { + return false; + } + + // People may not want to set a boundary. + $boundary = $this->get_callback_boundary(); + if (!empty($boundary) && floatval($predictedvalue) < $boundary) { + return false; + } + + return true; + } + /** * Returns the minimum value that triggers the callback. * diff --git a/analytics/classes/local/time_splitting/base.php b/analytics/classes/local/time_splitting/base.php index 109084e8fdb..0212d29b7b9 100644 --- a/analytics/classes/local/time_splitting/base.php +++ b/analytics/classes/local/time_splitting/base.php @@ -371,12 +371,9 @@ abstract class base { $metadata = array( 'timesplitting' => $this->get_id(), // If no target the first column is the sampleid, if target the last column is the target. + // This will need to be updated when we support unsupervised learning models. 'nfeatures' => count(current($dataset)) - 1 ); - if ($target) { - $metadata['targetclasses'] = json_encode($target::get_classes()); - $metadata['targettype'] = ($target->is_linear()) ? 'linear' : 'discrete'; - } // The first 2 samples will be used to store metadata about the dataset. $metadatacolumns = []; diff --git a/analytics/classes/model.php b/analytics/classes/model.php index 46c461b70a6..7ad6d8f83f1 100644 --- a/analytics/classes/model.php +++ b/analytics/classes/model.php @@ -80,6 +80,11 @@ class model { */ const MIN_SCORE = 0.7; + /** + * Minimum prediction confidence (from 0 to 1) to accept a prediction as reliable enough. + */ + const PREDICTION_MIN_SCORE = 0.6; + /** * Maximum standard deviation between different evaluation repetitions to consider that evaluation results are stable. */ @@ -524,8 +529,13 @@ class model { $outputdir = $this->get_output_dir(array('evaluation', $dashestimesplittingid)); // Evaluate the dataset, the deviation we accept in the results depends on the amount of iterations. - $predictorresult = $predictor->evaluate($this->model->id, self::ACCEPTED_DEVIATION, + if ($this->get_target()->is_linear()) { + $predictorresult = $predictor->evaluate_regression($this->get_unique_id(), self::ACCEPTED_DEVIATION, self::EVALUATION_ITERATIONS, $dataset, $outputdir); + } else { + $predictorresult = $predictor->evaluate_classification($this->get_unique_id(), self::ACCEPTED_DEVIATION, + self::EVALUATION_ITERATIONS, $dataset, $outputdir); + } $result->status = $predictorresult->status; $result->info = $predictorresult->info; @@ -599,7 +609,11 @@ class model { $samplesfile = $datasets[$this->model->timesplitting]; // Train using the dataset. - $predictorresult = $predictor->train($this->get_unique_id(), $samplesfile, $outputdir); + if ($this->get_target()->is_linear()) { + $predictorresult = $predictor->train_regression($this->get_unique_id(), $samplesfile, $outputdir); + } else { + $predictorresult = $predictor->train_classification($this->get_unique_id(), $samplesfile, $outputdir); + } $result = new \stdClass(); $result->status = $predictorresult->status; @@ -678,8 +692,12 @@ class model { $result->predictions = $this->get_static_predictions($indicatorcalculations); } else { - // Prediction process runs on the machine learning backend. - $predictorresult = $predictor->predict($this->get_unique_id(), $samplesfile, $outputdir); + // Estimation and classification processes run on the machine learning backend side. + if ($this->get_target()->is_linear()) { + $predictorresult = $predictor->estimate($this->get_unique_id(), $samplesfile, $outputdir); + } else { + $predictorresult = $predictor->classify($this->get_unique_id(), $samplesfile, $outputdir); + } $result->status = $predictorresult->status; $result->info = $predictorresult->info; $result->predictions = $this->format_predictor_predictions($predictorresult); diff --git a/analytics/classes/predictor.php b/analytics/classes/predictor.php index 944c76eeb2d..277c6a96dd2 100644 --- a/analytics/classes/predictor.php +++ b/analytics/classes/predictor.php @@ -43,34 +43,67 @@ interface predictor { public function is_ready(); /** - * Train the provided dataset. + * Train this processor classification model using the provided supervised learning dataset. * - * @param int $modelid + * @param string $uniqueid * @param \stored_file $dataset * @param string $outputdir * @return \stdClass */ - public function train($modelid, \stored_file $dataset, $outputdir); + public function train_classification($uniqueid, \stored_file $dataset, $outputdir); /** - * Predict the provided dataset samples. + * Classifies the provided dataset samples. * - * @param int $modelid + * @param string $uniqueid * @param \stored_file $dataset * @param string $outputdir * @return \stdClass */ - public function predict($modelid, \stored_file $dataset, $outputdir); + public function classify($uniqueid, \stored_file $dataset, $outputdir); /** - * evaluate + * Evaluates this processor classification model using the provided supervised learning dataset. * - * @param int $modelid + * @param string $uniqueid * @param float $maxdeviation * @param int $niterations * @param \stored_file $dataset * @param string $outputdir * @return \stdClass */ - public function evaluate($modelid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir); + public function evaluate_classification($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir); + + /** + * Train this processor regression model using the provided supervised learning dataset. + * + * @param string $uniqueid + * @param \stored_file $dataset + * @param string $outputdir + * @return \stdClass + */ + public function train_regression($uniqueid, \stored_file $dataset, $outputdir); + + /** + * Estimates linear values for the provided dataset samples. + * + * @param string $uniqueid + * @param \stored_file $dataset + * @param mixed $outputdir + * @return void + */ + public function estimate($uniqueid, \stored_file $dataset, $outputdir); + + /** + * Evaluates this processor regression model using the provided supervised learning dataset. + * + * @param string $uniqueid + * @param float $maxdeviation + * @param int $niterations + * @param \stored_file $dataset + * @param string $outputdir + * @return \stdClass + */ + public function evaluate_regression($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir); + } diff --git a/course/classes/analytics/indicator/no_teacher.php b/course/classes/analytics/indicator/no_teacher.php index 99051c771dc..51e18070231 100644 --- a/course/classes/analytics/indicator/no_teacher.php +++ b/course/classes/analytics/indicator/no_teacher.php @@ -64,6 +64,23 @@ class no_teacher extends \core_analytics\local\indicator\binary { return array('context', 'course'); } + /** + * Reversed because the indicator is in 'negative' and the max returned value means teacher present. + * + * @param float $value + * @param string $subtype + * @return string + */ + public function get_display_value($value, $subtype = false) { + + // No subtypes for binary values by default. + if ($value == -1) { + return get_string('yes'); + } else if ($value == 1) { + return get_string('no'); + } + } + /** * calculate_sample * diff --git a/lib/mlbackend/php/classes/processor.php b/lib/mlbackend/php/classes/processor.php index 1981102bcb4..f6a2922b06b 100644 --- a/lib/mlbackend/php/classes/processor.php +++ b/lib/mlbackend/php/classes/processor.php @@ -73,14 +73,14 @@ class processor implements \core_analytics\predictor { } /** - * Trains a machine learning algorithm with the provided training set. + * Train this processor classification model using the provided supervised learning dataset. * * @param string $uniqueid * @param \stored_file $dataset * @param string $outputdir * @return \stdClass */ - public function train($uniqueid, \stored_file $dataset, $outputdir) { + public function train_classification($uniqueid, \stored_file $dataset, $outputdir) { // Output directory is already unique to the model. $modelfilepath = $outputdir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME; @@ -134,14 +134,14 @@ class processor implements \core_analytics\predictor { } /** - * Predicts the provided samples + * Classifies the provided dataset samples. * * @param string $uniqueid * @param \stored_file $dataset * @param string $outputdir * @return \stdClass */ - public function predict($uniqueid, \stored_file $dataset, $outputdir) { + public function classify($uniqueid, \stored_file $dataset, $outputdir) { // Output directory is already unique to the model. $modelfilepath = $outputdir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME; @@ -199,7 +199,7 @@ class processor implements \core_analytics\predictor { } /** - * Evaluates the provided dataset. + * Evaluates this processor classification model using the provided supervised learning dataset. * * During evaluation we need to shuffle the evaluation dataset samples to detect deviated results, * if the dataset is massive we can not load everything into memory. We know that 2GB is the @@ -216,7 +216,7 @@ class processor implements \core_analytics\predictor { * @param string $outputdir * @return \stdClass */ - public function evaluate($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) { + public function evaluate_classification($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) { $fh = $dataset->get_content_file_handle(); // The first lines are var names and the second one values. @@ -351,6 +351,47 @@ class processor implements \core_analytics\predictor { return $resultobj; } + /** + * Train this processor regression model using the provided supervised learning dataset. + * + * @throws new \coding_exception + * @param string $uniqueid + * @param \stored_file $dataset + * @param string $outputdir + * @return \stdClass + */ + public function train_regression($uniqueid, \stored_file $dataset, $outputdir) { + throw new \coding_exception('This predictor does not support regression yet.'); + } + + /** + * Estimates linear values for the provided dataset samples. + * + * @throws new \coding_exception + * @param string $uniqueid + * @param \stored_file $dataset + * @param mixed $outputdir + * @return void + */ + public function estimate($uniqueid, \stored_file $dataset, $outputdir) { + throw new \coding_exception('This predictor does not support regression yet.'); + } + + /** + * Evaluates this processor regression model using the provided supervised learning dataset. + * + * @throws new \coding_exception + * @param string $uniqueid + * @param float $maxdeviation + * @param int $niterations + * @param \stored_file $dataset + * @param string $outputdir + * @return \stdClass + */ + public function evaluate_regression($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) { + throw new \coding_exception('This predictor does not support regression yet.'); + } + /** * Returns the Phi correlation coefficient. * diff --git a/lib/mlbackend/python/classes/processor.php b/lib/mlbackend/python/classes/processor.php index 5b05fa6daab..570c91e767a 100644 --- a/lib/mlbackend/python/classes/processor.php +++ b/lib/mlbackend/python/classes/processor.php @@ -79,7 +79,7 @@ class processor implements \core_analytics\predictor { * @param string $outputdir * @return \stdClass */ - public function train($uniqueid, \stored_file $dataset, $outputdir) { + public function train_classification($uniqueid, \stored_file $dataset, $outputdir) { // Obtain the physical route to the file. $datasetpath = $this->get_file_path($dataset); @@ -113,14 +113,14 @@ class processor implements \core_analytics\predictor { } /** - * Returns predictions for the provided dataset samples. + * Classifies the provided dataset samples. * * @param string $uniqueid * @param \stored_file $dataset * @param string $outputdir * @return \stdClass */ - public function predict($uniqueid, \stored_file $dataset, $outputdir) { + public function classify($uniqueid, \stored_file $dataset, $outputdir) { // Obtain the physical route to the file. $datasetpath = $this->get_file_path($dataset); @@ -154,7 +154,7 @@ class processor implements \core_analytics\predictor { } /** - * Evaluates the provided dataset. + * Evaluates this processor classification model using the provided supervised learning dataset. * * @param string $uniqueid * @param float $maxdeviation @@ -163,7 +163,7 @@ class processor implements \core_analytics\predictor { * @param string $outputdir * @return \stdClass */ - public function evaluate($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) { + public function evaluate_classification($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) { // Obtain the physical route to the file. $datasetpath = $this->get_file_path($dataset); @@ -195,6 +195,47 @@ class processor implements \core_analytics\predictor { return $resultobj; } + /** + * Train this processor regression model using the provided supervised learning dataset. + * + * @throws new \coding_exception + * @param string $uniqueid + * @param \stored_file $dataset + * @param string $outputdir + * @return \stdClass + */ + public function train_regression($uniqueid, \stored_file $dataset, $outputdir) { + throw new \coding_exception('This predictor does not support regression yet.'); + } + + /** + * Estimates linear values for the provided dataset samples. + * + * @throws new \coding_exception + * @param string $uniqueid + * @param \stored_file $dataset + * @param mixed $outputdir + * @return void + */ + public function estimate($uniqueid, \stored_file $dataset, $outputdir) { + throw new \coding_exception('This predictor does not support regression yet.'); + } + + /** + * Evaluates this processor regression model using the provided supervised learning dataset. + * + * @throws new \coding_exception + * @param string $uniqueid + * @param float $maxdeviation + * @param int $niterations + * @param \stored_file $dataset + * @param string $outputdir + * @return \stdClass + */ + public function evaluate_regression($uniqueid, $maxdeviation, $niterations, \stored_file $dataset, $outputdir) { + throw new \coding_exception('This predictor does not support regression yet.'); + } + /** * Returns the path to the dataset file. * diff --git a/report/insights/classes/output/insight.php b/report/insights/classes/output/insight.php index 51b7fe652b8..134283e8ac6 100644 --- a/report/insights/classes/output/insight.php +++ b/report/insights/classes/output/insight.php @@ -134,7 +134,7 @@ class insight implements \renderable, \templatable { * Returns a CSS class from the calculated value outcome. * * @param \core_analytics\calculable $calculable - * @param mixed $value + * @param float $value * @param string|false $subtype * @return string */ @@ -159,8 +159,8 @@ class insight implements \renderable, \templatable { default: throw new \coding_exception('The outcome returned by ' . get_class($calculable) . '::get_calculation_outcome is ' . 'not one of the accepted values. Please use \core_analytics\calculable::OUTCOME_VERY_POSITIVE, ' . - '\core_analytics\calculable::OUTCOME_OK, \core_analytics\calculable::OUTCOME_NEGATIVE or ' . - '\core_analytics\calculable::OUTCOME_VERY_NEGATIVE'); + '\core_analytics\calculable::OUTCOME_OK, \core_analytics\calculable::OUTCOME_NEGATIVE, ' . + '\core_analytics\calculable::OUTCOME_VERY_NEGATIVE or \core_analytics\calculable::OUTCOME_NEUTRAL'); } return $style; }