From ebec727cd38adee580222be0bd17a9b285a0714d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Monlla=C3=B3?= Date: Thu, 3 Oct 2019 09:45:57 +0800 Subject: [PATCH 1/2] MDL-66004 mlbackend_python: Purge testing output dirs properly --- analytics/tests/prediction_test.php | 24 +++++++++++++++++++++- lib/mlbackend/python/classes/processor.php | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/analytics/tests/prediction_test.php b/analytics/tests/prediction_test.php index b0471f2ef8e..2b437f6270c 100644 --- a/analytics/tests/prediction_test.php +++ b/analytics/tests/prediction_test.php @@ -54,6 +54,23 @@ require_once(__DIR__ . '/../../course/lib.php'); */ class core_analytics_prediction_testcase extends advanced_testcase { + /** + * Purge all the mlbackend outputs. + * + * This is done automatically for mlbackends using the web server dataroot but + * other mlbackends may store files elsewhere and these files need to be removed. + * + * @return null + */ + public function tearDown() { + $this->setAdminUser(); + + $models = \core_analytics\manager::get_all_models(); + foreach ($models as $model) { + $model->delete(); + } + } + /** * test_static_prediction * @@ -414,7 +431,8 @@ class core_analytics_prediction_testcase extends advanced_testcase { // Training should work correctly if at least 1 sample of each class is included. $dir = make_request_directory(); - $result = $predictionsprocessor->train_classification('whatever' . microtime(), $dataset, $dir); + $modeluniqueid = 'whatever' . microtime(); + $result = $predictionsprocessor->train_classification($modeluniqueid, $dataset, $dir); switch ($success) { case 'yes': @@ -429,6 +447,10 @@ class core_analytics_prediction_testcase extends advanced_testcase { // what we really want to check is that an exception was not thrown. $this->assertInstanceOf(\stdClass::class, $result); } + + // Purge the directory used in this test (useful in case the mlbackend is storing files + // somewhere out of the default moodledata/models dir. + $predictionsprocessor->delete_output_dir($dir, $modeluniqueid); } /** diff --git a/lib/mlbackend/python/classes/processor.php b/lib/mlbackend/python/classes/processor.php index 25861cead21..c0729bdc44c 100644 --- a/lib/mlbackend/python/classes/processor.php +++ b/lib/mlbackend/python/classes/processor.php @@ -38,7 +38,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regresso /** * The required version of the python package that performs all calculations. */ - const REQUIRED_PIP_PACKAGE_VERSION = '2.2.0'; + const REQUIRED_PIP_PACKAGE_VERSION = '2.2.1'; /** * The python package is installed in a server. From f5c65ba37fba2b27cb25e494ca4484ac496f296d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Monlla=C3=B3?= Date: Tue, 1 Oct 2019 14:16:37 +0800 Subject: [PATCH 2/2] MDL-66476 mlbackend: Use F1 as the main accuracy metric --- lib/mlbackend/php/classes/processor.php | 60 +++++++--------------- lib/mlbackend/php/upgrade.txt | 8 +++ lib/mlbackend/python/classes/processor.php | 2 +- lib/mlbackend/python/upgrade.txt | 7 +++ 4 files changed, 34 insertions(+), 43 deletions(-) create mode 100644 lib/mlbackend/php/upgrade.txt create mode 100644 lib/mlbackend/python/upgrade.txt diff --git a/lib/mlbackend/php/classes/processor.php b/lib/mlbackend/php/classes/processor.php index 98b98cb2c2b..840236a03d2 100644 --- a/lib/mlbackend/php/classes/processor.php +++ b/lib/mlbackend/php/classes/processor.php @@ -31,6 +31,7 @@ use Phpml\CrossValidation\RandomSplit; use Phpml\Dataset\ArrayDataset; use Phpml\ModelManager; use Phpml\Classification\Linear\LogisticRegression; +use Phpml\Metric\ClassificationReport; /** * PHP predictions processor. @@ -309,7 +310,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor return $resultobj; } - $phis = array(); + $scores = array(); // Evaluate the model multiple times to confirm the results are not significantly random due to a short amount of data. for ($i = 0; $i < $niterations; $i++) { @@ -322,39 +323,43 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor $classifier->train($data->getTrainSamples(), $data->getTrainLabels()); $predictedlabels = $classifier->predict($data->getTestSamples()); - $phis[] = $this->get_phi($data->getTestLabels(), $predictedlabels); + $report = new ClassificationReport($data->getTestLabels(), $predictedlabels, + ClassificationReport::WEIGHTED_AVERAGE); } else { $predictedlabels = $classifier->predict($samples); - $phis[] = $this->get_phi($targets, $predictedlabels); + $report = new ClassificationReport($targets, $predictedlabels, + ClassificationReport::WEIGHTED_AVERAGE); } + $averages = $report->getAverage(); + $scores[] = $averages['f1score']; } // Let's fill the results changing the returned status code depending on the phi-related calculated metrics. - return $this->get_evaluation_result_object($dataset, $phis, $maxdeviation); + return $this->get_evaluation_result_object($dataset, $scores, $maxdeviation); } /** * Returns the results objects from all evaluations. * * @param \stored_file $dataset - * @param array $phis + * @param array $scores * @param float $maxdeviation * @return \stdClass */ - protected function get_evaluation_result_object(\stored_file $dataset, $phis, $maxdeviation) { + protected function get_evaluation_result_object(\stored_file $dataset, $scores, $maxdeviation) { - // Average phi of all evaluations as final score. - if (count($phis) === 1) { - $avgphi = reset($phis); + // Average f1 score of all evaluations as final score. + if (count($scores) === 1) { + $avgscore = reset($scores); } else { - $avgphi = \Phpml\Math\Statistic\Mean::arithmetic($phis); + $avgscore = \Phpml\Math\Statistic\Mean::arithmetic($scores); } // Standard deviation should ideally be calculated against the area under the curve. - if (count($phis) === 1) { + if (count($scores) === 1) { $modeldev = 0; } else { - $modeldev = \Phpml\Math\Statistic\StandardDeviation::population($phis); + $modeldev = \Phpml\Math\Statistic\StandardDeviation::population($scores); } // Let's fill the results object. @@ -363,9 +368,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor // Zero is ok, now we add other bits if something is not right. $resultobj->status = \core_analytics\model::OK; $resultobj->info = array(); - - // Convert phi to a standard score (from -1 to 1 to a value between 0 and 1). - $resultobj->score = ($avgphi + 1) / 2; + $resultobj->score = $avgscore; // If each iteration results varied too much we need more data to confirm that this is a valid model. if ($modeldev > $maxdeviation) { @@ -523,33 +526,6 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor return $modeldir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME; } - /** - * Returns the Phi correlation coefficient. - * - * @param array $testlabels - * @param array $predictedlabels - * @return float - */ - protected function get_phi($testlabels, $predictedlabels) { - - // Binary here only as well. - $matrix = \Phpml\Metric\ConfusionMatrix::compute($testlabels, $predictedlabels, array(0, 1)); - - $tptn = $matrix[0][0] * $matrix[1][1]; - $fpfn = $matrix[1][0] * $matrix[0][1]; - $tpfp = $matrix[0][0] + $matrix[1][0]; - $tpfn = $matrix[0][0] + $matrix[0][1]; - $tnfp = $matrix[1][1] + $matrix[1][0]; - $tnfn = $matrix[1][1] + $matrix[0][1]; - if ($tpfp === 0 || $tpfn === 0 || $tnfp === 0 || $tnfn === 0) { - $phi = 0; - } else { - $phi = ( $tptn - $fpfn ) / sqrt( $tpfp * $tpfn * $tnfp * $tnfn); - } - - return $phi; - } - /** * Extracts metadata from the dataset file. * diff --git a/lib/mlbackend/php/upgrade.txt b/lib/mlbackend/php/upgrade.txt new file mode 100644 index 00000000000..917447c1218 --- /dev/null +++ b/lib/mlbackend/php/upgrade.txt @@ -0,0 +1,8 @@ +This files describes API changes in the mlbackend_php code, the +information provided here is intended especially for developers. + +=== 3.8 === + +* The phi coefficient (Matthews' correlation coefficient) has been replaced by + the F1 score as the main accuracy metric. Therefore, \mlbackend_php\processor::get_phi + method has been removed. diff --git a/lib/mlbackend/python/classes/processor.php b/lib/mlbackend/python/classes/processor.php index c0729bdc44c..32b971b6ffb 100644 --- a/lib/mlbackend/python/classes/processor.php +++ b/lib/mlbackend/python/classes/processor.php @@ -38,7 +38,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regresso /** * The required version of the python package that performs all calculations. */ - const REQUIRED_PIP_PACKAGE_VERSION = '2.2.1'; + const REQUIRED_PIP_PACKAGE_VERSION = '2.3.0'; /** * The python package is installed in a server. diff --git a/lib/mlbackend/python/upgrade.txt b/lib/mlbackend/python/upgrade.txt new file mode 100644 index 00000000000..b799d116b46 --- /dev/null +++ b/lib/mlbackend/python/upgrade.txt @@ -0,0 +1,7 @@ +This files describes API changes in the mlbackend_python code, the +information provided here is intended especially for developers. + +=== 3.8 === + +* The phi coefficient (Matthews' correlation coefficient) has been replaced by + the F1 score as the main accuracy metric.