L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MDL-66476 mlbackend: Use F1 as the main accuracy metric

Browse Source
This commit is contained in:
David Monllaó
2019-10-04 05:20:49 +08:00
parent ebec727cd3
commit f5c65ba37f
4 changed files with 34 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/mlbackend/php/classes/processor.php
+18 -42
View File
@@ -31,6 +31,7 @@ use Phpml\CrossValidation\RandomSplit;
use Phpml\Dataset\ArrayDataset;
use Phpml\ModelManager;
use Phpml\Classification\Linear\LogisticRegression;
use Phpml\Metric\ClassificationReport;
/**
* PHP predictions processor.
@@ -309,7 +310,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
return $resultobj;
}
$phis = array();
$scores = array();
// Evaluate the model multiple times to confirm the results are not significantly random due to a short amount of data.
for ($i = 0; $i < $niterations; $i++) {
@@ -322,39 +323,43 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
$classifier->train($data->getTrainSamples(), $data->getTrainLabels());
$predictedlabels = $classifier->predict($data->getTestSamples());
$phis[] = $this->get_phi($data->getTestLabels(), $predictedlabels);
$report = new ClassificationReport($data->getTestLabels(), $predictedlabels,
ClassificationReport::WEIGHTED_AVERAGE);
} else {
$predictedlabels = $classifier->predict($samples);
$phis[] = $this->get_phi($targets, $predictedlabels);
$report = new ClassificationReport($targets, $predictedlabels,
ClassificationReport::WEIGHTED_AVERAGE);
}
$averages = $report->getAverage();
$scores[] = $averages['f1score'];
}
// Let's fill the results changing the returned status code depending on the phi-related calculated metrics.
return $this->get_evaluation_result_object($dataset, $phis, $maxdeviation);
return $this->get_evaluation_result_object($dataset, $scores, $maxdeviation);
}
/**
* Returns the results objects from all evaluations.
*
* @param \stored_file $dataset
* @param array $phis
* @param array $scores
* @param float $maxdeviation
* @return \stdClass
*/
protected function get_evaluation_result_object(\stored_file $dataset, $phis, $maxdeviation) {
protected function get_evaluation_result_object(\stored_file $dataset, $scores, $maxdeviation) {
// Average phi of all evaluations as final score.
if (count($phis) === 1) {
$avgphi = reset($phis);
// Average f1 score of all evaluations as final score.
if (count($scores) === 1) {
$avgscore = reset($scores);
} else {
$avgphi = \Phpml\Math\Statistic\Mean::arithmetic($phis);
$avgscore = \Phpml\Math\Statistic\Mean::arithmetic($scores);
}
// Standard deviation should ideally be calculated against the area under the curve.
if (count($phis) === 1) {
if (count($scores) === 1) {
$modeldev = 0;
} else {
$modeldev = \Phpml\Math\Statistic\StandardDeviation::population($phis);
$modeldev = \Phpml\Math\Statistic\StandardDeviation::population($scores);
}
// Let's fill the results object.
@@ -363,9 +368,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
// Zero is ok, now we add other bits if something is not right.
$resultobj->status = \core_analytics\model::OK;
$resultobj->info = array();
// Convert phi to a standard score (from -1 to 1 to a value between 0 and 1).
$resultobj->score = ($avgphi + 1) / 2;
$resultobj->score = $avgscore;
// If each iteration results varied too much we need more data to confirm that this is a valid model.
if ($modeldev > $maxdeviation) {
@@ -523,33 +526,6 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
return $modeldir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME;
}
/**
* Returns the Phi correlation coefficient.
*
* @param array $testlabels
* @param array $predictedlabels
* @return float
*/
protected function get_phi($testlabels, $predictedlabels) {
// Binary here only as well.
$matrix = \Phpml\Metric\ConfusionMatrix::compute($testlabels, $predictedlabels, array(0, 1));
$tptn = $matrix[0][0] * $matrix[1][1];
$fpfn = $matrix[1][0] * $matrix[0][1];
$tpfp = $matrix[0][0] + $matrix[1][0];
$tpfn = $matrix[0][0] + $matrix[0][1];
$tnfp = $matrix[1][1] + $matrix[1][0];
$tnfn = $matrix[1][1] + $matrix[0][1];
if ($tpfp === 0 || $tpfn === 0 || $tnfp === 0 || $tnfn === 0) {
$phi = 0;
} else {
$phi = ( $tptn - $fpfn ) / sqrt( $tpfp * $tpfn * $tnfp * $tnfn);
}
return $phi;
}
/**
* Extracts metadata from the dataset file.
*
lib/mlbackend/php/upgrade.txt
+8
View File
@@ -0,0 +1,8 @@
This files describes API changes in the mlbackend_php code, the
information provided here is intended especially for developers.
=== 3.8 ===
* The phi coefficient (Matthews' correlation coefficient) has been replaced by
the F1 score as the main accuracy metric. Therefore, \mlbackend_php\processor::get_phi
method has been removed.
lib/mlbackend/python/classes/processor.php
+1 -1
View File
@@ -38,7 +38,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regresso
/**
* The required version of the python package that performs all calculations.
*/
const REQUIRED_PIP_PACKAGE_VERSION = '2.2.1';
const REQUIRED_PIP_PACKAGE_VERSION = '2.3.0';
/**
* The python package is installed in a server.
lib/mlbackend/python/upgrade.txt
+7
View File
@@ -0,0 +1,7 @@
This files describes API changes in the mlbackend_python code, the
information provided here is intended especially for developers.
=== 3.8 ===
* The phi coefficient (Matthews' correlation coefficient) has been replaced by
the F1 score as the main accuracy metric.