diff --git a/lib/mlbackend/php/classes/processor.php b/lib/mlbackend/php/classes/processor.php index 98b98cb2c2b..840236a03d2 100644 --- a/lib/mlbackend/php/classes/processor.php +++ b/lib/mlbackend/php/classes/processor.php @@ -31,6 +31,7 @@ use Phpml\CrossValidation\RandomSplit; use Phpml\Dataset\ArrayDataset; use Phpml\ModelManager; use Phpml\Classification\Linear\LogisticRegression; +use Phpml\Metric\ClassificationReport; /** * PHP predictions processor. @@ -309,7 +310,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor return $resultobj; } - $phis = array(); + $scores = array(); // Evaluate the model multiple times to confirm the results are not significantly random due to a short amount of data. for ($i = 0; $i < $niterations; $i++) { @@ -322,39 +323,43 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor $classifier->train($data->getTrainSamples(), $data->getTrainLabels()); $predictedlabels = $classifier->predict($data->getTestSamples()); - $phis[] = $this->get_phi($data->getTestLabels(), $predictedlabels); + $report = new ClassificationReport($data->getTestLabels(), $predictedlabels, + ClassificationReport::WEIGHTED_AVERAGE); } else { $predictedlabels = $classifier->predict($samples); - $phis[] = $this->get_phi($targets, $predictedlabels); + $report = new ClassificationReport($targets, $predictedlabels, + ClassificationReport::WEIGHTED_AVERAGE); } + $averages = $report->getAverage(); + $scores[] = $averages['f1score']; } // Let's fill the results changing the returned status code depending on the phi-related calculated metrics. - return $this->get_evaluation_result_object($dataset, $phis, $maxdeviation); + return $this->get_evaluation_result_object($dataset, $scores, $maxdeviation); } /** * Returns the results objects from all evaluations. * * @param \stored_file $dataset - * @param array $phis + * @param array $scores * @param float $maxdeviation * @return \stdClass */ - protected function get_evaluation_result_object(\stored_file $dataset, $phis, $maxdeviation) { + protected function get_evaluation_result_object(\stored_file $dataset, $scores, $maxdeviation) { - // Average phi of all evaluations as final score. - if (count($phis) === 1) { - $avgphi = reset($phis); + // Average f1 score of all evaluations as final score. + if (count($scores) === 1) { + $avgscore = reset($scores); } else { - $avgphi = \Phpml\Math\Statistic\Mean::arithmetic($phis); + $avgscore = \Phpml\Math\Statistic\Mean::arithmetic($scores); } // Standard deviation should ideally be calculated against the area under the curve. - if (count($phis) === 1) { + if (count($scores) === 1) { $modeldev = 0; } else { - $modeldev = \Phpml\Math\Statistic\StandardDeviation::population($phis); + $modeldev = \Phpml\Math\Statistic\StandardDeviation::population($scores); } // Let's fill the results object. @@ -363,9 +368,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor // Zero is ok, now we add other bits if something is not right. $resultobj->status = \core_analytics\model::OK; $resultobj->info = array(); - - // Convert phi to a standard score (from -1 to 1 to a value between 0 and 1). - $resultobj->score = ($avgphi + 1) / 2; + $resultobj->score = $avgscore; // If each iteration results varied too much we need more data to confirm that this is a valid model. if ($modeldev > $maxdeviation) { @@ -523,33 +526,6 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor return $modeldir . DIRECTORY_SEPARATOR . self::MODEL_FILENAME; } - /** - * Returns the Phi correlation coefficient. - * - * @param array $testlabels - * @param array $predictedlabels - * @return float - */ - protected function get_phi($testlabels, $predictedlabels) { - - // Binary here only as well. - $matrix = \Phpml\Metric\ConfusionMatrix::compute($testlabels, $predictedlabels, array(0, 1)); - - $tptn = $matrix[0][0] * $matrix[1][1]; - $fpfn = $matrix[1][0] * $matrix[0][1]; - $tpfp = $matrix[0][0] + $matrix[1][0]; - $tpfn = $matrix[0][0] + $matrix[0][1]; - $tnfp = $matrix[1][1] + $matrix[1][0]; - $tnfn = $matrix[1][1] + $matrix[0][1]; - if ($tpfp === 0 || $tpfn === 0 || $tnfp === 0 || $tnfn === 0) { - $phi = 0; - } else { - $phi = ( $tptn - $fpfn ) / sqrt( $tpfp * $tpfn * $tnfp * $tnfn); - } - - return $phi; - } - /** * Extracts metadata from the dataset file. * diff --git a/lib/mlbackend/php/upgrade.txt b/lib/mlbackend/php/upgrade.txt new file mode 100644 index 00000000000..917447c1218 --- /dev/null +++ b/lib/mlbackend/php/upgrade.txt @@ -0,0 +1,8 @@ +This files describes API changes in the mlbackend_php code, the +information provided here is intended especially for developers. + +=== 3.8 === + +* The phi coefficient (Matthews' correlation coefficient) has been replaced by + the F1 score as the main accuracy metric. Therefore, \mlbackend_php\processor::get_phi + method has been removed. diff --git a/lib/mlbackend/python/classes/processor.php b/lib/mlbackend/python/classes/processor.php index c0729bdc44c..32b971b6ffb 100644 --- a/lib/mlbackend/python/classes/processor.php +++ b/lib/mlbackend/python/classes/processor.php @@ -38,7 +38,7 @@ class processor implements \core_analytics\classifier, \core_analytics\regresso /** * The required version of the python package that performs all calculations. */ - const REQUIRED_PIP_PACKAGE_VERSION = '2.2.1'; + const REQUIRED_PIP_PACKAGE_VERSION = '2.3.0'; /** * The python package is installed in a server. diff --git a/lib/mlbackend/python/upgrade.txt b/lib/mlbackend/python/upgrade.txt new file mode 100644 index 00000000000..b799d116b46 --- /dev/null +++ b/lib/mlbackend/python/upgrade.txt @@ -0,0 +1,7 @@ +This files describes API changes in the mlbackend_python code, the +information provided here is intended especially for developers. + +=== 3.8 === + +* The phi coefficient (Matthews' correlation coefficient) has been replaced by + the F1 score as the main accuracy metric.