L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Grading evaluation - best subplugin

Browse Source 
I am not happy with the algorithm at all. We should replace it with some
more sophisticated subplugin, using ICC or some similar statistics.
This commit is contained in:
David Mudrak
2010-01-04 18:14:01 +00:00
parent b286b6efe4
commit ad6a8f6911
5 changed files with 500 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mod/workshop/aggregate.php
+4 -4
View File
@@ -46,10 +46,10 @@ if ($confirm) {
if (!confirm_sesskey()) {
throw new moodle_exception('confirmsesskeybad');
}
$workshop->aggregate_submission_grades();
//$evaluator->update_grading_grades();
//$workshop->aggregate_grading_grades();
//$workshop->aggregate_total_grades();
$workshop->aggregate_submission_grades(); // updates 'grade' in {workshop_submissions}
$evaluator->update_grading_grades(); // updates 'gradinggrade' in {workshop_assessments}
$workshop->aggregate_grading_grades(); // updates 'gradinggrade' in {workshop_aggregations}
$workshop->aggregate_total_grades(); // updates 'totalgrade' in {workshop_aggregations}
redirect($workshop->view_url());
}
mod/workshop/eval/best/lib.php
+246 -24
View File
@@ -48,7 +48,10 @@ class workshop_best_evaluation implements workshop_evaluation {
}
/**
* TODO
* Calculates the grades for assessment and updates 'gradinggrade' fields in 'workshop_assessments' table
*
* This function relies on the grading strategy subplugin providing get_assessments_recordset() method.
* {@see self::process_assessments()} for the required structure of the recordset.
*
* @param null|int|array $restrict If null, update all reviewers, otherwise update just grades for the given reviewers(s)
*
@@ -63,8 +66,11 @@ class workshop_best_evaluation implements workshop_evaluation {
support this method of grading evaluation.');
}
// get the information about the assessment dimensions
$diminfo = $grader->eval_best_dimensions_info();
// fetch a recordset with all assessments to process
$rs = $grader->get_assessments_recordset($restrict);
$rs = $grader->eval_best_get_assessments_recordset($restrict);
$batch = array(); // will contain a set of all assessments of a single submission
$previous = null; // a previous record in the recordset
foreach ($rs as $current) {
@@ -76,52 +82,268 @@ class workshop_best_evaluation implements workshop_evaluation {
$batch[] = $current;
} else {
// process all the assessments of a sigle submission
$this->process_assessments($batch);
$this->process_assessments($batch, $diminfo);
// start with a new batch to be processed
$batch = array($current);
$previous = $current;
}
}
// do not forget to process the last batch!
$this->process_assessments($batch);
$this->process_assessments($batch, $diminfo);
$rs->close();
}
////////////////////////////////////////////////////////////////////////////////
// Internal methods //
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// Internal methods //
////////////////////////////////////////////////////////////////////////////////
/**
* Given a list of all assessments of a single submission, updates the grading grades in database
*
* @param array $assessments of stdClass Object(->assessmentid ->assessmentweight ->reviewerid ->submissionid
* ->dimensionid ->grade ->dimensionweight)
* @param array $assessments of stdClass (->assessmentid ->assessmentweight ->reviewerid ->gradinggrade ->submissionid ->dimensionid ->grade)
* @param array $diminfo of stdClass (->id ->weight ->max ->min)
* @return void
*/
protected function process_assessments(array $assessments) {
protected function process_assessments(array $assessments, array $diminfo) {
global $DB;
$grades = $this->evaluate_assessments($assessments);
foreach ($grades as $assessmentid => $grade) {
$record = new stdClass();
$record->id = $assessmentid;
$record->gradinggrade = grade_floatval($grade);
$DB->update_record('workshop_assessments', $record, true);
// reindex the passed flat structure to be indexed by assessmentid
$assessments = $this->prepare_data_from_recordset($assessments);
// normalize the dimension grades to the interval 0 - 100
$assessments = $this->normalize_grades($assessments, $diminfo);
// get a hypothetical average assessment
$average = $this->average_assessment($assessments);
// calculate variance of dimension grades
$variances = $this->weighted_variance($assessments);
foreach ($variances as $dimid => $variance) {
$diminfo[$dimid]->variance = $variance;
}
// for every assessment, calculate its distance from the average one
$distances = array();
foreach ($assessments as $asid => $assessment) {
$distances[$asid] = $this->assessments_distance($assessment, $average, $diminfo);
}
// identify the best assessments - it est those with the shortest distance from the best assessment
$bestids = array_keys($distances, min($distances));
// for every assessment, calculate its distance from the nearest best assessment
$distances = array();
foreach ($bestids as $bestid) {
$best = $assessments[$bestid];
foreach ($assessments as $asid => $assessment) {
$d = $this->assessments_distance($assessment, $best, $diminfo);
if (!isset($distances[$asid]) or $d < $distances[$asid]) {
$distances[$asid] = $d;
}
}
}
// calculate the grading grade
foreach ($distances as $asid => $distance) {
$gradinggrade = (100 - $distance);
/**
if ($gradinggrade < 0) {
$gradinggrade = 0;
}
if ($gradinggrade > 100) {
$gradinggrade = 100;
}
*/
$grades[$asid] = grade_floatval($gradinggrade);
}
// if the new grading grade differs from the one stored in database, update it
// we do not use set_field() here because we want to pass $bulk param
foreach ($grades as $assessmentid => $grade) {
if (grade_floats_different($grade, $assessments[$assessmentid]->gradinggrade)) {
// the value has changed
$record = new stdClass();
$record->id = $assessmentid;
$record->gradinggrade = grade_floatval($grade);
$DB->update_record('workshop_assessments', $record, true); // bulk operations expected
}
}
// done. easy, heh? ;-)
}
/**
* Given a list of all assessments of a single submission, calculates the grading grades for them
* Prepares a structure of assessments and given grades
*
* @param array $assessments same structure as for {@link self::process_assessments()}
* @return array [(int)assessmentid => (float)gradinggrade] to be saved into {workshop_assessments}
* @param array $assessments batch of recordset items as returned by the grading strategy
* @return array
*/
protected function evaluate_assessments(array $assessments) {
$gradinggrades = array();
foreach ($assessments as $assessment) {
$gradinggrades[$assessment->assessmentid] = grade_floatval(rand(0, 100)); // todo
protected function prepare_data_from_recordset($assessments) {
$data = array(); // to be returned
foreach ($assessments as $a) {
$id = $a->assessmentid; // just an abbrevation
if (!isset($data[$id])) {
$data[$id] = new stdClass();
$data[$id]->assessmentid = $a->assessmentid;
$data[$id]->weight = $a->assessmentweight;
$data[$id]->reviewerid = $a->reviewerid;
$data[$id]->gradinggrade = $a->gradinggrade;
$data[$id]->submissionid = $a->submissionid;
$data[$id]->dimgrades = array();
}
$data[$id]->dimgrades[$a->dimensionid] = $a->grade;
}
return $gradinggrades;
return $data;
}
/**
* Normalizes the dimension grades to the interval 0.00000 - 100.00000
*
* Note: this heavily relies on PHP5 way of handling references in array of stdClasses. Hopefuly
* it will not change again soon.
*
* @param array $assessments of stdClass as returned by {@see self::prepare_data_from_recordset()}
* @param array $diminfo of stdClass
* @return array of stdClass with the same structure as $assessments
*/
protected function normalize_grades(array $assessments, array $diminfo) {
foreach ($assessments as $asid => $assessment) {
foreach ($assessment->dimgrades as $dimid => $dimgrade) {
$dimmin = $diminfo[$dimid]->min;
$dimmax = $diminfo[$dimid]->max;
$assessment->dimgrades[$dimid] = grade_floatval(($dimgrade - $dimmin) / ($dimmax - $dimmin) * 100);
}
}
return $assessments;
}
/**
* Given a set of a submission's assessments, returns a hypothetical average assessment
*
* The passed structure must be array of assessments objects with ->weight and ->dimgrades properties.
*
* @param array $assessments as prepared by {@link self::prepare_data_from_recordset()}
* @return null|stdClass
*/
protected function average_assessment(array $assessments) {
$sumdimgrades = array();
foreach ($assessments as $a) {
foreach ($a->dimgrades as $dimid => $dimgrade) {
if (!isset($sumdimgrades[$dimid])) {
$sumdimgrades[$dimid] = 0;
}
$sumdimgrades[$dimid] += $dimgrade * $a->weight;
}
}
$sumweights = 0;
foreach ($assessments as $a) {
$sumweights += $a->weight;
}
if ($sumweights == 0) {
// unable to calculate average assessment
return null;
}
$average = new stdClass();
$average->dimgrades = array();
foreach ($sumdimgrades as $dimid => $sumdimgrade) {
$average->dimgrades[$dimid] = grade_floatval($sumdimgrade / $sumweights);
}
return $average;
}
/**
* Given a set of a submission's assessments, returns standard deviations of all their dimensions
*
* The passed structure must be array of assessments objects with at least ->weight
* and ->dimgrades properties. This implementation uses weighted incremental algorithm as
* suggested in "D. H. D. West (1979). Communications of the ACM, 22, 9, 532-535:
* Updating Mean and Variance Estimates: An Improved Method"
* {@link http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Weighted_incremental_algorithm}
*
* @param array $assessments as prepared by {@link self::prepare_data_from_recordset()}
* @return null|array indexed by dimension id
*/
protected function weighted_variance(array $assessments) {
$first = reset($assessments);
if (empty($first)) {
return null;
}
$dimids = array_keys($first->dimgrades);
$asids = array_keys($assessments);
$vars = array(); // to be returned
foreach ($dimids as $dimid) {
$n = 0;
$s = 0;
$sumweight = 0;
foreach ($asids as $asid) {
$x = $assessments[$asid]->dimgrades[$dimid]; // value (data point)
$weight = $assessments[$asid]->weight; // the values's weight
if ($weight == 0) {
continue;
}
if ($n == 0) {
$n = 1;
$mean = $x;
$s = 0;
$sumweight = $weight;
} else {
$n++;
$temp = $weight + $sumweight;
$q = $x - $mean;
$r = $q * $weight / $temp;
$s = $s + $sumweight * $q * $r;
$mean = $mean + $r;
$sumweight = $temp;
}
}
if ($sumweight > 0 and $n > 1) {
// for the sample: $vars[$dimid] = ($s * $n) / (($n - 1) * $sumweight);
// for the population:
$vars[$dimid] = $s / $sumweight;
} else {
$vars[$dimid] = null;
}
}
return $vars;
}
/**
* Measures the distance of the assessment from a referential one
*
* The passed data structures must contain ->dimgrades property. The referential
* assessment is supposed to be close to the average assessment. All dimension grades are supposed to be
* normalized to the interval 0 - 100.
*
* @param stdClass $assessment the assessment being measured
* @param stdClass $referential assessment
* @param array $diminfo of stdClass(->weight ->min ->max ->variance) indexed by dimension id
* @return float|null rounded to 5 valid decimals
*/
protected function assessments_distance(stdClass $assessment, stdClass $referential, array $diminfo) {
$distance = 0;
$n = 0;
foreach (array_keys($assessment->dimgrades) as $dimid) {
$agrade = $assessment->dimgrades[$dimid];
$rgrade = $referential->dimgrades[$dimid];
$var = $diminfo[$dimid]->variance;
$weight = $diminfo[$dimid]->weight;
// variations very close to zero are too sensitive to a small change of data values
if ($var > 0.01 and $agrade != $rgrade) {
$absdelta = abs($agrade - $rgrade);
// todo the following constant is the param. For 1 it is very strict, for 5 it is quite lax
$reldelta = pow($agrade - $rgrade, 2) / (5 * $var);
$distance += $absdelta * $reldelta * $weight;
$n += $weight;
}
}
if ($n > 0) {
// average distance across all dimensions
return grade_floatval($distance / $n);
} else {
return null;
}
}
}
mod/workshop/eval/best/simpletest/testlib.php
+194
View File
@@ -0,0 +1,194 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Unit tests for grading evaluation method "best"
*
* @package mod-workshop
* @copyright 2009 David Mudrak <[email protected]>
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
defined('MOODLE_INTERNAL') || die();
// Include the code to test
require_once($CFG->dirroot . '/mod/workshop/locallib.php');
require_once($CFG->dirroot . '/mod/workshop/eval/best/lib.php');
require_once($CFG->libdir . '/gradelib.php');
global $DB;
Mock::generate(get_class($DB), 'mockDB');
/**
* Test subclass that makes all the protected methods we want to test public.
*/
class testable_workshop_best_evaluation extends workshop_best_evaluation {
public function normalize_grades(array $assessments, array $diminfo) {
return parent::normalize_grades($assessments, $diminfo);
}
public function average_assessment(array $assessments) {
return parent::average_assessment($assessments);
}
public function weighted_variance(array $assessments) {
return parent::weighted_variance($assessments);
}
}
class workshop_best_evaluation_test extends UnitTestCase {
/** real database */
protected $realDB;
/** workshop instance emulation */
protected $workshop;
/** instance of the grading evaluator being tested */
protected $evaluator;
/**
* Setup testing environment
*/
public function setUp() {
global $DB;
$this->realDB = $DB;
$DB = new mockDB();
$cm = new stdClass();
$course = new stdClass();
$context = new stdClass();
$workshop = (object)array('id' => 42, 'evaluation' => 'best');
$this->workshop = new workshop($workshop, $cm, $course, $context);
$this->evaluator = new testable_workshop_best_evaluation($this->workshop);
}
public function tearDown() {
global $DB;
$DB = $this->realDB;
$this->workshop = null;
$this->evaluator = null;
}
public function test_normalize_grades() {
// fixture set-up
$assessments = array();
$assessments[1] = (object)array(
'dimgrades' => array(3 => 1.0000, 4 => 13.42300),
);
$assessments[3] = (object)array(
'dimgrades' => array(3 => 2.0000, 4 => 19.1000),
);
$assessments[7] = (object)array(
'dimgrades' => array(3 => 3.0000, 4 => 0.00000),
);
$diminfo = array(
3 => (object)array('min' => 1, 'max' => 3),
4 => (object)array('min' => 0, 'max' => 20),
);
// excersise SUT
$norm = $this->evaluator->normalize_grades($assessments, $diminfo);
// validate
$this->assertIsA($norm, 'array');
// the following grades from a scale
$this->assertEqual($norm[1]->dimgrades[3], 0);
$this->assertEqual($norm[3]->dimgrades[3], 50);
$this->assertEqual($norm[7]->dimgrades[3], 100);
// the following grades from an interval 0 - 20
$this->assertEqual($norm[1]->dimgrades[4], grade_floatval(13.423 / 20 * 100));
$this->assertEqual($norm[3]->dimgrades[4], grade_floatval(19.1 / 20 * 100));
$this->assertEqual($norm[7]->dimgrades[4], 0);
}
public function test_average_assessment() {
// fixture set-up
$assessments = array();
$assessments[11] = (object)array(
'weight' => 1,
'dimgrades' => array(3 => 10.0, 4 => 13.4, 5 => 95.0),
'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
);
$assessments[13] = (object)array(
'weight' => 3,
'dimgrades' => array(3 => 11.0, 4 => 10.1, 5 => 92.0),
'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
);
$assessments[17] = (object)array(
'weight' => 1,
'dimgrades' => array(3 => 11.0, 4 => 8.1, 5 => 88.0),
'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
);
// excersise SUT
$average = $this->evaluator->average_assessment($assessments);
// validate
$this->assertIsA($average->dimgrades, 'array');
$this->assertEqual(grade_floatval($average->dimgrades[3]), grade_floatval((10.0 + 11.0*3 + 11.0)/5));
$this->assertEqual(grade_floatval($average->dimgrades[4]), grade_floatval((13.4 + 10.1*3 + 8.1)/5));
$this->assertEqual(grade_floatval($average->dimgrades[5]), grade_floatval((95.0 + 92.0*3 + 88.0)/5));
}
public function test_average_assessment_noweight() {
// fixture set-up
$assessments = array();
$assessments[11] = (object)array(
'weight' => 0,
'dimgrades' => array(3 => 10.0, 4 => 13.4, 5 => 95.0),
'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
);
$assessments[17] = (object)array(
'weight' => 0,
'dimgrades' => array(3 => 11.0, 4 => 8.1, 5 => 88.0),
'dimweights' => array(3 => 1, 4 => 1, 5 => 1)
);
// excersise SUT
$average = $this->evaluator->average_assessment($assessments);
// validate
$this->assertNull($average);
}
public function test_weighted_variance() {
// fixture set-up
$assessments[11] = (object)array(
'weight' => 1,
'dimgrades' => array(3 => 11, 4 => 2),
);
$assessments[13] = (object)array(
'weight' => 3,
'dimgrades' => array(3 => 11, 4 => 4),
);
$assessments[17] = (object)array(
'weight' => 2,
'dimgrades' => array(3 => 11, 4 => 5),
);
$assessments[20] = (object)array(
'weight' => 1,
'dimgrades' => array(3 => 11, 4 => 7),
);
$assessments[25] = (object)array(
'weight' => 1,
'dimgrades' => array(3 => 11, 4 => 9),
);
// excersise SUT
$variance = $this->evaluator->weighted_variance($assessments);
// validate
// dimension [3] have all the grades equal to 11
$this->assertEqual($variance[3], 0);
// dimension [4] represents data 2, 4, 4, 4, 5, 5, 7, 9 having stdev=2 (stdev is sqrt of variance)
$this->assertEqual($variance[4], 4);
}
}
mod/workshop/form/accumulative/lib.php
+43 -13
View File
@@ -260,27 +260,25 @@ class workshop_accumulative_strategy implements workshop_strategy {
return false;
}
////////////////////////////////////////////////////////////////////////////////
// Methods needed by 'best' evaluation plugin //
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// Methods required by the 'best' evaluation plugin //
////////////////////////////////////////////////////////////////////////////////
/**
* TODO: short description.
* TODO
*
* @param resource $restrict
* @return TODO
*/
public function get_assessments_recordset($restrict) {
public function eval_best_get_assessments_recordset($restrict) {
global $DB;
$sql = 'SELECT a.id AS assessmentid, a.weight AS assessmentweight, a.reviewerid, a.gradinggrade,
s.id AS submissionid,
g.dimensionid, g.grade,
d.weight AS dimensionweight
$sql = 'SELECT s.id AS submissionid,
a.id AS assessmentid, a.weight AS assessmentweight, a.reviewerid, a.gradinggrade,
g.dimensionid, g.grade
FROM {workshop_submissions} s
JOIN {workshop_assessments} a ON (a.submissionid = s.id)
JOIN {workshop_grades} g ON (g.assessmentid = a.id AND g.strategy = :strategy)
JOIN {workshopform_accumulative} d ON (d.id = g.dimensionid)
WHERE s.example=0 AND s.workshopid=:workshopid'; // to be cont.
$params = array('workshopid' => $this->workshop->id, 'strategy' => $this->workshop->strategy);
@@ -299,9 +297,41 @@ class workshop_accumulative_strategy implements workshop_strategy {
return $DB->get_recordset_sql($sql, $params);
}
////////////////////////////////////////////////////////////////////////////////
// Internal methods //
////////////////////////////////////////////////////////////////////////////////
/**
* TODO: short description.
*
* @return array [dimid] => stdClass (->id ->max ->min ->weight)
*/
public function eval_best_dimensions_info() {
global $DB;
$sql = 'SELECT d.id, d.grade, d.weight, s.scale
FROM {workshopform_accumulative} d
LEFT JOIN {scale} s ON (d.grade < 0 AND -d.grade = s.id)
WHERE d.workshopid = :workshopid';
$params = array('workshopid' => $this->workshop->id);
$dimrecords = $DB->get_records_sql($sql, $params);
$diminfo = array();
foreach ($dimrecords as $dimid => $dimrecord) {
$diminfo[$dimid] = new stdClass();
$diminfo[$dimid]->id = $dimid;
$diminfo[$dimid]->weight = $dimrecord->weight;
if ($dimrecord->grade < 0) {
// the dimension uses a scale
$diminfo[$dimid]->min = 1;
$diminfo[$dimid]->max = count(explode(',', $dimrecord->scale));
} else {
// the dimension uses points
$diminfo[$dimid]->min = 0;
$diminfo[$dimid]->max = grade_floatval($dimrecord->grade);
}
}
return $diminfo;
}
////////////////////////////////////////////////////////////////////////////////
// Internal methods //
////////////////////////////////////////////////////////////////////////////////
/**
* Loads the fields of the assessment form currently used in this workshop
mod/workshop/locallib.php
+13
View File
@@ -1200,6 +1200,19 @@ class workshop {
// todo
}
/**
* Calculates the workshop total grades for the given participant(s)
*
* @param null|int|array $restrict If null, update all reviewers, otherwise update just grades for the given reviewer(s)
* @return void
*/
public function aggregate_total_grades($restrict=null) {
global $DB;
// todo
}
////////////////////////////////////////////////////////////////////////////////
// Internal methods (implementation details) //
////////////////////////////////////////////////////////////////////////////////