L-electron-Rare/moodle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MDL-66498 analytics: Avoid get_in_or_equal for big arrays

Browse Source 
Also, join directly with context table.
This commit is contained in:
David Monllaó
2019-09-11 12:44:55 +08:00
parent 500111c5d3
commit 4fd74fba6f
3 changed files with 43 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
analytics/classes/manager.php
+16 -37
View File
@@ -553,26 +553,14 @@ class manager {
public static function cleanup() {
global $DB;
// Clean up stuff that depends on contexts that do not exist anymore.
$sql = "SELECT DISTINCT ap.contextid FROM {analytics_predictions} ap
LEFT JOIN {context} ctx ON ap.contextid = ctx.id
WHERE ctx.id IS NULL";
$apcontexts = $DB->get_records_sql($sql);
$DB->execute("DELETE FROM {analytics_prediction_actions} WHERE predictionid IN
(SELECT ap.id FROM {analytics_predictions} ap
LEFT JOIN {context} ctx ON ap.contextid = ctx.id
WHERE ctx.id IS NULL)");
$sql = "SELECT DISTINCT aic.contextid FROM {analytics_indicator_calc} aic
LEFT JOIN {context} ctx ON aic.contextid = ctx.id
WHERE ctx.id IS NULL";
$indcalccontexts = $DB->get_records_sql($sql);
$contexts = $apcontexts + $indcalccontexts;
if ($contexts) {
list($sql, $params) = $DB->get_in_or_equal(array_keys($contexts));
$DB->execute("DELETE FROM {analytics_prediction_actions} WHERE predictionid IN
(SELECT ap.id FROM {analytics_predictions} ap WHERE ap.contextid $sql)", $params);
$DB->delete_records_select('analytics_predictions', "contextid $sql", $params);
$DB->delete_records_select('analytics_indicator_calc', "contextid $sql", $params);
}
$contextsql = "SELECT id FROM {context} ctx";
$DB->delete_records_select('analytics_predictions', "contextid NOT IN ($contextsql)");
$DB->delete_records_select('analytics_indicator_calc', "contextid NOT IN ($contextsql)");
// Clean up stuff that depends on analysable ids that do not exist anymore.
@@ -607,28 +595,19 @@ class manager {
$param = ['modelid' => $model->get_id()];
if ($predictsamplesanalysableids) {
$chunks = array_chunk(array_flip($predictsamplesanalysableids), 1000);
foreach ($chunks as $chunk) {
list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_predict_samples', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
list($idssql, $idsparams) = $DB->get_in_or_equal(array_flip($predictsamplesanalysableids), SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_predict_samples', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
if ($trainsamplesanalysableids) {
$chunks = array_chunk(array_flip($trainsamplesanalysableids), 1000);
foreach ($chunks as $chunk) {
list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_train_samples', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
list($idssql, $idsparams) = $DB->get_in_or_equal(array_flip($trainsamplesanalysableids), SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_train_samples', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
if ($usedanalysablesanalysableids) {
$chunks = array_chunk(array_flip($usedanalysablesanalysableids), 1000);
foreach ($chunks as $chunk) {
list($idssql, $idsparams) = $DB->get_in_or_equal($chunk, SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_used_analysables', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
list($idssql, $idsparams) = $DB->get_in_or_equal(array_flip($usedanalysablesanalysableids), SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_used_analysables', "modelid = :modelid AND analysableid $idssql",
$param + $idsparams);
}
}
}
analytics/classes/model.php
+25 -23
View File
@@ -1722,12 +1722,8 @@ class model {
$predictor->clear_model($this->get_unique_id(), $this->get_output_dir());
}
$predictionids = $DB->get_fieldset_select('analytics_predictions', 'id', 'modelid = :modelid',
array('modelid' => $this->get_id()));
if ($predictionids) {
list($sql, $params) = $DB->get_in_or_equal($predictionids);
$DB->delete_records_select('analytics_prediction_actions', "predictionid $sql", $params);
}
$DB->delete_records_select('analytics_prediction_actions', "predictionid IN
(SELECT id FROM {analytics_predictions} WHERE modelid = :modelid)", ['modelid' => $this->get_id()]);
$DB->delete_records('analytics_predictions', array('modelid' => $this->model->id));
$DB->delete_records('analytics_predict_samples', array('modelid' => $this->model->id));
@@ -1833,25 +1829,31 @@ class model {
$contextids = array_map(function($predictionobj) {
return $predictionobj->contextid;
}, $predictionrecords);
list($contextsql, $contextparams) = $DB->get_in_or_equal($contextids, SQL_PARAMS_NAMED);
// We select the fields that will allow us to map ids to $predictionrecords. Given that we already filter by modelid
// we have enough with sampleid and rangeindex. The reason is that the sampleid relation to a site is N - 1.
$fields = 'id, sampleid, rangeindex';
// Limited to 30000 records as a middle point between the ~65000 params limit in pgsql and the size limit for mysql which
// can be increased if required up to a reasonable point.
$chunks = array_chunk($contextids, 30000);
foreach ($chunks as $contextidschunk) {
list($contextsql, $contextparams) = $DB->get_in_or_equal($contextidschunk, SQL_PARAMS_NAMED);
// We include the contextid and the timecreated filter to reduce the number of records in $dbpredictions. We can not
// add as many OR conditions as records in $predictionrecords.
$sql = "SELECT $fields
FROM {analytics_predictions}
WHERE modelid = :modelid
AND contextid $contextsql
AND timecreated >= :firsttimecreated";
$params = $contextparams + ['modelid' => $this->model->id, 'firsttimecreated' => $firstprediction->timecreated];
$dbpredictions = $DB->get_recordset_sql($sql, $params);
foreach ($dbpredictions as $id => $dbprediction) {
// The append_rangeindex implementation is the same regardless of the time splitting method in use.
$uniqueid = $this->get_time_splitting()->append_rangeindex($dbprediction->sampleid, $dbprediction->rangeindex);
$predictionrecords[$uniqueid]->id = $dbprediction->id;
// We select the fields that will allow us to map ids to $predictionrecords. Given that we already filter by modelid
// we have enough with sampleid and rangeindex. The reason is that the sampleid relation to a site is N - 1.
$fields = 'id, sampleid, rangeindex';
// We include the contextid and the timecreated filter to reduce the number of records in $dbpredictions. We can not
// add as many OR conditions as records in $predictionrecords.
$sql = "SELECT $fields
FROM {analytics_predictions}
WHERE modelid = :modelid
AND contextid $contextsql
AND timecreated >= :firsttimecreated";
$params = $contextparams + ['modelid' => $this->model->id, 'firsttimecreated' => $firstprediction->timecreated];
$dbpredictions = $DB->get_recordset_sql($sql, $params);
foreach ($dbpredictions as $id => $dbprediction) {
// The append_rangeindex implementation is the same regardless of the time splitting method in use.
$uniqueid = $this->get_time_splitting()->append_rangeindex($dbprediction->sampleid, $dbprediction->rangeindex);
$predictionrecords[$uniqueid]->id = $dbprediction->id;
}
}
return $predictionrecords;
analytics/classes/privacy/provider.php
+2 -6
View File
@@ -306,13 +306,9 @@ class provider implements
$idssql = "SELECT ap.id FROM {analytics_predictions} ap
WHERE ap.contextid = :contextid AND ap.modelid = :modelid";
$idsparams = ['contextid' => $context->id, 'modelid' => $modelid];
$predictionids = $DB->get_fieldset_sql($idssql, $idsparams);
if ($predictionids) {
list($predictionidssql, $params) = $DB->get_in_or_equal($predictionids, SQL_PARAMS_NAMED);
$DB->delete_records_select('analytics_prediction_actions', "predictionid IN ($idssql)", $idsparams);
$DB->delete_records_select('analytics_predictions', "id $predictionidssql", $params);
}
$DB->delete_records_select('analytics_prediction_actions', "predictionid IN ($idssql)", $idsparams);
$DB->delete_records_select('analytics_predictions', "contextid = :contextid AND modelid = :modelid", $idsparams);
}
// We delete them all this table is just a cache and we don't know which model filled it.