diff --git a/analytics/classes/model.php b/analytics/classes/model.php index 421e48c1996..e5ca153c6d4 100644 --- a/analytics/classes/model.php +++ b/analytics/classes/model.php @@ -1030,7 +1030,7 @@ class model { } // Get all samples data. - list($sampleids, $samplesdata) = $this->get_analyser()->get_samples($sampleids); + list($sampleids, $samplesdata) = $this->get_samples($sampleids); // Calculate the targets. $predictions = array(); @@ -1344,7 +1344,7 @@ class model { return $prediction->sampleid; }, $predictions); - list($unused, $samplesdata) = $this->get_analyser()->get_samples($sampleids); + list($unused, $samplesdata) = $this->get_samples($sampleids); $current = 0; @@ -1410,7 +1410,7 @@ class model { */ public function prediction_sample_data($predictionobj) { - list($unused, $samplesdata) = $this->get_analyser()->get_samples(array($predictionobj->sampleid)); + list($unused, $samplesdata) = $this->get_samples(array($predictionobj->sampleid)); if (empty($samplesdata[$predictionobj->sampleid])) { throw new \moodle_exception('errorsamplenotavailable', 'analytics'); @@ -1859,6 +1859,64 @@ class model { return $predictionrecords; } + /** + * Wrapper around analyser's get_samples to skip DB's max-number-of-params exception. + * + * @param array $sampleids + * @return array + */ + public function get_samples(array $sampleids): array { + + if (empty($sampleids)) { + throw new \coding_exception('No sample ids provided'); + } + + $chunksize = count($sampleids); + + // We start with just 1 chunk, if it is too large for the db we split the list of sampleids in 2 and we + // try again. We repeat this process until the chunk is small enough for the db engine to process. The + // >= has been added in case there are other \dml_read_exceptions unrelated to the max number of params. + while (empty($done) && $chunksize >= 1) { + + $chunks = array_chunk($sampleids, $chunksize); + $allsampleids = []; + $allsamplesdata = []; + + foreach ($chunks as $index => $chunk) { + + try { + list($chunksampleids, $chunksamplesdata) = $this->get_analyser()->get_samples($chunk); + } catch (\dml_read_exception $e) { + + // Reduce the chunksize, we use floor() so the $chunksize is always less than the previous $chunksize value. + $chunksize = floor($chunksize / 2); + break; + } + + // We can sum as these two arrays are indexed by sampleid and there are no collisions. + $allsampleids = $allsampleids + $chunksampleids; + $allsamplesdata = $allsamplesdata + $chunksamplesdata; + + if ($index === count($chunks) - 1) { + // We successfully processed all the samples in all chunks, we are done. + $done = true; + } + } + } + + if (empty($done)) { + if (!empty($e)) { + // Throw the last exception we caught, the \dml_read_exception we have been catching is unrelated to the max number + // of param's exception. + throw new \dml_read_exception($e); + } else { + throw new \coding_exception('We should never reach this point, there is a bug in ' . + 'core_analytics\\model::get_samples\'s code'); + } + } + return [$allsampleids, $allsamplesdata]; + } + /** * Purges the insights cache. */ diff --git a/analytics/tests/model_test.php b/analytics/tests/model_test.php index 8e82a47c225..55596511a79 100644 --- a/analytics/tests/model_test.php +++ b/analytics/tests/model_test.php @@ -505,6 +505,61 @@ class analytics_model_testcase extends advanced_testcase { $this->assertArrayHasKey('\core\analytics\time_splitting\quarters', $this->model->get_potential_timesplittings()); } + /** + * Tests model::get_samples() + * + * @return null + */ + public function test_get_samples() { + $this->resetAfterTest(); + + if (!PHPUNIT_LONGTEST) { + $this->markTestSkipped('PHPUNIT_LONGTEST is not defined'); + } + + // 10000 should be enough to make oracle and mssql fail, if we want pgsql to fail we need around 70000 + // users, that is a few minutes just to create the users. + $nusers = 10000; + + $userids = []; + for ($i = 0; $i < $nusers; $i++) { + $user = $this->getDataGenerator()->create_user(); + $userids[] = $user->id; + } + + $upcomingactivities = null; + foreach (\core_analytics\manager::get_all_models() as $model) { + if (get_class($model->get_target()) === 'core_user\\analytics\\target\\upcoming_activities_due') { + $upcomingactivities = $model; + } + } + + list($sampleids, $samplesdata) = $upcomingactivities->get_samples($userids); + $this->assertCount($nusers, $sampleids); + $this->assertCount($nusers, $samplesdata); + + $subset = array_slice($userids, 0, 100); + list($sampleids, $samplesdata) = $upcomingactivities->get_samples($subset); + $this->assertCount(100, $sampleids); + $this->assertCount(100, $samplesdata); + + $subset = array_slice($userids, 0, 2); + list($sampleids, $samplesdata) = $upcomingactivities->get_samples($subset); + $this->assertCount(2, $sampleids); + $this->assertCount(2, $samplesdata); + + $subset = array_slice($userids, 0, 1); + list($sampleids, $samplesdata) = $upcomingactivities->get_samples($subset); + $this->assertCount(1, $sampleids); + $this->assertCount(1, $samplesdata); + + // Unexisting, so nothing returned, but still 2 arrays. + list($sampleids, $samplesdata) = $upcomingactivities->get_samples([1231231231231231]); + $this->assertEmpty($sampleids); + $this->assertEmpty($samplesdata); + + } + /** * Generates a model log record. */