diff --git a/.upgradenotes/MDL-78030-2024071102213324.yml b/.upgradenotes/MDL-78030-2024071102213324.yml new file mode 100644 index 00000000000..68a926a2623 --- /dev/null +++ b/.upgradenotes/MDL-78030-2024071102213324.yml @@ -0,0 +1,18 @@ +issueNumber: MDL-78030 +notes: + core: + - message: | + Two new functions have been introduced in the \moodle_database class: + - `get_counted_records_sql()` + - `get_counted_recordset_sql()` + These methods are compatible with all databases. + They will check the current running database engine and apply the COUNT window function if it is supported, + otherwise, they will use the usual COUNT function. + + The COUNT window function optimization is applied to the following databases: + - PostgreSQL + - MariaDB + - Oracle + MySQL and SQL Server do not use this optimization due to insignificant performance differences before and + after the improvement. + type: improved diff --git a/lib/dml/mariadb_native_moodle_database.php b/lib/dml/mariadb_native_moodle_database.php index 4b94f220b17..66d24d09ebd 100644 --- a/lib/dml/mariadb_native_moodle_database.php +++ b/lib/dml/mariadb_native_moodle_database.php @@ -113,4 +113,13 @@ class mariadb_native_moodle_database extends mysqli_native_moodle_database { } return false; } + + /** + * MariaDB supports the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + return true; + } } diff --git a/lib/dml/moodle_database.php b/lib/dml/moodle_database.php index 8ff9c802854..648cdd22039 100644 --- a/lib/dml/moodle_database.php +++ b/lib/dml/moodle_database.php @@ -2925,4 +2925,97 @@ abstract class moodle_database { // No support unless specified. return false; } + + /** + * Whether the database is able to support the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + // No support unless specified. + return false; + } + + /** + * Retrieve records with a select query and count the total number of records. + * + * @param string $sql The query string. + * @param string $fullcountcolumn The column name used for counting total records. + * @param string $sort (Optional) Sorting criteria for the records. + * The reason to separate $sort from $sql are: + * 1. The $sort needs to be placed outside the full count subquery + * in order to function properly in MariaDB. + * 2. For unsupported databases, it is not allowed to run a query to get the total with the $sort. + * Please refer to the {@see ::generate_fullcount_sql()} for details. + * @param array|null $params (Optional) Parameters to bind with the query. + * @param int $limitfrom (Optional) Offset for pagination. + * @param int $limitnum (Optional) Limit for pagination. + * @return array Fetched records. + */ + public function get_counted_records_sql( + string $sql, + string $fullcountcolumn, + string $sort = '', + ?array $params = null, + int $limitfrom = 0, + int $limitnum = 0, + ): array { + $fullcountsql = $this->generate_fullcount_sql($sql, $params, $fullcountcolumn); + if ($sort) { + $fullcountsql .= " ORDER BY " . $sort; + } + return $this->get_records_sql($fullcountsql, $params, $limitfrom, $limitnum); + } + + /** + * Retrieve a recordset with a select query and count the total number of records. + * + * @param string $sql The query string. + * @param string $fullcountcolumn The column name used for counting total records. + * @param string $sort (Optional) Sorting criteria for the records. + * The reason to separate $sort from $sql are: + * 1. The $sort needs to be placed outside the full count subquery + * in order to function properly in MariaDB. + * 2. For unsupported databases, it is not allowed to run a query to get the total with the $sort. + * Please refer to the {@see ::generate_fullcount_sql()} for details. + * @param array|null $params (Optional) Parameters to bind with the query. + * @param int $limitfrom (Optional) Offset for pagination. + * @param int $limitnum (Optional) Limit for pagination. + * @return moodle_recordset A moodle_recordset instance.. + */ + public function get_counted_recordset_sql( + string $sql, + string $fullcountcolumn, + string $sort = '', + ?array $params = null, + int $limitfrom = 0, + int $limitnum = 0, + ): moodle_recordset { + $fullcountsql = $this->generate_fullcount_sql($sql, $params, $fullcountcolumn); + if ($sort) { + $fullcountsql .= " ORDER BY " . $sort; + } + return $this->get_recordset_sql($fullcountsql, $params, $limitfrom, $limitnum); + } + + /** + * Helper function to generate window COUNT() aggregate function to the SQL query. + * + * @param string $sql The SQL select query to execute. + * @param array|null $params array of sql parameters + * @param string $fullcountcolumn An alias column name for the window function results. + * @return string The generated query. + */ + private function generate_fullcount_sql( + string $sql, + ?array $params, + string $fullcountcolumn, + ): string { + $fullcountvalue = "COUNT(1) OVER()"; + if (!$this->is_count_window_function_supported()) { + $sqlcount = "SELECT COUNT(1) FROM ($sql) results"; + $fullcountvalue = $this->count_records_sql($sqlcount, $params); + } + return "SELECT results.*, $fullcountvalue AS $fullcountcolumn FROM ($sql) results"; + } } diff --git a/lib/dml/oci_native_moodle_database.php b/lib/dml/oci_native_moodle_database.php index f2382382489..65046bcea09 100644 --- a/lib/dml/oci_native_moodle_database.php +++ b/lib/dml/oci_native_moodle_database.php @@ -1874,4 +1874,13 @@ class oci_native_moodle_database extends moodle_database { $this->commit_status = OCI_COMMIT_ON_SUCCESS; $this->query_end($result); } + + /** + * Oracle supports the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + return true; + } } diff --git a/lib/dml/pgsql_native_moodle_database.php b/lib/dml/pgsql_native_moodle_database.php index 97456d2ba6d..974c7108b3d 100644 --- a/lib/dml/pgsql_native_moodle_database.php +++ b/lib/dml/pgsql_native_moodle_database.php @@ -1722,4 +1722,13 @@ class pgsql_native_moodle_database extends moodle_database { public function is_fulltext_search_supported() { return true; } + + /** + * Postgresql supports the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + return true; + } } diff --git a/lib/dml/tests/dml_test.php b/lib/dml/tests/dml_test.php index 33383df8aa2..31a753f9aa6 100644 --- a/lib/dml/tests/dml_test.php +++ b/lib/dml/tests/dml_test.php @@ -6432,6 +6432,62 @@ EOD; "Found invalid DB server version format when reading version from DB: '{$version}' ({$description})."); $db2->dispose(); } + + /** + * Test the COUNT() window function with the actual DB Server. + * + * @covers \moodle_database::get_counted_recordset_sql() + * @covers \moodle_database::get_counted_records_sql() + * @covers \moodle_database::generate_fullcount_sql() + * @return void + */ + public function test_count_window_function(): void { + $DB = $this->tdb; + $dbman = $DB->get_manager(); + + $table = $this->get_test_table(); + $tablename = $table->getName(); + + $table->add_field('id', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('course', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, null, '0'); + $table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']); + $dbman->create_table($table); + + for ($i = 1; $i <= 5; $i++) { + $DB->insert_record($tablename, ['course' => $i], false); + } + + // Test with the get_recordset_select(). + $rs = $DB->get_counted_recordset_sql( + sql: "SELECT * FROM {{$tablename}}", + fullcountcolumn: 'fullcount', + sort: "course DESC", + limitfrom: 1, + limitnum: 3, + ); + // Check whether the fullcount column returns the correct number. + $this->assertEquals(5, $rs->current()->fullcount); + // Check whether the `limitfrom` works properly. + $this->assertEquals(4, $rs->current()->course); + // Check whether the 'limitnum' works properly. + $this->assertEquals(3, iterator_count($rs)); + + // Test with the get_records_select(). + $rs = $DB->get_counted_records_sql( + sql: "SELECT * FROM {{$tablename}}", + fullcountcolumn: 'fullcount', + sort: "course DESC", + limitfrom: 3, + limitnum: 2, + ); + $resetrs = reset($rs); + // Check whether the fullcount column returns the correct number. + $this->assertEquals(5, $resetrs->fullcount); + // Check whether the 'limitfrom' works properly. + $this->assertEquals(2, $resetrs->course); + // Check whether the 'limitnum' works properly. + $this->assertEquals(2, count($rs)); + } } /**