From 42664ee49af459ce4c67728a10b087ef2278efbd Mon Sep 17 00:00:00 2001 From: meirzamoodle Date: Thu, 16 May 2024 11:19:18 +0700 Subject: [PATCH] MDL-78030 dml: get total count records using the window function The COUNT window function can be helpful to fasten the complex query, but it's not recommended to be used for a single table query. The patch is specifically for PostgreSQL, MariaDB and Oracle databases as they have better performance compared to MySQL and MSSQL. Although these databases support the COUNT window functions, the results indicate that the query runs slower when using them. --- .upgradenotes/MDL-78030-2024071102213324.yml | 18 ++++ lib/dml/mariadb_native_moodle_database.php | 9 ++ lib/dml/moodle_database.php | 93 ++++++++++++++++++++ lib/dml/oci_native_moodle_database.php | 9 ++ lib/dml/pgsql_native_moodle_database.php | 9 ++ lib/dml/tests/dml_test.php | 56 ++++++++++++ 6 files changed, 194 insertions(+) create mode 100644 .upgradenotes/MDL-78030-2024071102213324.yml diff --git a/.upgradenotes/MDL-78030-2024071102213324.yml b/.upgradenotes/MDL-78030-2024071102213324.yml new file mode 100644 index 00000000000..68a926a2623 --- /dev/null +++ b/.upgradenotes/MDL-78030-2024071102213324.yml @@ -0,0 +1,18 @@ +issueNumber: MDL-78030 +notes: + core: + - message: | + Two new functions have been introduced in the \moodle_database class: + - `get_counted_records_sql()` + - `get_counted_recordset_sql()` + These methods are compatible with all databases. + They will check the current running database engine and apply the COUNT window function if it is supported, + otherwise, they will use the usual COUNT function. + + The COUNT window function optimization is applied to the following databases: + - PostgreSQL + - MariaDB + - Oracle + MySQL and SQL Server do not use this optimization due to insignificant performance differences before and + after the improvement. + type: improved diff --git a/lib/dml/mariadb_native_moodle_database.php b/lib/dml/mariadb_native_moodle_database.php index 4b94f220b17..66d24d09ebd 100644 --- a/lib/dml/mariadb_native_moodle_database.php +++ b/lib/dml/mariadb_native_moodle_database.php @@ -113,4 +113,13 @@ class mariadb_native_moodle_database extends mysqli_native_moodle_database { } return false; } + + /** + * MariaDB supports the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + return true; + } } diff --git a/lib/dml/moodle_database.php b/lib/dml/moodle_database.php index 8ff9c802854..648cdd22039 100644 --- a/lib/dml/moodle_database.php +++ b/lib/dml/moodle_database.php @@ -2925,4 +2925,97 @@ abstract class moodle_database { // No support unless specified. return false; } + + /** + * Whether the database is able to support the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + // No support unless specified. + return false; + } + + /** + * Retrieve records with a select query and count the total number of records. + * + * @param string $sql The query string. + * @param string $fullcountcolumn The column name used for counting total records. + * @param string $sort (Optional) Sorting criteria for the records. + * The reason to separate $sort from $sql are: + * 1. The $sort needs to be placed outside the full count subquery + * in order to function properly in MariaDB. + * 2. For unsupported databases, it is not allowed to run a query to get the total with the $sort. + * Please refer to the {@see ::generate_fullcount_sql()} for details. + * @param array|null $params (Optional) Parameters to bind with the query. + * @param int $limitfrom (Optional) Offset for pagination. + * @param int $limitnum (Optional) Limit for pagination. + * @return array Fetched records. + */ + public function get_counted_records_sql( + string $sql, + string $fullcountcolumn, + string $sort = '', + ?array $params = null, + int $limitfrom = 0, + int $limitnum = 0, + ): array { + $fullcountsql = $this->generate_fullcount_sql($sql, $params, $fullcountcolumn); + if ($sort) { + $fullcountsql .= " ORDER BY " . $sort; + } + return $this->get_records_sql($fullcountsql, $params, $limitfrom, $limitnum); + } + + /** + * Retrieve a recordset with a select query and count the total number of records. + * + * @param string $sql The query string. + * @param string $fullcountcolumn The column name used for counting total records. + * @param string $sort (Optional) Sorting criteria for the records. + * The reason to separate $sort from $sql are: + * 1. The $sort needs to be placed outside the full count subquery + * in order to function properly in MariaDB. + * 2. For unsupported databases, it is not allowed to run a query to get the total with the $sort. + * Please refer to the {@see ::generate_fullcount_sql()} for details. + * @param array|null $params (Optional) Parameters to bind with the query. + * @param int $limitfrom (Optional) Offset for pagination. + * @param int $limitnum (Optional) Limit for pagination. + * @return moodle_recordset A moodle_recordset instance.. + */ + public function get_counted_recordset_sql( + string $sql, + string $fullcountcolumn, + string $sort = '', + ?array $params = null, + int $limitfrom = 0, + int $limitnum = 0, + ): moodle_recordset { + $fullcountsql = $this->generate_fullcount_sql($sql, $params, $fullcountcolumn); + if ($sort) { + $fullcountsql .= " ORDER BY " . $sort; + } + return $this->get_recordset_sql($fullcountsql, $params, $limitfrom, $limitnum); + } + + /** + * Helper function to generate window COUNT() aggregate function to the SQL query. + * + * @param string $sql The SQL select query to execute. + * @param array|null $params array of sql parameters + * @param string $fullcountcolumn An alias column name for the window function results. + * @return string The generated query. + */ + private function generate_fullcount_sql( + string $sql, + ?array $params, + string $fullcountcolumn, + ): string { + $fullcountvalue = "COUNT(1) OVER()"; + if (!$this->is_count_window_function_supported()) { + $sqlcount = "SELECT COUNT(1) FROM ($sql) results"; + $fullcountvalue = $this->count_records_sql($sqlcount, $params); + } + return "SELECT results.*, $fullcountvalue AS $fullcountcolumn FROM ($sql) results"; + } } diff --git a/lib/dml/oci_native_moodle_database.php b/lib/dml/oci_native_moodle_database.php index f2382382489..65046bcea09 100644 --- a/lib/dml/oci_native_moodle_database.php +++ b/lib/dml/oci_native_moodle_database.php @@ -1874,4 +1874,13 @@ class oci_native_moodle_database extends moodle_database { $this->commit_status = OCI_COMMIT_ON_SUCCESS; $this->query_end($result); } + + /** + * Oracle supports the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + return true; + } } diff --git a/lib/dml/pgsql_native_moodle_database.php b/lib/dml/pgsql_native_moodle_database.php index 97456d2ba6d..974c7108b3d 100644 --- a/lib/dml/pgsql_native_moodle_database.php +++ b/lib/dml/pgsql_native_moodle_database.php @@ -1722,4 +1722,13 @@ class pgsql_native_moodle_database extends moodle_database { public function is_fulltext_search_supported() { return true; } + + /** + * Postgresql supports the COUNT() window function and provides a performance improvement. + * + * @return bool + */ + public function is_count_window_function_supported(): bool { + return true; + } } diff --git a/lib/dml/tests/dml_test.php b/lib/dml/tests/dml_test.php index 33383df8aa2..31a753f9aa6 100644 --- a/lib/dml/tests/dml_test.php +++ b/lib/dml/tests/dml_test.php @@ -6432,6 +6432,62 @@ EOD; "Found invalid DB server version format when reading version from DB: '{$version}' ({$description})."); $db2->dispose(); } + + /** + * Test the COUNT() window function with the actual DB Server. + * + * @covers \moodle_database::get_counted_recordset_sql() + * @covers \moodle_database::get_counted_records_sql() + * @covers \moodle_database::generate_fullcount_sql() + * @return void + */ + public function test_count_window_function(): void { + $DB = $this->tdb; + $dbman = $DB->get_manager(); + + $table = $this->get_test_table(); + $tablename = $table->getName(); + + $table->add_field('id', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('course', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, null, '0'); + $table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']); + $dbman->create_table($table); + + for ($i = 1; $i <= 5; $i++) { + $DB->insert_record($tablename, ['course' => $i], false); + } + + // Test with the get_recordset_select(). + $rs = $DB->get_counted_recordset_sql( + sql: "SELECT * FROM {{$tablename}}", + fullcountcolumn: 'fullcount', + sort: "course DESC", + limitfrom: 1, + limitnum: 3, + ); + // Check whether the fullcount column returns the correct number. + $this->assertEquals(5, $rs->current()->fullcount); + // Check whether the `limitfrom` works properly. + $this->assertEquals(4, $rs->current()->course); + // Check whether the 'limitnum' works properly. + $this->assertEquals(3, iterator_count($rs)); + + // Test with the get_records_select(). + $rs = $DB->get_counted_records_sql( + sql: "SELECT * FROM {{$tablename}}", + fullcountcolumn: 'fullcount', + sort: "course DESC", + limitfrom: 3, + limitnum: 2, + ); + $resetrs = reset($rs); + // Check whether the fullcount column returns the correct number. + $this->assertEquals(5, $resetrs->fullcount); + // Check whether the 'limitfrom' works properly. + $this->assertEquals(2, $resetrs->course); + // Check whether the 'limitnum' works properly. + $this->assertEquals(2, count($rs)); + } } /**