MDL-78030 dml: get total count records using the window function

The COUNT window function can be helpful to fasten the complex query,
but it's not recommended to be used for a single table query.

The patch is specifically for PostgreSQL, MariaDB and Oracle databases as they have better performance
compared to MySQL and MSSQL. Although these databases support the COUNT window functions,
the results indicate that the query runs slower when using them.

.upgradenotes/MDL-78030-2024071102213324.yml

@@ -0,0 +1,18 @@
+issueNumber: MDL-78030
+notes:
+  core:
+    - message: |
+        Two new functions have been introduced in the \moodle_database class:
+        - `get_counted_records_sql()`
+        - `get_counted_recordset_sql()`
+        These methods are compatible with all databases.
+        They will check the current running database engine and apply the COUNT window function if it is supported,
+        otherwise, they will use the usual COUNT function.
+
+        The COUNT window function optimization is applied to the following databases:
+        - PostgreSQL
+        - MariaDB
+        - Oracle
+        MySQL and SQL Server do not use this optimization due to insignificant performance differences before and
+        after the improvement.
+      type: improved

lib/dml/mariadb_native_moodle_database.php

@@ -113,4 +113,13 @@ class mariadb_native_moodle_database extends mysqli_native_moodle_database {
         }
         return false;
     }
+
+    /**
+     * MariaDB supports the COUNT() window function and provides a performance improvement.
+     *
+     * @return bool
+     */
+    public function is_count_window_function_supported(): bool {
+        return true;
+    }
 }

lib/dml/moodle_database.php

@@ -2925,4 +2925,97 @@ abstract class moodle_database {
         // No support unless specified.
         return false;
     }
+
+    /**
+     * Whether the database is able to support the COUNT() window function and provides a performance improvement.
+     *
+     * @return bool
+     */
+    public function is_count_window_function_supported(): bool {
+        // No support unless specified.
+        return false;
+    }
+
+    /**
+     * Retrieve records with a select query and count the total number of records.
+     *
+     * @param string $sql The query string.
+     * @param string $fullcountcolumn The column name used for counting total records.
+     * @param string $sort (Optional) Sorting criteria for the records.
+     *                      The reason to separate $sort from $sql are:
+     *                      1. The $sort needs to be placed outside the full count subquery
+     *                         in order to function properly in MariaDB.
+     *                      2. For unsupported databases, it is not allowed to run a query to get the total with the $sort.
+     *                         Please refer to the {@see ::generate_fullcount_sql()} for details.
+     * @param array|null $params (Optional) Parameters to bind with the query.
+     * @param int $limitfrom (Optional) Offset for pagination.
+     * @param int $limitnum (Optional) Limit for pagination.
+     * @return array Fetched records.
+     */
+    public function get_counted_records_sql(
+        string $sql,
+        string $fullcountcolumn,
+        string $sort = '',
+        ?array $params = null,
+        int $limitfrom = 0,
+        int $limitnum = 0,
+    ): array {
+        $fullcountsql = $this->generate_fullcount_sql($sql, $params, $fullcountcolumn);
+        if ($sort) {
+            $fullcountsql .= " ORDER BY " . $sort;
+        }
+        return $this->get_records_sql($fullcountsql, $params, $limitfrom, $limitnum);
+    }
+
+    /**
+     * Retrieve a recordset with a select query and count the total number of records.
+     *
+     * @param string $sql The query string.
+     * @param string $fullcountcolumn The column name used for counting total records.
+     * @param string $sort (Optional) Sorting criteria for the records.
+     *                      The reason to separate $sort from $sql are:
+     *                      1. The $sort needs to be placed outside the full count subquery
+     *                         in order to function properly in MariaDB.
+     *                      2. For unsupported databases, it is not allowed to run a query to get the total with the $sort.
+     *                         Please refer to the {@see ::generate_fullcount_sql()} for details.
+     * @param array|null $params (Optional) Parameters to bind with the query.
+     * @param int $limitfrom (Optional) Offset for pagination.
+     * @param int $limitnum (Optional) Limit for pagination.
+     * @return moodle_recordset A moodle_recordset instance..
+     */
+    public function get_counted_recordset_sql(
+        string $sql,
+        string $fullcountcolumn,
+        string $sort = '',
+        ?array $params = null,
+        int $limitfrom = 0,
+        int $limitnum = 0,
+    ): moodle_recordset {
+        $fullcountsql = $this->generate_fullcount_sql($sql, $params, $fullcountcolumn);
+        if ($sort) {
+            $fullcountsql .= " ORDER BY " . $sort;
+        }
+        return $this->get_recordset_sql($fullcountsql, $params, $limitfrom, $limitnum);
+    }
+
+    /**
+     * Helper function to generate window COUNT() aggregate function to the SQL query.
+     *
+     * @param string $sql The SQL select query to execute.
+     * @param array|null $params array of sql parameters
+     * @param string $fullcountcolumn An alias column name for the window function results.
+     * @return string The generated query.
+     */
+    private function generate_fullcount_sql(
+        string $sql,
+        ?array $params,
+        string $fullcountcolumn,
+    ): string {
+        $fullcountvalue = "COUNT(1) OVER()";
+        if (!$this->is_count_window_function_supported()) {
+            $sqlcount = "SELECT COUNT(1) FROM ($sql) results";
+            $fullcountvalue = $this->count_records_sql($sqlcount, $params);
+        }
+        return "SELECT results.*, $fullcountvalue AS $fullcountcolumn FROM ($sql) results";
+    }
 }

lib/dml/oci_native_moodle_database.php

@@ -1874,4 +1874,13 @@ class oci_native_moodle_database extends moodle_database {
         $this->commit_status = OCI_COMMIT_ON_SUCCESS;
         $this->query_end($result);
     }
+
+    /**
+     * Oracle supports the COUNT() window function and provides a performance improvement.
+     *
+     * @return bool
+     */
+    public function is_count_window_function_supported(): bool {
+        return true;
+    }
 }

lib/dml/pgsql_native_moodle_database.php

@@ -1722,4 +1722,13 @@ class pgsql_native_moodle_database extends moodle_database {
     public function is_fulltext_search_supported() {
         return true;
     }
+
+    /**
+     * Postgresql supports the COUNT() window function and provides a performance improvement.
+     *
+     * @return bool
+     */
+    public function is_count_window_function_supported(): bool {
+        return true;
+    }
 }

lib/dml/tests/dml_test.php

@@ -6432,6 +6432,62 @@ EOD;
             "Found invalid DB server version format when reading version from DB: '{$version}' ({$description}).");
         $db2->dispose();
     }
+
+    /**
+     * Test the COUNT() window function with the actual DB Server.
+     *
+     * @covers \moodle_database::get_counted_recordset_sql()
+     * @covers \moodle_database::get_counted_records_sql()
+     * @covers \moodle_database::generate_fullcount_sql()
+     * @return void
+     */
+    public function test_count_window_function(): void {
+        $DB = $this->tdb;
+        $dbman = $DB->get_manager();
+
+        $table = $this->get_test_table();
+        $tablename = $table->getName();
+
+        $table->add_field('id', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
+        $table->add_field('course', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, null, '0');
+        $table->add_key('primary', XMLDB_KEY_PRIMARY, ['id']);
+        $dbman->create_table($table);
+
+        for ($i = 1; $i <= 5; $i++) {
+            $DB->insert_record($tablename, ['course' => $i], false);
+        }
+
+        // Test with the get_recordset_select().
+        $rs = $DB->get_counted_recordset_sql(
+            sql: "SELECT * FROM {{$tablename}}",
+            fullcountcolumn: 'fullcount',
+            sort: "course DESC",
+            limitfrom: 1,
+            limitnum: 3,
+        );
+        // Check whether the fullcount column returns the correct number.
+        $this->assertEquals(5, $rs->current()->fullcount);
+        // Check whether the `limitfrom` works properly.
+        $this->assertEquals(4, $rs->current()->course);
+        // Check whether the 'limitnum' works properly.
+        $this->assertEquals(3, iterator_count($rs));
+
+        // Test with the get_records_select().
+        $rs = $DB->get_counted_records_sql(
+            sql: "SELECT * FROM {{$tablename}}",
+            fullcountcolumn: 'fullcount',
+            sort: "course DESC",
+            limitfrom: 3,
+            limitnum: 2,
+        );
+        $resetrs = reset($rs);
+        // Check whether the fullcount column returns the correct number.
+        $this->assertEquals(5, $resetrs->fullcount);
+        // Check whether the 'limitfrom' works properly.
+        $this->assertEquals(2, $resetrs->course);
+        // Check whether the 'limitnum' works properly.
+        $this->assertEquals(2, count($rs));
+    }
 }
 
 /**