From 478861b4d15b3efc247e493367fb7d4a10322629 Mon Sep 17 00:00:00 2001 From: Paul Holden Date: Tue, 7 Mar 2023 16:48:51 +0000 Subject: [PATCH 1/2] MDL-77555 reportbuilder: method to ensure unique parameters in SQL. --- .../classes/local/helpers/database.php | 23 ++++++++++++++++ reportbuilder/classes/local/report/column.php | 12 ++++----- .../tests/local/helpers/database_test.php | 27 +++++++++++++++++++ .../tests/local/report/column_test.php | 23 ++++++++++++---- reportbuilder/upgrade.txt | 5 ++++ 5 files changed, 79 insertions(+), 11 deletions(-) diff --git a/reportbuilder/classes/local/helpers/database.php b/reportbuilder/classes/local/helpers/database.php index 9bb3d082feb..ba1fe977599 100644 --- a/reportbuilder/classes/local/helpers/database.php +++ b/reportbuilder/classes/local/helpers/database.php @@ -76,6 +76,29 @@ class database { return true; } + /** + * Replace parameter names within given SQL expression, allowing caller to specify callback to handle their replacement + * primarily to ensure uniqueness when the expression is to be used as part of a larger query + * + * @param string $sql + * @param array $params + * @param callable $callback Method that takes a single string parameter, and returns another string + * @return string + */ + public static function sql_replace_parameter_names(string $sql, array $params, callable $callback): string { + foreach ($params as $param) { + + // Pattern to look for param within the SQL. + $pattern = '/:(?' . preg_quote($param) . ')\b/'; + + $sql = preg_replace_callback($pattern, function(array $matches) use ($callback): string { + return ':' . $callback($matches['param']); + }, $sql); + } + + return $sql; + } + /** * Generate SQL expression for sorting group concatenated fields * diff --git a/reportbuilder/classes/local/report/column.php b/reportbuilder/classes/local/report/column.php index 159c80caeeb..7e1d3d02e9b 100644 --- a/reportbuilder/classes/local/report/column.php +++ b/reportbuilder/classes/local/report/column.php @@ -367,12 +367,12 @@ final class column { $fields = []; foreach ($this->fields as $alias => $sql) { - // Ensure params within SQL are prefixed with column index. - foreach ($this->params as $name => $value) { - $sql = preg_replace_callback('/:(?' . preg_quote($name, '\b/') . ')/', function(array $matches): string { - return ':' . $this->unique_param_name($matches['param']); - }, $sql); - } + + // Ensure parameter names within SQL are prefixed with column index. + $params = array_keys($this->params); + $sql = database::sql_replace_parameter_names($sql, $params, function(string $param): string { + return $this->unique_param_name($param); + }); $fields[$alias] = [ 'sql' => $sql, diff --git a/reportbuilder/tests/local/helpers/database_test.php b/reportbuilder/tests/local/helpers/database_test.php index 286c3be47ec..c8e09198cac 100644 --- a/reportbuilder/tests/local/helpers/database_test.php +++ b/reportbuilder/tests/local/helpers/database_test.php @@ -78,4 +78,31 @@ class database_test extends advanced_testcase { $this->expectExceptionMessage('Invalid parameter names (invalidfoo, invalidbar)'); database::validate_params($params); } + + /** + * Test replacement of parameter names within SQL statements + */ + public function test_sql_replace_parameter_names(): void { + global $DB; + + // Predefine parameter names, to ensure they don't overwrite each other. + [$param0, $param1, $param10] = ['rbparam0', 'rbparam1', 'rbparam10']; + + $sql = "SELECT :{$param0} AS field0, :{$param1} AS field1, :{$param10} AS field10" . $DB->sql_null_from_clause(); + $sql = database::sql_replace_parameter_names($sql, [$param0, $param1, $param10], static function(string $param): string { + return "prefix_{$param}"; + }); + + $record = $DB->get_record_sql($sql, [ + "prefix_{$param0}" => 'Zero', + "prefix_{$param1}" => 'One', + "prefix_{$param10}" => 'Ten', + ]); + + $this->assertEquals((object) [ + 'field0' => 'Zero', + 'field1' => 'One', + 'field10' => 'Ten', + ], $record); + } } diff --git a/reportbuilder/tests/local/report/column_test.php b/reportbuilder/tests/local/report/column_test.php index 4442747fd54..85c048598d4 100644 --- a/reportbuilder/tests/local/report/column_test.php +++ b/reportbuilder/tests/local/report/column_test.php @@ -173,19 +173,32 @@ class column_test extends advanced_testcase { * Test adding params to field, and retrieving them */ public function test_add_field_with_params(): void { - $param = database::generate_param_name(); + $param0 = database::generate_param_name(); + $param1 = database::generate_param_name(); $column = $this->create_column('test') ->set_index(1) - ->add_field(":{$param}", 'foo', [$param => 'bar']); + ->add_field(":{$param0}", 'foo', [$param0 => 'foo']) + ->add_field(":{$param1}", 'bar', [$param1 => 'bar']); // Select will look like the following: "p_rbparam", where index is the column index and counter is // a static value of the report helper class. - $select = $column->get_fields(); - preg_match('/:(?p1_rbparam[\d]+) AS c1_foo/', $select[0], $matches); + $fields = $column->get_fields(); + $this->assertCount(2, $fields); + preg_match('/:(?p1_rbparam[\d]+) AS c1_foo/', $fields[0], $matches); $this->assertArrayHasKey('paramname', $matches); - $this->assertEquals([$matches['paramname'] => 'bar'], $column->get_params()); + $fieldparam0 = $matches['paramname']; + + preg_match('/:(?p1_rbparam[\d]+) AS c1_bar/', $fields[1], $matches); + $this->assertArrayHasKey('paramname', $matches); + $fieldparam1 = $matches['paramname']; + + // Ensure column parameters have been renamed appropriately. + $this->assertEquals([ + $fieldparam0 => 'foo', + $fieldparam1 => 'bar', + ], $column->get_params()); } /** diff --git a/reportbuilder/upgrade.txt b/reportbuilder/upgrade.txt index 58d71093128..d7e36ec9179 100644 --- a/reportbuilder/upgrade.txt +++ b/reportbuilder/upgrade.txt @@ -1,6 +1,11 @@ This file describes API changes in /reportbuilder/* Information provided here is intended especially for developers. +=== 4.0.8 === + +* New database helper method `sql_replace_parameter_names` to help ensure uniqueness of parameters within an expression (where + that expression can be used multiple times as part of a larger query) + === 4.0.7 === * The schedule helper `create_schedule` method accepts a `$timenow` parameter to use for comparisons against current date From 551edbb9a608d1228fe17f134ac537b04ff6fa89 Mon Sep 17 00:00:00 2001 From: Paul Holden Date: Tue, 7 Mar 2023 22:09:32 +0000 Subject: [PATCH 2/2] MDL-77555 reportbuilder: improve SQL generation within filters. Use native ANSI SQL syntax for numeric comparisons where possible, define filter API for the case where filters must re-use the given field SQL while ensuring uniqueness of any field parameters. --- reportbuilder/classes/local/filters/base.php | 3 ++ reportbuilder/classes/local/filters/date.php | 32 +++++++++--------- .../classes/local/filters/number.php | 6 ++-- reportbuilder/classes/local/report/filter.php | 33 +++++++++++++++++++ .../tests/local/report/filter_test.php | 12 +++++++ reportbuilder/upgrade.txt | 2 ++ 6 files changed, 70 insertions(+), 18 deletions(-) diff --git a/reportbuilder/classes/local/filters/base.php b/reportbuilder/classes/local/filters/base.php index 5316a656913..86dac541fa2 100644 --- a/reportbuilder/classes/local/filters/base.php +++ b/reportbuilder/classes/local/filters/base.php @@ -102,6 +102,9 @@ abstract class base { /** * Returns the filter clauses to be used with SQL where * + * Ideally the field SQL should be included only once in the returned expression, however if that is unavoidable then + * use the {@see filter::get_field_sql_and_params} helper to ensure uniqueness of any parameters included within + * * @param array $values * @return array [$sql, [...$params]] */ diff --git a/reportbuilder/classes/local/filters/date.php b/reportbuilder/classes/local/filters/date.php index 696625b1201..f4f618cd4a4 100644 --- a/reportbuilder/classes/local/filters/date.php +++ b/reportbuilder/classes/local/filters/date.php @@ -164,30 +164,32 @@ class date extends base { switch ($operator) { case self::DATE_NOT_EMPTY: - $sql = "{$fieldsql} IS NOT NULL AND {$fieldsql} <> 0"; + $sql = "COALESCE({$fieldsql}, 0) <> 0"; break; case self::DATE_EMPTY: - $sql = "{$fieldsql} IS NULL OR {$fieldsql} = 0"; + $sql = "COALESCE({$fieldsql}, 0) = 0"; break; case self::DATE_RANGE: - $clauses = []; + $sql = ''; $datefrom = (int)($values["{$this->name}_from"] ?? 0); - if ($datefrom > 0) { - $paramdatefrom = database::generate_param_name(); - $clauses[] = "{$fieldsql} >= :{$paramdatefrom}"; - $params[$paramdatefrom] = $datefrom; - } - $dateto = (int)($values["{$this->name}_to"] ?? 0); - if ($dateto > 0) { - $paramdateto = database::generate_param_name(); - $clauses[] = "{$fieldsql} < :{$paramdateto}"; + + $paramdatefrom = database::generate_param_name(); + $paramdateto = database::generate_param_name(); + + if ($datefrom > 0 && $dateto > 0) { + $sql = "{$fieldsql} BETWEEN :{$paramdatefrom} AND :{$paramdateto}"; + $params[$paramdatefrom] = $datefrom; + $params[$paramdateto] = $dateto; + } else if ($datefrom > 0) { + $sql = "{$fieldsql} >= :{$paramdatefrom}"; + $params[$paramdatefrom] = $datefrom; + } else if ($dateto > 0) { + $sql = "{$fieldsql} < :{$paramdateto}"; $params[$paramdateto] = $dateto; } - $sql = implode(' AND ', $clauses); - break; // Relative helper method can handle these three cases. case self::DATE_LAST: @@ -202,7 +204,7 @@ class date extends base { $paramdatefrom = database::generate_param_name(); $paramdateto = database::generate_param_name(); - $sql = "{$fieldsql} >= :{$paramdatefrom} AND {$fieldsql} <= :{$paramdateto}"; + $sql = "{$fieldsql} BETWEEN :{$paramdatefrom} AND :{$paramdateto}"; [ $params[$paramdatefrom], $params[$paramdateto], diff --git a/reportbuilder/classes/local/filters/number.php b/reportbuilder/classes/local/filters/number.php index ae0efa82258..482bd737c3f 100644 --- a/reportbuilder/classes/local/filters/number.php +++ b/reportbuilder/classes/local/filters/number.php @@ -132,10 +132,10 @@ class number extends base { case self::ANY_VALUE: return ['', []]; case self::IS_NOT_EMPTY: - $res = "{$fieldsql} IS NOT NULL AND {$fieldsql} <> 0"; + $res = "COALESCE({$fieldsql}, 0) <> 0"; break; case self::IS_EMPTY: - $res = "{$fieldsql} IS NULL OR {$fieldsql} = 0"; + $res = "COALESCE({$fieldsql}, 0) = 0"; break; case self::LESS_THAN: $res = "{$fieldsql} < :{$param}"; @@ -158,7 +158,7 @@ class number extends base { $params[$param] = $value1; break; case self::RANGE: - $res = "({$fieldsql} >= :{$param} AND {$fieldsql} <= :{$param2})"; + $res = "{$fieldsql} BETWEEN :{$param} AND :{$param2}"; $params[$param] = $value1; $params[$param2] = $value2; break; diff --git a/reportbuilder/classes/local/report/filter.php b/reportbuilder/classes/local/report/filter.php index d45466b70d3..dbd731d778e 100644 --- a/reportbuilder/classes/local/report/filter.php +++ b/reportbuilder/classes/local/report/filter.php @@ -21,6 +21,7 @@ namespace core_reportbuilder\local\report; use lang_string; use moodle_exception; use core_reportbuilder\local\filters\base; +use core_reportbuilder\local\helpers\database; use core_reportbuilder\local\models\filter as filter_model; /** @@ -212,6 +213,38 @@ final class filter { return $this->fieldparams; } + /** + * Retrieve SQL expression and parameters for the field + * + * @param int $index + * @return array [$sql, [...$params]] + */ + public function get_field_sql_and_params(int $index = 0): array { + $fieldsql = $this->get_field_sql(); + $fieldparams = $this->get_field_params(); + + // Shortcut if there aren't any parameters. + if (empty($fieldparams)) { + return [$fieldsql, $fieldparams]; + } + + // Simple callback for replacement of parameter names within filter SQL. + $transform = function(string $param) use ($index): string { + return "{$param}_{$index}"; + }; + + $paramnames = array_keys($fieldparams); + $sql = database::sql_replace_parameter_names($fieldsql, $paramnames, $transform); + + $params = []; + foreach ($paramnames as $paramname) { + $paramnametransform = $transform($paramname); + $params[$paramnametransform] = $fieldparams[$paramname]; + } + + return [$sql, $params]; + } + /** * Set the SQL expression for the field that is being filtered. It will be passed to the filter class * diff --git a/reportbuilder/tests/local/report/filter_test.php b/reportbuilder/tests/local/report/filter_test.php index ed180b5d5a6..299d599a11a 100644 --- a/reportbuilder/tests/local/report/filter_test.php +++ b/reportbuilder/tests/local/report/filter_test.php @@ -108,6 +108,18 @@ class filter_test extends advanced_testcase { $this->assertEquals(['foo' => 'bar'], $filter->get_field_params()); } + /** + * Test getting field SQL and params, while providing index for uniqueness + */ + public function test_get_field_sql_and_params(): void { + $filter = $this->create_filter('username', 'u.username = :username AND u.idnumber = :idnumber', + ['username' => 'test', 'idnumber' => 'bar']); + + [$sql, $params] = $filter->get_field_sql_and_params(1); + $this->assertEquals('u.username = :username_1 AND u.idnumber = :idnumber_1', $sql); + $this->assertEquals(['username_1' => 'test', 'idnumber_1' => 'bar'], $params); + } + /** * Test adding single join */ diff --git a/reportbuilder/upgrade.txt b/reportbuilder/upgrade.txt index d7e36ec9179..5585c2881a1 100644 --- a/reportbuilder/upgrade.txt +++ b/reportbuilder/upgrade.txt @@ -5,6 +5,8 @@ Information provided here is intended especially for developers. * New database helper method `sql_replace_parameter_names` to help ensure uniqueness of parameters within an expression (where that expression can be used multiple times as part of a larger query) +* The local report filter class has a new `get_field_sql_and_params` method which should be used by filter types that re-use + the filter field SQL within their generated expression, to ensure SQL containing parameters works correctly === 4.0.7 ===