From b4d2e7f267b9a4b7fac58bbd4437a0867dc6c79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20S=CC=8Ckoda?= Date: Thu, 28 Nov 2013 12:59:16 +0800 Subject: [PATCH] MDL-31625 fix multiple global search-replace issues Includes following fixes: * support for MS SQL Server * optional trimming of of oversized VARCHAR fields * conversion to forms library * full localisation * other cleanup --- admin/tool/replace/classes/form.php | 69 +++++++++++++++++++++ admin/tool/replace/index.php | 59 ++++++------------ admin/tool/replace/lang/en/tool_replace.php | 11 ++-- admin/tool/replace/version.php | 2 +- lib/adminlib.php | 9 ++- lib/dml/moodle_database.php | 46 ++++++++++++++ lib/dml/mssql_native_moodle_database.php | 10 +++ lib/dml/mysqli_native_moodle_database.php | 10 +++ lib/dml/pgsql_native_moodle_database.php | 10 +++ lib/dml/sqlsrv_native_moodle_database.php | 10 +++ lib/dml/tests/dml_test.php | 58 +++++++++++++++++ 11 files changed, 245 insertions(+), 49 deletions(-) create mode 100644 admin/tool/replace/classes/form.php diff --git a/admin/tool/replace/classes/form.php b/admin/tool/replace/classes/form.php new file mode 100644 index 00000000000..5c605107265 --- /dev/null +++ b/admin/tool/replace/classes/form.php @@ -0,0 +1,69 @@ +. + +/** + * Site wide search-replace form. + * + * @package tool_replace + * @copyright 2013 Petr Skoda {@link http://skodak.org} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +require_once("$CFG->libdir/formslib.php"); + +/** + * Site wide search-replace form. + */ +class tool_replace_form extends moodleform { + function definition() { + global $CFG, $DB; + + $mform = $this->_form; + + $mform->addElement('header', 'searchhdr', get_string('pluginname', 'tool_replace')); + $mform->setExpanded('searchhdr', true); + + $mform->addElement('text', 'search', get_string('searchwholedb', 'tool_replace'), 'size="50"'); + $mform->setType('search', PARAM_RAW); + $mform->addElement('static', 'searchst', '', get_string('searchwholedbhelp', 'tool_replace')); + $mform->addRule('search', get_string('required'), 'required', null, 'client'); + + $mform->addElement('text', 'replace', get_string('replacewith', 'tool_replace'), 'size="50"', PARAM_RAW); + $mform->addElement('static', 'replacest', '', get_string('replacewithhelp', 'tool_replace')); + $mform->setType('replace', PARAM_RAW); + $mform->addElement('checkbox', 'shorten', get_string('shortenoversized', 'tool_replace')); + $mform->addRule('replace', get_string('required'), 'required', null, 'client'); + + $mform->addElement('header', 'confirmhdr', get_string('confirm')); + $mform->setExpanded('confirmhdr', true); + $mform->addElement('checkbox', 'sure', get_string('disclaimer', 'tool_replace')); + $mform->addRule('sure', get_string('required'), 'required', null, 'client'); + + $this->add_action_buttons(false, get_string('doit', 'tool_replace')); + } + + function validation($data, $files) { + $errors = parent::validation($data, $files); + + if (empty($data['shorten']) and core_text::strlen($data['search']) < core_text::strlen($data['replace'])) { + $errors['shorten'] = get_string('required'); + } + + return $errors; + } +} diff --git a/admin/tool/replace/index.php b/admin/tool/replace/index.php index 67d945ee427..b3013d232be 100644 --- a/admin/tool/replace/index.php +++ b/admin/tool/replace/index.php @@ -17,8 +17,7 @@ /** * Search and replace strings throughout all texts in the whole database * - * @package tool - * @subpackage replace + * @package tool_replace * @copyright 1999 onwards Martin Dougiamas (http://dougiamas.com) * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ @@ -31,56 +30,38 @@ require_once($CFG->libdir.'/adminlib.php'); admin_externalpage_setup('toolreplace'); -$search = optional_param('search', '', PARAM_RAW); -$replace = optional_param('replace', '', PARAM_RAW); -$sure = optional_param('sure', 0, PARAM_BOOL); - -################################################################### echo $OUTPUT->header(); echo $OUTPUT->heading(get_string('pageheader', 'tool_replace')); -if ($DB->get_dbfamily() !== 'mysql' and $DB->get_dbfamily() !== 'postgres') { - //TODO: add $DB->text_replace() to DML drivers +if (!$DB->replace_all_text_supported()) { echo $OUTPUT->notification(get_string('notimplemented', 'tool_replace')); echo $OUTPUT->footer(); die; } -if (!data_submitted() or !$search or !$replace or !confirm_sesskey() or !$sure) { /// Print a form - echo $OUTPUT->notification(get_string('notsupported', 'tool_replace')); - echo $OUTPUT->notification(get_string('excludedtables', 'tool_replace')); - - echo $OUTPUT->box_start(); - echo '
'; - echo '
'; - echo ''; - echo '
('. - get_string('searchwholedbhelp', 'tool_replace').')
'; - echo '
('. - get_string('replacewithhelp', 'tool_replace').')
'; - echo '
'; - echo '
'; - echo '
'; - echo '
'; - echo $OUTPUT->box_end(); - echo $OUTPUT->footer(); - die; -} - echo $OUTPUT->box_start(); -db_replace($search, $replace); +echo $OUTPUT->notification(get_string('notsupported', 'tool_replace')); +echo $OUTPUT->notification(get_string('excludedtables', 'tool_replace')); echo $OUTPUT->box_end(); -/// Rebuild course cache which might be incorrect now -echo $OUTPUT->notification(get_string('notifyrebuilding', 'tool_replace'), 'notifysuccess'); -rebuild_course_cache(); -echo $OUTPUT->notification(get_string('notifyfinished', 'tool_replace'), 'notifysuccess'); +$form = new tool_replace_form(); + +if (!$data = $form->get_data()) { + $form->display(); + echo $OUTPUT->footer(); + die(); +} + +// Scroll to the end when finished. +$PAGE->requires->js_init_code("window.scrollTo(0, 5000000);"); + +echo $OUTPUT->box_start(); +db_replace($data->search, $data->replace); +echo $OUTPUT->box_end(); + +// Course caches are now rebuilt on the fly. echo $OUTPUT->continue_button(new moodle_url('/admin/index.php')); echo $OUTPUT->footer(); - - diff --git a/admin/tool/replace/lang/en/tool_replace.php b/admin/tool/replace/lang/en/tool_replace.php index 9103e890ebc..919ed67ae0f 100644 --- a/admin/tool/replace/lang/en/tool_replace.php +++ b/admin/tool/replace/lang/en/tool_replace.php @@ -23,15 +23,18 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ -$string['disclaimer'] = 'I understand the risks of this operation:'; +$string['cannotfit'] = 'The replacement is longer than original and shortening is not allow, cannot continue.'; +$string['disclaimer'] = 'I understand the risks of this operation'; +$string['doit'] = 'Yes, do it!'; $string['excludedtables'] = 'Several tables are not updated as part of the text replacement. This include configuration, log, events, and session tables.'; $string['pageheader'] = 'Search and replace text throughout the whole database'; $string['notifyfinished'] = '...finished'; $string['notifyrebuilding'] = 'Rebuilding course cache...'; -$string['notimplemented'] = 'Sorry, this feature is implemented only for MySQL and PostgreSQL databases.'; +$string['notimplemented'] = 'Sorry, this feature is not implemented in your database driver.'; $string['notsupported'] ='This script is not supported, always make complete backup before proceeding!
This operation can not be reverted!'; $string['pluginname'] = 'DB search and replace'; -$string['replacewith'] = 'Replace with this string:'; +$string['replacewith'] = 'Replace with this string'; $string['replacewithhelp'] = 'usually new server URL'; -$string['searchwholedb'] = 'Search whole database for:'; +$string['searchwholedb'] = 'Search whole database for'; $string['searchwholedbhelp'] = 'usually previous server URL'; +$string['shortenoversized'] = 'Shorten result if necessary'; diff --git a/admin/tool/replace/version.php b/admin/tool/replace/version.php index 4496dcd7a4c..307be8fb900 100644 --- a/admin/tool/replace/version.php +++ b/admin/tool/replace/version.php @@ -25,7 +25,7 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2013110500; // The current plugin version (Date: YYYYMMDDXX) +$plugin->version = 2013110501; // The current plugin version (Date: YYYYMMDDXX) $plugin->requires = 2013110500; // Requires this Moodle version $plugin->component = 'tool_replace'; // Full name of the plugin (used for diagnostics) diff --git a/lib/adminlib.php b/lib/adminlib.php index 074ad6426b4..3cd720803dc 100644 --- a/lib/adminlib.php +++ b/lib/adminlib.php @@ -6741,11 +6741,8 @@ function db_replace($search, $replace) { if ($columns = $DB->get_columns($table)) { $DB->set_debug(true); - foreach ($columns as $column => $data) { - if (in_array($data->meta_type, array('C', 'X'))) { // Text stuff only - //TODO: this should be definitively moved to DML driver to do the actual replace, this is not going to work for MSSQL and Oracle... - $DB->execute("UPDATE {".$table."} SET $column = REPLACE($column, ?, ?)", array($search, $replace)); - } + foreach ($columns as $column) { + $DB->replace_all_text($table, $column, $search, $replace); } $DB->set_debug(false); } @@ -6776,6 +6773,8 @@ function db_replace($search, $replace) { echo $OUTPUT->notification("...finished", 'notifysuccess'); } + purge_all_caches(); + return true; } diff --git a/lib/dml/moodle_database.php b/lib/dml/moodle_database.php index 156d39c4e76..de17fc8b6fc 100644 --- a/lib/dml/moodle_database.php +++ b/lib/dml/moodle_database.php @@ -2126,6 +2126,52 @@ abstract class moodle_database { return ''; } + /** + * Does this driver support tool_replace? + * + * @since 2.6.1 + * @return bool + */ + public function replace_all_text_supported() { + return false; + } + + /** + * Replace given text in all rows of column. + * + * @since 2.6.1 + * @param string $table name of the table + * @param database_column_info $column + * @param string $search + * @param string $replace + */ + public function replace_all_text($table, database_column_info $column, $search, $replace) { + if (!$this->replace_all_text_supported()) { + return; + } + + // NOTE: override this methods if following standard compliant SQL + // does not work for your driver. + + $columnname = $column->name; + $sql = "UPDATE {".$table."} + SET $columnname = REPLACE($columnname, ?, ?) + WHERE $columnname IS NOT NULL"; + + if ($column->meta_type === 'X') { + $this->execute($sql, array($search, $replace)); + + } else if ($column->meta_type === 'C') { + if (core_text::strlen($search) < core_text::strlen($replace)) { + $colsize = $column->max_length; + $sql = "UPDATE {".$table."} + SET $columnname = SUBSTRING(REPLACE($columnname, ?, ?), 1, $colsize) + WHERE $columnname IS NOT NULL"; + } + $this->execute($sql, array($search, $replace)); + } + } + /** * Checks and returns true if transactions are supported. * diff --git a/lib/dml/mssql_native_moodle_database.php b/lib/dml/mssql_native_moodle_database.php index ec4e983548b..cc682e5cb82 100644 --- a/lib/dml/mssql_native_moodle_database.php +++ b/lib/dml/mssql_native_moodle_database.php @@ -1252,6 +1252,16 @@ s only returning name of SQL substring function, it now requires all parameters. } } + /** + * Does this driver support tool_replace? + * + * @since 2.6.1 + * @return bool + */ + public function replace_all_text_supported() { + return true; + } + public function session_lock_supported() { return true; } diff --git a/lib/dml/mysqli_native_moodle_database.php b/lib/dml/mysqli_native_moodle_database.php index 5196484b5c9..33d07b99b70 100644 --- a/lib/dml/mysqli_native_moodle_database.php +++ b/lib/dml/mysqli_native_moodle_database.php @@ -1392,6 +1392,16 @@ class mysqli_native_moodle_database extends moodle_database { return ' CAST(' . $fieldname . ' AS SIGNED) '; } + /** + * Does this driver support tool_replace? + * + * @since 2.6.1 + * @return bool + */ + public function replace_all_text_supported() { + return true; + } + public function session_lock_supported() { return true; } diff --git a/lib/dml/pgsql_native_moodle_database.php b/lib/dml/pgsql_native_moodle_database.php index 3754d9b1df4..f4cf21dd0e7 100644 --- a/lib/dml/pgsql_native_moodle_database.php +++ b/lib/dml/pgsql_native_moodle_database.php @@ -1216,6 +1216,16 @@ class pgsql_native_moodle_database extends moodle_database { return $positivematch ? '~*' : '!~*'; } + /** + * Does this driver support tool_replace? + * + * @since 2.6.1 + * @return bool + */ + public function replace_all_text_supported() { + return true; + } + public function session_lock_supported() { return true; } diff --git a/lib/dml/sqlsrv_native_moodle_database.php b/lib/dml/sqlsrv_native_moodle_database.php index 6b9bce2fc8b..093cda6ae66 100644 --- a/lib/dml/sqlsrv_native_moodle_database.php +++ b/lib/dml/sqlsrv_native_moodle_database.php @@ -1314,6 +1314,16 @@ class sqlsrv_native_moodle_database extends moodle_database { } } + /** + * Does this driver support tool_replace? + * + * @since 2.6.1 + * @return bool + */ + public function replace_all_text_supported() { + return true; + } + public function session_lock_supported() { return true; } diff --git a/lib/dml/tests/dml_test.php b/lib/dml/tests/dml_test.php index 146f615914e..d44714eba80 100644 --- a/lib/dml/tests/dml_test.php +++ b/lib/dml/tests/dml_test.php @@ -4234,6 +4234,64 @@ class core_dml_testcase extends database_driver_testcase { $this->assertCount($currentcount, $results); } + public function test_replace_all_text() { + $DB = $this->tdb; + $dbman = $DB->get_manager(); + + if (!$DB->replace_all_text_supported()) { + $this->markTestSkipped($DB->get_name().' does not support replacing of texts'); + } + + $table = $this->get_test_table(); + $tablename = $table->getName(); + + $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('name', XMLDB_TYPE_CHAR, '20', null, null); + $table->add_field('intro', XMLDB_TYPE_TEXT, 'big', null, null); + $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id')); + $dbman->create_table($table); + + $id1 = (string)$DB->insert_record($tablename, array('name' => null, 'intro' => null)); + $id2 = (string)$DB->insert_record($tablename, array('name' => '', 'intro' => '')); + $id3 = (string)$DB->insert_record($tablename, array('name' => 'xxyy', 'intro' => 'vvzz')); + $id4 = (string)$DB->insert_record($tablename, array('name' => 'aa bb aa bb', 'intro' => 'cc dd cc aa')); + $id5 = (string)$DB->insert_record($tablename, array('name' => 'kkllll', 'intro' => 'kkllll')); + + $expected = $DB->get_records($tablename, array(), 'id ASC'); + + $columns = $DB->get_columns($tablename); + + $DB->replace_all_text($tablename, $columns['name'], 'aa', 'o'); + $result = $DB->get_records($tablename, array(), 'id ASC'); + $expected[$id4]->name = 'o bb o bb'; + $this->assertEquals($expected, $result); + + $DB->replace_all_text($tablename, $columns['intro'], 'aa', 'o'); + $result = $DB->get_records($tablename, array(), 'id ASC'); + $expected[$id4]->intro = 'cc dd cc o'; + $this->assertEquals($expected, $result); + + $DB->replace_all_text($tablename, $columns['name'], '_', '*'); + $DB->replace_all_text($tablename, $columns['name'], '?', '*'); + $DB->replace_all_text($tablename, $columns['name'], '%', '*'); + $DB->replace_all_text($tablename, $columns['intro'], '_', '*'); + $DB->replace_all_text($tablename, $columns['intro'], '?', '*'); + $DB->replace_all_text($tablename, $columns['intro'], '%', '*'); + $result = $DB->get_records($tablename, array(), 'id ASC'); + $this->assertEquals($expected, $result); + + $long = '1234567890123456789'; + $DB->replace_all_text($tablename, $columns['name'], 'kk', $long); + $result = $DB->get_records($tablename, array(), 'id ASC'); + $expected[$id5]->name = core_text::substr($long.'llll', 0, 20); + $this->assertEquals($expected, $result); + + $DB->replace_all_text($tablename, $columns['intro'], 'kk', $long); + $result = $DB->get_records($tablename, array(), 'id ASC'); + $expected[$id5]->intro = $long.'llll'; + $this->assertEquals($expected, $result); + } + public function test_onelevel_commit() { $DB = $this->tdb; $dbman = $DB->get_manager();