diff --git a/admin/cli/mysql_collation.php b/admin/cli/mysql_collation.php new file mode 100644 index 00000000000..9938a3af9f8 --- /dev/null +++ b/admin/cli/mysql_collation.php @@ -0,0 +1,211 @@ +. + +/** + * MySQL collation conversion tool. + * + * @package core + * @copyright 2012 Petr Skoda (http://skodak.org) + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +define('CLI_SCRIPT', true); + +require(dirname(dirname(dirname(__FILE__))).'/config.php'); +require_once($CFG->libdir.'/clilib.php'); // cli only functions + +if ($DB->get_dbfamily() !== 'mysql') { + cli_error('This function is designed for MySQL databases only!'); +} + +// now get cli options +list($options, $unrecognized) = cli_get_params(array('help'=>false, 'list'=>false, 'collation'=>false, 'available'=>false), + array('h'=>'help', 'l'=>'list', 'a'=>'available')); + +if ($unrecognized) { + $unrecognized = implode("\n ", $unrecognized); + cli_error(get_string('cliunknowoption', 'admin', $unrecognized)); +} + +$help = + "MySQL collation conversions script. + +It is strongly recommended to stop the web server before the conversion. +This script may be executed before the main upgrade - 1.9.x data for example. + +Options: +--collation=COLLATION Convert MySQL tables to different collation +-l, --list Show table and column information +-a, --available Show list of available collations +-h, --help Print out this help + +Example: +\$ sudo -u www-data /usr/bin/php admin/cli/mysql_collation.php --collation=utf8_general_ci +"; + +if (!empty($options['collation'])) { + $collations = mysql_get_collations(); + $collation = clean_param($options['collation'], PARAM_ALPHANUMEXT); + $collation = strtolower($collation); + if (!isset($collations[$collation])) { + cli_error("Error: collation '$collation' is not available on this server!"); + } + + echo "Converting tables and columns to '$collation' for $CFG->wwwroot:\n"; + $prefix = $DB->get_prefix(); + $prefix = str_replace('_', '\\_', $prefix); + $sql = "SHOW TABLE STATUS WHERE Name LIKE BINARY '$prefix%'"; + $rs = $DB->get_recordset_sql($sql); + $converted = 0; + $skipped = 0; + $errors = 0; + foreach ($rs as $table) { + echo str_pad($table->name, 40). " - "; + + if ($table->collation === $collation) { + echo "NO CHANGE\n"; + $skipped++; + + } else { + $DB->change_database_structure("ALTER TABLE $table->name DEFAULT COLLATE = $collation"); + echo "CONVERTED\n"; + $converted++; + } + + $sql = "SHOW FULL COLUMNS FROM $table->name WHERE collation IS NOT NULL"; + $rs2 = $DB->get_recordset_sql($sql); + foreach ($rs2 as $column) { + $column = (object)array_change_key_case((array)$column, CASE_LOWER); + echo ' '.str_pad($column->field, 36). " - "; + if ($column->collation === $collation) { + echo "NO CHANGE\n"; + $skipped++; + continue; + } + + if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text' or $column->type === 'longtext') { + $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL'; + $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : ''; + // primary, unique and inc are not supported for texts + $sql = "ALTER TABLE $table->name MODIFY COLUMN $column->field $column->type COLLATE $collation $notnull $default"; + $DB->change_database_structure($sql); + + } else if (strpos($column->type, 'varchar') === 0) { + $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL'; + $default = !is_null($column->default) ? "DEFAULT '$column->default'" : ''; + // primary, unique and inc are not supported for texts + $sql = "ALTER TABLE $table->name MODIFY COLUMN $column->field $column->type COLLATE $collation $notnull $default"; + $DB->change_database_structure($sql); + } else { + echo "ERROR (unknown column type: $column->type)\n"; + $error++; + continue; + } + echo "CONVERTED\n"; + $converted++; + } + $rs2->close(); + } + $rs->close(); + echo "Converted: $converted, skipped: $skipped, errors: $errors\n"; + exit(0); // success + +} else if (!empty($options['list'])) { + echo "List of tables for $CFG->wwwroot:\n"; + $prefix = $DB->get_prefix(); + $prefix = str_replace('_', '\\_', $prefix); + $sql = "SHOW TABLE STATUS WHERE Name LIKE BINARY '$prefix%'"; + $rs = $DB->get_recordset_sql($sql); + $counts = array(); + foreach ($rs as $table) { + if (isset($counts[$table->collation])) { + $counts[$table->collation]++; + } else { + $counts[$table->collation] = 1; + } + echo str_pad($table->name, 40); + echo $table->collation. "\n"; + $collations = mysql_get_column_collations($table->name); + foreach ($collations as $columname=>$collation) { + if (isset($counts[$collation])) { + $counts[$collation]++; + } else { + $counts[$collation] = 1; + } + echo ' '; + echo str_pad($columname, 36); + echo $collation. "\n"; + } + } + $rs->close(); + + echo "\n"; + echo "Table collations summary for $CFG->wwwroot:\n"; + foreach ($counts as $collation => $count) { + echo "$collation: $count\n"; + } + exit(0); // success + +} else if (!empty($options['available'])) { + echo "List of available MySQL collations for $CFG->wwwroot:\n"; + $collations = mysql_get_collations(); + foreach ($collations as $collation) { + echo " $collation\n"; + } + die; + +} else { + echo $help; + die; +} + + + +// ========== Some functions ============== + +function mysql_get_collations() { + global $DB; + + $collations = array(); + $sql = "SHOW COLLATION WHERE Collation LIKE 'utf8\_%' AND Charset = 'utf8'"; + $rs = $DB->get_recordset_sql($sql); + foreach ($rs as $collation) { + $collations[$collation->collation] = $collation->collation; + } + $rs->close(); + + $collation = $DB->get_dbcollation(); + if (isset($collations[$collation])) { + $collations[$collation] .= ' (default)'; + } + + return $collations; +} + +function mysql_get_column_collations($tablename) { + global $DB; + + $collations = array(); + $sql = "SELECT column_name, collation_name + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_schema = DATABASE() AND table_name = ? AND collation_name IS NOT NULL"; + $rs = $DB->get_recordset_sql($sql, array($tablename)); + foreach($rs as $record) { + $collations[$record->column_name] = $record->collation_name; + } + $rs->close(); + return $collations; +} diff --git a/lib/ddl/mysql_sql_generator.php b/lib/ddl/mysql_sql_generator.php index 6ae310f6411..abd5ddde856 100644 --- a/lib/ddl/mysql_sql_generator.php +++ b/lib/ddl/mysql_sql_generator.php @@ -117,23 +117,25 @@ class mysql_sql_generator extends sql_generator { * by any of its comments, indexes and sequence creation SQL statements. */ public function getCreateTableSQL($xmldb_table) { - // first find out if want some special db engine - $engine = null; - if (method_exists($this->mdb, 'get_dbengine')) { - $engine = $this->mdb->get_dbengine(); - } + // First find out if want some special db engine. + $engine = $this->mdb->get_dbengine(); + // Do we know collation? + $collation = $this->mdb->get_dbcollation(); $sqlarr = parent::getCreateTableSQL($xmldb_table); - if (!$engine) { - // we rely on database defaults - return $sqlarr; - } - - // let's inject the engine into SQL + // Let's inject the extra MySQL tweaks. foreach ($sqlarr as $i=>$sql) { if (strpos($sql, 'CREATE TABLE ') === 0) { - $sqlarr[$i] .= " ENGINE = $engine"; + if ($engine) { + $sqlarr[$i] .= " ENGINE = $engine"; + } + if ($collation) { + if (strpos($collation, 'utf8_') === 0) { + $sqlarr[$i] .= " DEFAULT CHARACTER SET utf8"; + } + $sqlarr[$i] .= " DEFAULT COLLATE = $collation"; + } } } @@ -148,9 +150,26 @@ class mysql_sql_generator extends sql_generator { * @return array of sql statements */ public function getCreateTempTableSQL($xmldb_table) { + // Do we know collation? + $collation = $this->mdb->get_dbcollation(); $this->temptables->add_temptable($xmldb_table->getName()); - $sqlarr = parent::getCreateTableSQL($xmldb_table); // we do not want the engine hack included in create table SQL - $sqlarr = preg_replace('/^CREATE TABLE (.*)/s', 'CREATE TEMPORARY TABLE $1', $sqlarr); + + $sqlarr = parent::getCreateTableSQL($xmldb_table); + + // Let's inject the extra MySQL tweaks. + foreach ($sqlarr as $i=>$sql) { + if (strpos($sql, 'CREATE TABLE ') === 0) { + // We do not want the engine hack included in create table SQL. + $sqlarr[$i] = preg_replace('/^CREATE TABLE (.*)/s', 'CREATE TEMPORARY TABLE $1', $sql); + if ($collation) { + if (strpos($collation, 'utf8_') === 0) { + $sqlarr[$i] .= " DEFAULT CHARACTER SET utf8"; + } + $sqlarr[$i] .= " DEFAULT COLLATE $collation"; + } + } + } + return $sqlarr; } @@ -231,9 +250,21 @@ class mysql_sql_generator extends sql_generator { $xmldb_length='255'; } $dbtype .= '(' . $xmldb_length . ')'; + if ($collation = $this->mdb->get_dbcollation()) { + if (strpos($collation, 'utf8_') === 0) { + $dbtype .= " CHARACTER SET utf8"; + } + $dbtype .= " COLLATE $collation"; + } break; case XMLDB_TYPE_TEXT: $dbtype = 'LONGTEXT'; + if ($collation = $this->mdb->get_dbcollation()) { + if (strpos($collation, 'utf8_') === 0) { + $dbtype .= " CHARACTER SET utf8"; + } + $dbtype .= " COLLATE $collation"; + } break; case XMLDB_TYPE_BINARY: $dbtype = 'LONGBLOB'; diff --git a/lib/dml/mysqli_native_moodle_database.php b/lib/dml/mysqli_native_moodle_database.php index 93311bd5fd4..9294c07c38e 100644 --- a/lib/dml/mysqli_native_moodle_database.php +++ b/lib/dml/mysqli_native_moodle_database.php @@ -82,7 +82,13 @@ class mysqli_native_moodle_database extends moodle_database { throw new dml_connection_exception($dberr); } - $result = $conn->query("CREATE DATABASE $dbname DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci"); + if (isset($dboptions['dbcollation']) and strpos($dboptions['dbcollation'], 'utf8_') === 0) { + $collation = $dboptions['dbcollation']; + } else { + $collation = 'utf8_unicode_ci'; + } + + $result = $conn->query("CREATE DATABASE $dbname DEFAULT CHARACTER SET utf8 DEFAULT COLLATE ".$collation); $conn->close(); @@ -146,22 +152,28 @@ class mysqli_native_moodle_database extends moodle_database { return $this->dboptions['dbengine']; } - $engine = null; - - if (!$this->external) { - // look for current engine of our config table (the first table that gets created), - // so that we create all tables with the same engine - $sql = "SELECT engine FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = DATABASE() AND table_name = '{$this->prefix}config'"; - $this->query_start($sql, NULL, SQL_QUERY_AUX); - $result = $this->mysqli->query($sql); - $this->query_end($result); - if ($rec = $result->fetch_assoc()) { - $engine = $rec['engine']; - } - $result->close(); + if ($this->external) { + return null; } + $engine = null; + + // Look for current engine of our config table (the first table that gets created), + // so that we create all tables with the same engine. + $sql = "SELECT engine + FROM INFORMATION_SCHEMA.TABLES + WHERE table_schema = DATABASE() AND table_name = '{$this->prefix}config'"; + $this->query_start($sql, NULL, SQL_QUERY_AUX); + $result = $this->mysqli->query($sql); + $this->query_end($result); + if ($rec = $result->fetch_assoc()) { + $engine = $rec['engine']; + } + $result->close(); + if ($engine) { + // Cache the result to improve performance. + $this->dboptions['dbengine'] = $engine; return $engine; } @@ -175,7 +187,7 @@ class mysqli_native_moodle_database extends moodle_database { } $result->close(); - if (!$this->external and $engine === 'MyISAM') { + if ($engine === 'MyISAM') { // we really do not want MyISAM for Moodle, InnoDB or XtraDB is a reasonable defaults if supported $sql = "SHOW STORAGE ENGINES"; $this->query_start($sql, NULL, SQL_QUERY_AUX); @@ -196,9 +208,77 @@ class mysqli_native_moodle_database extends moodle_database { } } + // Cache the result to improve performance. + $this->dboptions['dbengine'] = $engine; return $engine; } + /** + * Returns the current MySQL db collation. + * + * This is an ugly workaround for MySQL default collation problems. + * + * @return string or null MySQL collation name + */ + public function get_dbcollation() { + if (isset($this->dboptions['dbcollation'])) { + return $this->dboptions['dbcollation']; + } + if ($this->external) { + return null; + } + + $collation = null; + + // Look for current collation of our config table (the first table that gets created), + // so that we create all tables with the same collation. + $sql = "SELECT collation_name + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_schema = DATABASE() AND table_name = '{$this->prefix}config' AND column_name = 'value'"; + $this->query_start($sql, NULL, SQL_QUERY_AUX); + $result = $this->mysqli->query($sql); + $this->query_end($result); + if ($rec = $result->fetch_assoc()) { + $collation = $rec['collation_name']; + } + $result->close(); + + if (!$collation) { + // Get the default database collation, but only if using UTF-8. + $sql = "SELECT @@collation_database"; + $this->query_start($sql, NULL, SQL_QUERY_AUX); + $result = $this->mysqli->query($sql); + $this->query_end($result); + if ($rec = $result->fetch_assoc()) { + if (strpos($rec['@@collation_database'], 'utf8_') === 0) { + $collation = $rec['@@collation_database']; + } + } + $result->close(); + } + + if (!$collation) { + // We want only utf8 compatible collations. + $collation = null; + $sql = "SHOW COLLATION WHERE Collation LIKE 'utf8\_%' AND Charset = 'utf8'"; + $this->query_start($sql, NULL, SQL_QUERY_AUX); + $result = $this->mysqli->query($sql); + $this->query_end($result); + while ($res = $result->fetch_assoc()) { + $collation = $res['Collation']; + if (strtoupper($res['Default']) === 'YES') { + $collation = $res['Collation']; + break; + } + } + $result->close(); + } + + // Cache the result to improve performance. + $this->dboptions['dbcollation'] = $collation; + return $collation; + } + /** * Returns localised database type name * Note: can be used before connect()