diff --git a/admin/cli/mysql_compressed_rows.php b/admin/cli/mysql_compressed_rows.php new file mode 100644 index 00000000000..6c2c411d942 --- /dev/null +++ b/admin/cli/mysql_compressed_rows.php @@ -0,0 +1,202 @@ +. + +/** + * MySQL table row compression tool tool. + * + * @package core + * @copyright 2014 Totara Learning Solutions Ltd {@link http://www.totaralms.com/} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @author Petr Skoda + */ + +define('CLI_SCRIPT', true); + +require(dirname(__FILE__).'/../../config.php'); +require_once($CFG->libdir . '/clilib.php'); + +if ($DB->get_dbfamily() !== 'mysql') { + cli_error('This script is used for MySQL databases only.'); +} + +$engine = strtolower($DB->get_dbengine()); +if ($engine !== 'innodb' and $engine !== 'xtradb') { + cli_error('This script is for MySQL servers using InnoDB or XtraDB engines only.'); +} + +list($options, $unrecognized) = cli_get_params( + array('help' => false, 'info' => false, 'list' => false, 'fix' => false, 'showsql' => false), + array('h' => 'help', 'i' => 'info', 'l' => 'list', 'f' => 'fix', 's' => 'showsql') +); + +if ($unrecognized) { + $unrecognized = implode("\n ", $unrecognized); + cli_error(get_string('cliunknowoption', 'admin', $unrecognized)); +} + +$help = + "Script for detection of row size problems in MySQL InnoDB tables. + +By default InnoDB storage table is using legacy Antelope file format +which has major restriction on database row size. +Use this script to detect and fix database tables with potentail data +overflow problems. + +Options: +-i, --info Show database information +-l, --list List problematic tables +-f, --fix Attempt to fix all tables (requires SUPER privilege) +-s, --showsql Print SQL statements for fixing of tables +-h, --help Print out this help + +Example: +\$ sudo -u www-data /usr/bin/php admin/cli/mysql_compressed_rows.php -l +"; + +/** @var mysql_sql_generator $generator */ +$generator = $DB->get_manager()->generator; + +$info = $DB->get_server_info(); +$filepertable = $DB->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_per_table'"); +$filepertable = $filepertable ? $filepertable->value : ''; +$fileformat = $DB->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_format'"); +$fileformat = $fileformat ? $fileformat->value : ''; +$prefix = $DB->get_prefix(); +$database = $CFG->dbname; + +if (!empty($options['info'])) { + echo "Database version: " . $info['description'] . "\n"; + echo "Database name: $database\n"; + echo "Database engine: " . $DB->get_dbengine() . "\n"; + echo "innodb_file_per_table: $filepertable\n"; + echo "innodb_file_format: $fileformat\n"; + + exit(0); + +} else if (!empty($options['list'])) { + $problem = false; + foreach ($DB->get_tables(false) as $table) { + $columns = $DB->get_columns($table, false); + $size = $generator->guess_antolope_row_size($columns); + $format = $DB->get_row_format($table); + if ($size <= $generator::ANTELOPE_MAX_ROW_SIZE) { + continue; + } + + echo str_pad($prefix . $table, 32, ' ', STR_PAD_RIGHT); + echo str_pad($format, 11, ' ', STR_PAD_RIGHT); + + if ($format === 'Compact' or $format === 'Redundant') { + $problem = true; + echo " (needs fixing)\n"; + + } else if ($format !== 'Compressed' and $format !== 'Dynamic') { + echo " (unknown)\n"; + + } else { + echo "\n"; + } + } + + if ($problem) { + exit(1); + } + exit(0); + +} else if (!empty($options['fix'])) { + $fixtables = array(); + foreach ($DB->get_tables(false) as $table) { + $columns = $DB->get_columns($table, false); + $size = $generator->guess_antolope_row_size($columns); + $format = $DB->get_row_format($table); + if ($size <= $generator::ANTELOPE_MAX_ROW_SIZE) { + continue; + } + if ($format === 'Compact' or $format === 'Redundant') { + $fixtables[$table] = $table; + } + } + + if (!$fixtables) { + echo "No changes necessary\n"; + exit(0); + } + + if ($filepertable !== 'ON') { + try { + $DB->execute("SET GLOBAL innodb_file_per_table=1"); + } catch (dml_exception $e) { + echo "Cannot enable GLOBAL innodb_file_per_table setting, use --sqhowsql option and execute the statements manually."; + throw $e; + } + } + if ($fileformat !== 'Barracuda') { + try { + $DB->execute("SET GLOBAL innodb_file_format=Barracuda"); + } catch (dml_exception $e) { + echo "Cannot change GLOBAL innodb_file_format setting, use --sqhowsql option and execute the statements manually."; + throw $e; + } + } + + if (!$DB->is_compressed_row_format_supported(false)) { + echo "MySQL server is not compatible with compressed row format."; + exit(1); + } + + foreach ($fixtables as $table) { + $DB->change_database_structure("ALTER TABLE {$prefix}$table ROW_FORMAT=Compressed"); + echo str_pad($prefix . $table, 32, ' ', STR_PAD_RIGHT) . " ... Compressed\n"; + } + + exit(0); + +} else if (!empty($options['showsql'])) { + $fixtables = array(); + + foreach ($DB->get_tables(false) as $table) { + $columns = $DB->get_columns($table, false); + $size = $generator->guess_antolope_row_size($columns); + $format = $DB->get_row_format($table); + if ($size <= $generator::ANTELOPE_MAX_ROW_SIZE) { + continue; + } + if ($format === 'Compact' or $format === 'Redundant') { + $fixtables[$table] = $table; + } + } + if (!$fixtables) { + echo "No changes necessary\n"; + exit(0); + } + + echo "Copy the following SQL statements and execute them using account with SUPER privilege:\n\n"; + echo "USE $database;\n"; + echo "SET SESSION sql_mode=STRICT_ALL_TABLES;\n"; + echo "SET GLOBAL innodb_file_per_table=1;\n"; + echo "SET GLOBAL innodb_file_format=Barracuda;\n"; + foreach ($fixtables as $table) { + echo "ALTER TABLE {$prefix}$table ROW_FORMAT=Compressed;\n"; + } + echo "\n"; + exit(0); + +} else { + echo $help; + die; +} + + diff --git a/lib/ddl/mysql_sql_generator.php b/lib/ddl/mysql_sql_generator.php index bbd6de2a96e..f92288b5ea0 100644 --- a/lib/ddl/mysql_sql_generator.php +++ b/lib/ddl/mysql_sql_generator.php @@ -88,6 +88,9 @@ class mysql_sql_generator extends sql_generator { /** @var string SQL sentence to rename one key 'TABLENAME', 'OLDKEYNAME' and 'NEWKEYNAME' are dynamically replaced.*/ public $rename_key_sql = null; + /** Maximum size of InnoDB row in Antelope file format */ + const ANTELOPE_MAX_ROW_SIZE = 8126; + /** * Reset a sequence to the id field of a table. * @@ -108,6 +111,78 @@ class mysql_sql_generator extends sql_generator { return array("ALTER TABLE $this->prefix$tablename AUTO_INCREMENT = $value"); } + /** + * Calculate proximate row size when using InnoDB + * tables in Antelope row format. + * + * Note: the returned value is a bit higher to compensate for + * errors and changes of column data types. + * + * @param xmldb_field[]|database_column_info[] $columns + * @return int approximate row size in bytes + */ + public function guess_antolope_row_size(array $columns) { + if (empty($columns)) { + return 0; + } + + $size = 0; + $first = reset($columns); + + if (count($columns) > 1) { + // Do not start with zero because we need to cover changes of field types and + // this calculation is most probably not be accurate. + $size += 1000; + } + + if ($first instanceof xmldb_field) { + foreach ($columns as $field) { + switch ($field->getType()) { + case XMLDB_TYPE_TEXT: + $size += 768; + break; + case XMLDB_TYPE_BINARY: + $size += 768; + break; + case XMLDB_TYPE_CHAR: + $bytes = $field->getLength() * 3; + if ($bytes > 768) { + $bytes = 768; + } + $size += $bytes; + break; + default: + // Anything else is usually maximum 8 bytes. + $size += 8; + } + } + + } else if ($first instanceof database_column_info) { + foreach ($columns as $column) { + switch ($column->meta_type) { + case 'X': + $size += 768; + break; + case 'B': + $size += 768; + break; + case 'C': + $bytes = $column->max_length * 3; + if ($bytes > 768) { + $bytes = 768; + } + $size += $bytes; + break; + default: + // Anything else is usually maximum 8 bytes. + $size += 8; + } + } + } + + return $size; + } + /** * Given one correct xmldb_table, returns the SQL statements * to create it (inside one array). @@ -122,6 +197,15 @@ class mysql_sql_generator extends sql_generator { // Do we know collation? $collation = $this->mdb->get_dbcollation(); + // Do we need to use compressed format for rows? + $rowformat = ""; + $size = $this->guess_antolope_row_size($xmldb_table->getFields()); + if ($size > self::ANTELOPE_MAX_ROW_SIZE) { + if ($this->mdb->is_compressed_row_format_supported()) { + $rowformat = "\n ROW_FORMAT=Compressed"; + } + } + $sqlarr = parent::getCreateTableSQL($xmldb_table); // Let's inject the extra MySQL tweaks. @@ -136,12 +220,48 @@ class mysql_sql_generator extends sql_generator { } $sqlarr[$i] .= " DEFAULT COLLATE = $collation"; } + if ($rowformat) { + $sqlarr[$i] .= $rowformat; + } } } return $sqlarr; } + /** + * Given one xmldb_table and one xmldb_field, return the SQL statements needed to add the field to the table. + * + * @param xmldb_table $xmldb_table The table related to $xmldb_field. + * @param xmldb_field $xmldb_field The instance of xmldb_field to create the SQL from. + * @param string $skip_type_clause The type clause on alter columns, NULL by default. + * @param string $skip_default_clause The default clause on alter columns, NULL by default. + * @param string $skip_notnull_clause The null/notnull clause on alter columns, NULL by default. + * @return array The SQL statement for adding a field to the table. + */ + public function getAddFieldSQL($xmldb_table, $xmldb_field, $skip_type_clause = NULL, $skip_default_clause = NULL, $skip_notnull_clause = NULL) { + $sqls = parent::getAddFieldSQL($xmldb_table, $xmldb_field, $skip_type_clause, $skip_default_clause, $skip_notnull_clause); + + if ($this->table_exists($xmldb_table)) { + $tablename = $xmldb_table->getName(); + + $size = $this->guess_antolope_row_size($this->mdb->get_columns($tablename)); + $size += $this->guess_antolope_row_size(array($xmldb_field)); + + if ($size > self::ANTELOPE_MAX_ROW_SIZE) { + if ($this->mdb->is_compressed_row_format_supported()) { + $format = strtolower($this->mdb->get_row_format($tablename)); + if ($format === 'compact' or $format === 'redundant') { + // Change the format before conversion so that we do not run out of space. + array_unshift($sqls, "ALTER TABLE {$this->prefix}$tablename ROW_FORMAT=Compressed"); + } + } + } + } + + return $sqls; + } + /** * Given one correct xmldb_table, returns the SQL statements * to create temporary table (inside one array). diff --git a/lib/ddl/tests/ddl_test.php b/lib/ddl/tests/ddl_test.php index 7320f8c4f17..bc13318ec3d 100644 --- a/lib/ddl/tests/ddl_test.php +++ b/lib/ddl/tests/ddl_test.php @@ -508,6 +508,93 @@ class core_ddl_testcase extends database_driver_testcase { } } + /** + * Test if database supports tables with many TEXT fields, + * InnoDB is known to failed during data insertion instead + * of table creation when text fields contain actual data. + */ + public function test_row_size_limits() { + + $DB = $this->tdb; // Do not use global $DB! + $dbman = $this->tdb->get_manager(); + + $text = str_repeat('ลก', 1333); + + $data = new stdClass(); + $data->name = 'test'; + $table = new xmldb_table('test_innodb'); + $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('name', XMLDB_TYPE_CHAR, '30', null, null, null, null); + $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id')); + for ($i = 0; $i < 20; $i++) { + $table->add_field('text'.$i, XMLDB_TYPE_TEXT, null, null, null, null, null); + $data->{'text'.$i} = $text; + } + $dbman->create_table($table); + + try { + $id = $DB->insert_record('test_innodb', $data); + $expected = (array)$data; + $expected['id'] = (string)$id; + $this->assertEquals($expected, (array)$DB->get_record('test_innodb', array('id' => $id)), '', 0, 10, true); + } catch (dml_exception $e) { + // Give some nice error message when known problematic MySQL with InnoDB detected. + if ($DB->get_dbfamily() === 'mysql') { + $engine = strtolower($DB->get_dbengine()); + if ($engine === 'innodb' or $engine === 'xtradb') { + if (!$DB->is_compressed_row_format_supported()) { + $this->fail("Row size limit reached in MySQL using InnoDB, configure server to use innodb_file_format=Barracuda and innodb_file_per_table=1"); + } + } + } + throw $e; + } + + $dbman->drop_table($table); + + $data = new stdClass(); + $data->name = 'test'; + $table = new xmldb_table('test_innodb'); + $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('name', XMLDB_TYPE_CHAR, '30', null, null, null, null); + $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id')); + $dbman->create_table($table); + $DB->insert_record('test_innodb', array('name' => 'test')); + + for ($i = 0; $i < 20; $i++) { + $field = new xmldb_field('text'.$i, XMLDB_TYPE_TEXT, null, null, null, null, null); + $dbman->add_field($table, $field); + $data->{'text'.$i} = $text; + + $id = $DB->insert_record('test_innodb', $data); + $expected = (array)$data; + $expected['id'] = (string)$id; + $this->assertEquals($expected, (array)$DB->get_record('test_innodb', array('id' => $id)), '', 0, 10, true); + } + + $dbman->drop_table($table); + + // MySQL VARCHAR fields may hit a different 65535 row size limit when creating tables. + $data = new stdClass(); + $data->name = 'test'; + $table = new xmldb_table('test_innodb'); + $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null); + $table->add_field('name', XMLDB_TYPE_CHAR, '30', null, null, null, null); + $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id')); + for ($i = 0; $i < 15; $i++) { + $table->add_field('text'.$i, XMLDB_TYPE_CHAR, '1333', null, null, null, null); + $data->{'text'.$i} = $text; + } + $dbman->create_table($table); + + $id = $DB->insert_record('test_innodb', $data); + $expected = (array)$data; + $expected['id'] = (string)$id; + $this->assertEquals($expected, (array)$DB->get_record('test_innodb', array('id' => $id)), '', 0, 10, true); + + $dbman->drop_table($table); + } + /** * Test behaviour of drop_table() */ diff --git a/lib/dml/mysqli_native_moodle_database.php b/lib/dml/mysqli_native_moodle_database.php index 33d07b99b70..9cc95d67c07 100644 --- a/lib/dml/mysqli_native_moodle_database.php +++ b/lib/dml/mysqli_native_moodle_database.php @@ -38,6 +38,8 @@ require_once(__DIR__.'/mysqli_native_moodle_temptables.php'); class mysqli_native_moodle_database extends moodle_database { protected $mysqli = null; + /** @var bool is compressed row format supported cache */ + protected $compressedrowformatsupported = null; private $transactions_supported = null; @@ -279,6 +281,73 @@ class mysqli_native_moodle_database extends moodle_database { return $collation; } + /** + * Get the row format from the database schema. + * + * @param string $table + * @return string row_format name or null if not known or table does not exist. + */ + public function get_row_format($table) { + $rowformat = null; + $table = $this->mysqli->real_escape_string($table); + $sql = "SELECT row_format + FROM INFORMATION_SCHEMA.TABLES + WHERE table_schema = DATABASE() AND table_name = '{$this->prefix}$table'"; + $this->query_start($sql, NULL, SQL_QUERY_AUX); + $result = $this->mysqli->query($sql); + $this->query_end($result); + if ($rec = $result->fetch_assoc()) { + $rowformat = $rec['row_format']; + } + $result->close(); + + return $rowformat; + } + + /** + * Is this database compatible with compressed row format? + * This feature is necessary for support of large number of text + * columns in InnoDB/XtraDB database. + * + * @param bool $cached use cached result + * @return bool true if table can be created or changed to compressed row format. + */ + public function is_compressed_row_format_supported($cached = true) { + if ($cached and isset($this->compressedrowformatsupported)) { + return($this->compressedrowformatsupported); + } + + $engine = strtolower($this->get_dbengine()); + $info = $this->get_server_info(); + + if (version_compare($info['version'], '5.5.0') < 0) { + // MySQL 5.1 is not supported here because we cannot read the file format. + $this->compressedrowformatsupported = false; + + } else if ($engine !== 'innodb' and $engine !== 'xtradb') { + // Other engines are not supported, most probably not compatible. + $this->compressedrowformatsupported = false; + + } else if (!$filepertable = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_per_table'")) { + $this->compressedrowformatsupported = false; + + } else if ($filepertable->value !== 'ON') { + $this->compressedrowformatsupported = false; + + } else if (!$fileformat = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_format'")) { + $this->compressedrowformatsupported = false; + + } else if ($fileformat->value !== 'Barracuda') { + $this->compressedrowformatsupported = false; + + } else { + // All the tests passed, we can safely use ROW_FORMAT=Compressed in sql statements. + $this->compressedrowformatsupported = true; + } + + return $this->compressedrowformatsupported; + } + /** * Returns localised database type name * Note: can be used before connect()