From 4153a7483dba9e60753881a6b8484101a1a61969 Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Tue, 17 Jun 2014 16:53:36 +1200 Subject: [PATCH 1/2] MDL-45985 ddl: improve check_database_schema method in database_manager --- lib/ddl/database_manager.php | 183 +++++++++++++++++++++++++++------- lib/dtl/database_exporter.php | 3 +- lib/dtl/database_importer.php | 3 +- 3 files changed, 150 insertions(+), 39 deletions(-) diff --git a/lib/ddl/database_manager.php b/lib/ddl/database_manager.php index 741f7904363..eb940766699 100644 --- a/lib/ddl/database_manager.php +++ b/lib/ddl/database_manager.php @@ -904,67 +904,176 @@ class database_manager { /** * Checks the database schema against a schema specified by an xmldb_structure object * @param xmldb_structure $schema export schema describing all known tables + * @param array $options * @return array keyed by table name with array of difference messages as values */ - public function check_database_schema(xmldb_structure $schema) { + public function check_database_schema(xmldb_structure $schema, array $options = null) { + $alloptions = array( + 'extratables' => true, + 'missingtables' => true, + 'extracolumns' => true, + 'missingcolumns' => true, + 'changedcolumns' => true, + ); + + $typesmap = array( + 'I' => XMLDB_TYPE_INTEGER, + 'R' => XMLDB_TYPE_INTEGER, + 'N' => XMLDB_TYPE_NUMBER, + 'F' => XMLDB_TYPE_NUMBER, // Nobody should be using floats! + 'C' => XMLDB_TYPE_CHAR, + 'X' => XMLDB_TYPE_TEXT, + 'B' => XMLDB_TYPE_BINARY, + 'T' => XMLDB_TYPE_TIMESTAMP, + 'D' => XMLDB_TYPE_DATETIME, + ); + + $options = (array)$options; + $options = array_merge($alloptions, $options); + + // Note: the error descriptions are not supposed to be localised, + // it is intended for developers and skilled admins only. $errors = array(); - $dbtables = $this->mdb->get_tables(); - $tables = $schema->getTables(); + /** @var string[] $dbtables */ + $dbtables = $this->mdb->get_tables(false); + /** @var xmldb_table[] $tables */ + $tables = $schema->getTables(); - //TODO: maybe add several levels error/warning - - // make sure that current and schema tables match exactly foreach ($tables as $table) { $tablename = $table->getName(); - if (empty($dbtables[$tablename])) { - if (!isset($errors[$tablename])) { - $errors[$tablename] = array(); + + if ($options['missingtables']) { + // Missing tables are a fatal problem. + if (empty($dbtables[$tablename])) { + $errors[$tablename][] = "table is missing"; + continue; } - $errors[$tablename][] = "Table $tablename is missing in database."; //TODO: localize - continue; } - // a) check for required fields - $dbfields = $this->mdb->get_columns($tablename); - $fields = $table->getFields(); + /** @var database_column_info[] $dbfields */ + $dbfields = $this->mdb->get_columns($tablename, false); + /** @var xmldb_field[] $fields */ + $fields = $table->getFields(); + foreach ($fields as $field) { $fieldname = $field->getName(); if (empty($dbfields[$fieldname])) { - if (!isset($errors[$tablename])) { - $errors[$tablename] = array(); + if ($options['missingcolumns']) { + // Missing columns are a fatal problem. + $errors[$tablename][] = "column '$fieldname' is missing"; + } + } else if ($options['changedcolumns']) { + $dbfield = $dbfields[$fieldname]; + + if (!isset($typesmap[$dbfield->meta_type])) { + $errors[$tablename][] = "column '$fieldname' has unsupported type '$dbfield->meta_type'"; + } else { + $dbtype = $typesmap[$dbfield->meta_type]; + $type = $field->getType(); + if ($type == XMLDB_TYPE_FLOAT) { + $type = XMLDB_TYPE_NUMBER; + } + if ($type != $dbtype) { + if ($expected = array_search($type, $typesmap)) { + $errors[$tablename][] = "column '$fieldname' has incorrect type '$dbfield->meta_type', expected '$expected'"; + } else { + $errors[$tablename][] = "column '$fieldname' has incorrect type '$dbfield->meta_type'"; + } + } else { + if ($field->getNotNull() != $dbfield->not_null) { + if ($field->getNotNull()) { + $errors[$tablename][] = "column '$fieldname' should be NOT NULL ($dbfield->meta_type)"; + } else { + $errors[$tablename][] = "column '$fieldname' should allow NULL ($dbfield->meta_type)"; + } + } + if ($dbtype == XMLDB_TYPE_TEXT) { + // No length check necessary - there is one size only now. + + } else if ($dbtype == XMLDB_TYPE_NUMBER) { + if ($field->getType() == XMLDB_TYPE_FLOAT) { + // Do not use floats in any new code, they are deprecated in XMLDB editor! + + } else if ($field->getLength() != $dbfield->max_length or $field->getDecimals() != $dbfield->scale) { + $size = "({$field->getLength()},{$field->getDecimals()})"; + $dbsize = "($dbfield->max_length,$dbfield->scale)"; + $errors[$tablename][] = "column '$fieldname' size is $dbsize, expected $size ($dbfield->meta_type)"; + } + + } else if ($dbtype == XMLDB_TYPE_CHAR) { + // This is not critical, but they should ideally match. + if ($field->getLength() != $dbfield->max_length) { + $errors[$tablename][] = "column '$fieldname' length is $dbfield->max_length, expected {$field->getLength()} ($dbfield->meta_type)"; + } + + } else if ($dbtype == XMLDB_TYPE_INTEGER) { + // Integers may be bigger in some DBs. + $length = $field->getLength(); + if ($length > 18) { + // Integers are not supposed to be bigger than 18. + $length = 18; + } + if ($length > $dbfield->max_length) { + $errors[$tablename][] = "column '$fieldname' length is $dbfield->max_length, expected at least {$field->getLength()} ($dbfield->meta_type)"; + } + + } else if ($dbtype == XMLDB_TYPE_BINARY) { + // Ignore binary types. + continue; + + } else if ($dbtype == XMLDB_TYPE_TIMESTAMP) { + $errors[$tablename][] = "column '$fieldname' is a timestamp, this type is not supported ($dbfield->meta_type)"; + continue; + + } else if ($dbtype == XMLDB_TYPE_DATETIME) { + $errors[$tablename][] = "column '$fieldname' is a datetime, this type is not supported ($dbfield->meta_type)"; + continue; + + } else { + // Report all other unsupported types as problems. + $errors[$tablename][] = "column '$fieldname' has unknown type ($dbfield->meta_type)"; + continue; + } + + // Note: The empty string defaults are a bit messy... + if ($field->getDefault() != $dbfield->default_value) { + $default = is_null($field->getDefault()) ? 'NULL' : $field->getDefault(); + $dbdefault = is_null($dbfield->default_value) ? 'NULL' : $dbfield->default_value; + $errors[$tablename][] = "column '$fieldname' has default '$dbdefault', expected '$default' ($dbfield->meta_type)"; + } + } } - $errors[$tablename][] = "Field $fieldname is missing in table $tablename."; //TODO: localize } unset($dbfields[$fieldname]); } - // b) check for extra fields (indicates unsupported hacks) - modify install.xml if you want the script to continue ;-) - foreach ($dbfields as $fieldname=>$info) { - if (!isset($errors[$tablename])) { - $errors[$tablename] = array(); + // Check for extra columns (indicates unsupported hacks) - modify install.xml if you want to pass validation. + foreach ($dbfields as $fieldname => $dbfield) { + if ($options['extracolumns']) { + $errors[$tablename][] = "column '$fieldname' is not expected ($dbfield->meta_type)"; } - $errors[$tablename][] = "Field $fieldname is not expected in table $tablename."; //TODO: localize } unset($dbtables[$tablename]); } - // look for unsupported tables - local custom tables should be in /local/xxxx/db/install.xml ;-) - // if there is no prefix, we can not say if tale is ours :-( - if ($this->generator->prefix !== '') { - foreach ($dbtables as $tablename=>$unused) { - if (strpos($tablename, 'pma_') === 0) { - // ignore phpmyadmin tables for now - continue; + if ($options['extratables']) { + // Look for unsupported tables - local custom tables should be in /local/xxxx/db/install.xml file. + // If there is no prefix, we can not say if table is ours, sorry. + if ($this->generator->prefix !== '') { + foreach ($dbtables as $tablename => $unused) { + if (strpos($tablename, 'pma_') === 0) { + // Ignore phpmyadmin tables. + continue; + } + if (strpos($tablename, 'test') === 0) { + // Legacy simple test db tables need to be eventually removed, + // report them as problems! + $errors[$tablename][] = "table is not expected (it may be a leftover after Simpletest unit tests)"; + } else { + $errors[$tablename][] = "table is not expected"; + } } - if (strpos($tablename, 'test') === 0) { - // ignore broken results of unit tests - continue; - } - if (!isset($errors[$tablename])) { - $errors[$tablename] = array(); - } - $errors[$tablename][] = "Table $tablename is not expected."; //TODO: localize } } diff --git a/lib/dtl/database_exporter.php b/lib/dtl/database_exporter.php index 206da10efec..23677f6e83c 100644 --- a/lib/dtl/database_exporter.php +++ b/lib/dtl/database_exporter.php @@ -129,7 +129,8 @@ abstract class database_exporter { public function export_database($description=null) { global $CFG; - if ($this->check_schema and $errors = $this->manager->check_database_schema($this->schema)) { + $options = array('changedcolumns' => false); // Column types may be fixed by transfer. + if ($this->check_schema and $errors = $this->manager->check_database_schema($this->schema, $options)) { $details = ''; foreach ($errors as $table=>$items) { $details .= '
'.get_string('tablex', 'dbtransfer', $table); diff --git a/lib/dtl/database_importer.php b/lib/dtl/database_importer.php index d9cf1a7087d..b14bad71416 100644 --- a/lib/dtl/database_importer.php +++ b/lib/dtl/database_importer.php @@ -110,7 +110,8 @@ class database_importer { throw new dbtransfer_exception('importversionmismatchexception', $a); } - if ($this->check_schema and $errors = $this->manager->check_database_schema($this->schema)) { + $options = array('changedcolumns' => false); // Column types may be fixed by transfer. + if ($this->check_schema and $errors = $this->manager->check_database_schema($this->schema, $options)) { $details = ''; foreach ($errors as $table=>$items) { $details .= '
'.get_string('table').' '.$table.':'; From 4f4fb38cb583ea51766fcae314eb877712ed7e3e Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Tue, 17 Jun 2014 16:54:31 +1200 Subject: [PATCH 2/2] MDL-45985 cli: add new cli tool for validation of database structure --- admin/cli/check_database_schema.php | 76 +++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 admin/cli/check_database_schema.php diff --git a/admin/cli/check_database_schema.php b/admin/cli/check_database_schema.php new file mode 100644 index 00000000000..3164a27e22a --- /dev/null +++ b/admin/cli/check_database_schema.php @@ -0,0 +1,76 @@ +. + +/** + * Validate that the current db structure matches the install.xml files. + * + * @package core + * @copyright 2014 Totara Learning Solutions Ltd {@link http://www.totaralms.com/} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @author Petr Skoda + */ + +define('CLI_SCRIPT', true); + +require(__DIR__ . '/../../config.php'); +require_once($CFG->libdir.'/clilib.php'); + +$help = "Validate database structure + +Options: +-h, --help Print out this help. + +Example: +\$ sudo -u www-data /usr/bin/php admin/cli/check_database_schema.php +"; + +list($options, $unrecognized) = cli_get_params( + array( + 'help' => false, + ), + array( + 'h' => 'help', + ) +); + +if ($options['help']) { + echo $help; + exit(0); +} + +if (empty($CFG->version)) { + echo "Database is not yet installed.\n"; + exit(2); +} + +$dbmanager = $DB->get_manager(); +$schema = $dbmanager->get_install_xml_schema(); + +if (!$errors = $dbmanager->check_database_schema($schema)) { + echo "Database structure is ok.\n"; + exit(0); +} + +foreach ($errors as $table => $items) { + cli_separator(); + echo "$table\n"; + foreach ($items as $item) { + echo " * $item\n"; + } +} +cli_separator(); + +exit(1);