From b602463f070e4dbfcb1d3daed6351bcbe3bb4165 Mon Sep 17 00:00:00 2001 From: Dan Poltawski Date: Wed, 13 Apr 2016 18:12:12 +0100 Subject: [PATCH] MDL-53226 search_simpledb: Use databases full-text search capabilities --- search/classes/document.php | 6 ++- search/engine/simpledb/classes/engine.php | 48 ++++++++++++++++------ search/engine/simpledb/db/install.php | 50 +++++++++++++++++++++++ search/engine/simpledb/db/uninstall.php | 36 ++++++++++++++++ 4 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 search/engine/simpledb/db/install.php create mode 100644 search/engine/simpledb/db/uninstall.php diff --git a/search/classes/document.php b/search/classes/document.php index 1fea314dd80..7625f8173ce 100644 --- a/search/classes/document.php +++ b/search/classes/document.php @@ -407,7 +407,8 @@ class document implements \renderable, \templatable { * @return string */ public static function format_string_for_engine($string) { - return $string; + //FIXME: this shouldn't be required. Where is bad utf8 coming from? + return fix_utf8($string); } /** @@ -420,7 +421,8 @@ class document implements \renderable, \templatable { * @return string */ public static function format_text_for_engine($text) { - return $text; + //FIXME: this shouldn't be required. Where is bad utf8 coming from? + return fix_utf8($text); } /** diff --git a/search/engine/simpledb/classes/engine.php b/search/engine/simpledb/classes/engine.php index 47d41b15904..2bb43fb0301 100644 --- a/search/engine/simpledb/classes/engine.php +++ b/search/engine/simpledb/classes/engine.php @@ -120,16 +120,40 @@ class engine extends \core_search\engine { } // And finally the main query after applying all AND filters. - $ands[] = '(' . - $DB->sql_like('title', '?', false, false) . ' OR ' . - $DB->sql_like('content', '?', false, false) . ' OR ' . - $DB->sql_like('description1', '?', false, false) . ' OR ' . - $DB->sql_like('description2', '?', false, false) . - ')'; - $params[] = '%' . $data->q . '%'; - $params[] = '%' . $data->q . '%'; - $params[] = '%' . $data->q . '%'; - $params[] = '%' . $data->q . '%'; + switch ($DB->get_dbfamily()) { + case 'postgres': + $ands[] = "(" . + "to_tsvector('simple', title) @@ plainto_tsquery(?) OR ". + "to_tsvector('simple', content) @@ plainto_tsquery(?) OR ". + "to_tsvector('simple', description1) @@ plainto_tsquery(?) OR ". + "to_tsvector('simple', description2) @@ plainto_tsquery(?)". + ")"; + $params[] = $data->q; + $params[] = $data->q; + $params[] = $data->q; + $params[] = $data->q; + break; + case 'mysql': + $ands[] = "MATCH (title, content, description1, description2) AGAINST (?)"; + $params[] = $data->q; + break; + case 'mssql': + $ands[] = "CONTAINS ((title, content, description1, description2), ?)"; + $params[] = $data->q; + break; + default: + $ands[] = '(' . + $DB->sql_like('title', '?', false, false) . ' OR ' . + $DB->sql_like('content', '?', false, false) . ' OR ' . + $DB->sql_like('description1', '?', false, false) . ' OR ' . + $DB->sql_like('description2', '?', false, false) . + ')'; + $params[] = '%' . $data->q . '%'; + $params[] = '%' . $data->q . '%'; + $params[] = '%' . $data->q . '%'; + $params[] = '%' . $data->q . '%'; + break; + } $recordset = $DB->get_recordset_sql($sql . implode(' AND ', $ands), $params, 0, \core_search\manager::MAX_RESULTS); @@ -201,8 +225,8 @@ class engine extends \core_search\engine { $DB->insert_record('search_simpledb_index', $doc); } - } catch (dml_exception $ex) { - debugging('dml error while trying to insert document with id ' . $doc->docid . ': ' . $e->getMessage(), + } catch (\dml_exception $ex) { + debugging('dml error while trying to insert document with id ' . $doc->docid . ': ' . $ex->getMessage(), DEBUG_DEVELOPER); } diff --git a/search/engine/simpledb/db/install.php b/search/engine/simpledb/db/install.php new file mode 100644 index 00000000000..f7f06e37c02 --- /dev/null +++ b/search/engine/simpledb/db/install.php @@ -0,0 +1,50 @@ +. + +/** + * Post installation and migration code. + * + * @package search_simpledb + * @copyright 2016 Dan Poltawski + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die; + +function xmldb_search_simpledb_install() { + global $DB; + + switch ($DB->get_dbfamily()) { + case 'postgres': + // TODO: There are a few other ways of doing this which avoid the need for individual indicies.. + $DB->execute("CREATE INDEX psql_search_title ON {search_simpledb_index} USING gin(to_tsvector('simple', title))"); + $DB->execute("CREATE INDEX psql_search_content ON {search_simpledb_index} USING gin(to_tsvector('simple', content))"); + $DB->execute("CREATE INDEX psql_search_description1 ON {search_simpledb_index} USING gin(to_tsvector('simple', description1))"); + $DB->execute("CREATE INDEX psql_search_description2 ON {search_simpledb_index} USING gin(to_tsvector('simple', description2))"); + break; + case 'mysql': + $DB->execute("CREATE FULLTEXT INDEX mysql_search_index + ON {search_simpledb_index} (title, content, description1, description2)"); + break; + case 'mssql': + //TODO: workout if fulltext search is installed... select SERVERPROPERTY('IsFullTextInstalled') + $DB->execute("CREATE FULLTEXT CATALOG {search_simpledb_catalog}"); + $DB->execute("CREATE FULLTEXT INDEX ON {search_simpledb_index} (title, content, description1, description2) + KEY INDEX {searsimpinde_id_pk} ON {search_simpledb_catalog}"); + break; + } +} + diff --git a/search/engine/simpledb/db/uninstall.php b/search/engine/simpledb/db/uninstall.php new file mode 100644 index 00000000000..6acbba52cb4 --- /dev/null +++ b/search/engine/simpledb/db/uninstall.php @@ -0,0 +1,36 @@ +. + +/** + * Uninstall code. + * + * @package search_simpledb + * @copyright 2016 Dan Poltawski + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die; + +function xmldb_search_simpledb_uninstall() { + global $DB; + + switch ($DB->get_dbfamily()) { + case 'mssql': + $DB->execute("DROP FULLTEXT CATALOG {search_simpledb_catalog}"); + break; + } +} +