diff --git a/search/LISEZMOI.txt b/search/LISEZMOI.txt index b2cc8425549..16e4450b968 100644 --- a/search/LISEZMOI.txt +++ b/search/LISEZMOI.txt @@ -2,59 +2,56 @@ Cette distribution partielle contient une refonte du moteur de recherche globale de Moodle. Le moteur de recherche est capable d'indexer et de rechercher -des informations dans un grand nombre de contenus stockés -dans la plate-forme à travers la manipulation des activités et +des informations dans un grand nombre de contenus stockés +dans la plate-forme à travers la manipulation des activités et des blocs. -Le moteur de recherche procède à une première indexation des +Le moteur de recherche procède à une première indexation des ressources disponibles par action de l'administrateur. Une fois -cette indexation effectuée, le moteur maintient régulièrement les -indexes, en ajoutant les nouvelles entrées et en nettoyant les -entrées obsolètes. +cette indexation effectuée, le moteur maintient régulièrement les +indexes, en ajoutant les nouvelles entrées et en nettoyant les +entrées obsolètes. -La recherche permet d'obtenir des références d'accès au contexte +La recherche permet d'obtenir des références d'accès au contexte qui diffuse cette information, au nom de l'utilisateur courant. -Le filtrage des résultats enlève de la liste des réponses toute -ressource que la situation de l'utilisateur empêcherait de voir -s'il y accédait dans son contexte habituel. +Le filtrage des résultats enlève de la liste des réponses toute +ressource que la situation de l'utilisateur empêcherait de voir +s'il y accédait dans son contexte habituel. Mise en oeuvre ############## -Pour déployer le moteur : +La distribution fait désormais partie du noyau de Moodle. +Il sera probablement nécessaire d'ajouter un certain nombre de librairies additionnelles +pour la conversion de documents physiques en vue de leur indexation. Ces librairies sont +actuellement fournies dans le CVS dans la rubrique contrib/patches/global_search_libraries +(antiword et xpdf). La prise en charge des fichiers "shockwave" est assurée, sous réserve +de l'obtention des libairies de conversion auprès de Adobe (http://www.adobe.com/licensing/developer/) -* Copie de fichiers +1. Allez sur le bloc d'administration et réglez les paramètres du bloc Recherche Globale. +Ceci initialisera un certain nombre de fonctions dans le moteur. -1. Ajouter les deux librairies fournies aux librairies de Moodle -2. Ecraser le répertoire "search" par le répertoire fourni -3. Ecraser le bloc "blocs/search" par le bloc fourni. +2. Insérer un nouveau bloc de recherche globale dans la plate-forme -* Installation logique +3. Effectuer une recherche vide (en administrateur) -4. Aller dans les notifications administratives et dérouler la procédure d'installation/mise à jour du bloc. L'installation crée la table image -des documents indexés et utilisés dans le module search. +4. Aller sur la page des statistiques -5. Insérer un nouveau bloc de recherche globale dans la plate-forme +5. Activer l'indexation (indexsplash.php). Attention, si la plate-form contient beaucoup de contenus cette indexation peut être TRES LONGUE. -6. Effectuer une recherche vide (en administrateur) +Pour effectuer des recherches, une fois la première indexation terminée, retourner au bloc de recherche et tenter une recherche. -7. Aller sur la page des statistiques - -8. Activer l'indexation (indexsplash.php). Attention, si la plate-form contient beaucoup de contenus cette indexation peut être TRES LONGUE. - -Pour effectuer des recherches, une fois la première indexation terminée, retourner au bloc de recherche et tenter une recherche. - -Eléments pris en charge +Eléments pris en charge ####################### -Dans l'état actuel, les éléments indexés par le moteur sont : +Dans l'état actuel, les éléments indexés par le moteur sont : -- les entrées de forum -- les fiches de base de données -- les commentaires sur fiches de données -- les entrées de glossaire -- les commentaires sur entrées de glossaire +- les entrées de forum +- les fiches de base de données +- les commentaires sur fiches de données +- les entrées de glossaire +- les commentaires sur entrées de glossaire - les ressources natives Moodle - les ressources physiques de type MSWord - les ressources physiques de type PDF @@ -63,27 +60,30 @@ Dans l'état actuel, les éléments indexés par le moteur sont : - les ressources physiques de type XML (.xml) - les ressources physiques de type (Microsoft) Powerpoint (.ppt) - les pages de wiki -- les entités de projet technique - les sessions de chat +Des modules tiers ont été rendus indexables + +- Techproject + Extensions ########## -L'API du moteur de recherche permet désormais : +L'API du moteur de recherche permet désormais : - l'indexation de contenus de blocs. - l'indexation de modules contenant une information complexe ou de plusieurs types distincts -- la sécurisation des informations indexées lors des extractions de résultats -- l'indexation de tout module tiers par ajout d'un fichier php calibré -- l'indexation de toute nouvelle resource physique par ajout d'un fichier php calibré +- la sécurisation des informations indexées lors des extractions de résultats +- l'indexation de tout module tiers par ajout d'un fichier php calibré +- l'indexation de toute nouvelle resource physique par ajout d'un fichier php calibré Extensions futures ################## - De nouvelles prises en charge de contenus tels que les attachements des forums, les attachement des glossaires, ainsi que d'autres modules non encore -implémentés. +implémentés. -- l'extension mnet de la recherche dans un réseau de moodle interconnectés. +- l'extension mnet de la recherche dans un réseau de moodle interconnectés. diff --git a/search/README.txt b/search/README.txt index 64296be2f0b..a680fcf5601 100644 --- a/search/README.txt +++ b/search/README.txt @@ -1,153 +1,92 @@ -2006/09/08 ----------- -Google Summer of Code is finished, spent a couple of weeks away from -the project to think about it and also to take a break. Working on it -now I discovered bugs in the query parser (now fixed), and I also -un-convoluted the querylib logic (well slighlty). +This directoery contains the central implementation of +Moodle's Global Search Engine. -Updated ZFS files to latest SVN. +The Global Search Engine stores indexes about a huge quantity +of information from within modules, block or resources stored +by Moodle either in the database or the file system. -2006/08/21 ----------- -Fixed index document count, and created new config variable to store -the size. (Search now has 3 global vars in $CFG, date, size and complete, -see indexer.php for var names). Index size is cached to provide an always -current value for the index - this is to take into account the fact that -deleted documents are in fact not removed from the index, but instead just -marked as deleted and not returned in search results. The actual document -still features in the index, and skews sizes. When the index optimiser is -completed in ZFS, then these deleted documents will be pruned, thus -correctly modifying the index size. +The administrator initialy indexes the existing content. Once this +first initialization performed, the search engine maintains indexes +regularily, adding new entries, deleting obsolete one or updating +some that have changed. -Additional commenting added. +Search will produce links for acceding the information in a similar +context as usually accessed, from the current user point of view. +Results filtering removes from results any link to information the +current user would not be allowed to acces on a straight situation. -Query page logic very slightly modified to clean up GET string a bit (removed -'p' variable). +Deployment +########### -Add/delete functions added to other document types. +The search engine is now part of Moodle core distribution. -A few TODO fields added to source, indicating changes still to come (or at -least to be considered). +Some extra libraries might be added for converting physical documents to text +so it can be indexed. Moodle CVS (entry contrib/patches/global_search_libraries) +provides packs for antiword and xpdf GPL libraries the search engine is ready for +shockwave indexing, but will not provide Adobe Search converters that should be +obtained at http://www.adobe.com/licensing/developer/ -2006/08/16 ----------- -Add/delete/update cron functions finished - can be called seperately -or all at once via cron.php. +1. Go to the block administration panel and setup once the Global Search +block. This will initialize useful parameters for the global search engine. -Document date field added to index and database summary. +2. Insert a new Global Search block somewhere in a course or top-level screen. -Some index db functionality abstracted out to indexlib.php - can -use IndexDBControl class to add/del documents from database, and -to make sure the db table is functioning. +3. Launch an empty search (you must be administrator). -DB sql files changed to add some extra fields. +4. Go to the statistics screen. -Default 'simple' query modified to search title and author, as well -as contents of document, to provide better results for users. +5. Activate indexation (indexersplash.php). Beware, if your Moodle has +a large amount of content, indexing process may be VERY LONG. -2006/08/14 ----------- -First revision of the advanced search page completed. Functional, -but needs a date search field still. +To search, go back to the search block and try a query. -2006/08/02 ----------- -Added resource search type, and the ability to specify custom 'virtual' -models to search - allowing for non-module specific information to be -indexed. Specify the extra search types to use in lib.php. +Handled information for indexing +################################ -2006/07/28 ----------- -Added delete logic to documents; the moodle database log is checked -and any found delete events are used to remove the referenced documents -from the database table and search index. +In the actual state, the engine indexes the following information: -Added database table name constant to lib.php, must change files using -the static table name. +- assignment descriptions +- forum posts +- database records (using textual fields only) +- database comments +- glossary entries +- glossary comments on entries +- Moodle native resources +- physical MSWord files as resources (.doc) +- physical Powerpoint files as resources (.ppt) +- physical PDF files as resources +- physical text files as resources (.txt) +- physical html files as resources (.htm and .html) +- physical xml files as resources (.xml) +- wiki pages +- chat sessions +- lesson pages -Changed documents to use 'docid' instead of 'id' to reference the moodle -instance id, since Zend Search adds it's own internal 'id' field. Noticed -this whilst working on deletions. +Some third party plugins are also searchable using the new Search API implementation -Added some additional fields to the permissions checking method, must still -implement it though. +- Techproject -2006/07/25 ----------- -Query logic moved into the SearchQuery class in querylib.php. Should be able -to include this file in any page and run a query against the index (PHP 5 -checks must be added to those pages then, though). +Extensions +########## -Index info can be retrieved using IndexInfo class in indexlib.php. +The reviewed search engine API allows: -Abstracted some stuff away, to reduce rendundancy and decrease the -likelihood of errors. Improved the stats.php page to include some -diagnostics for adminstrators. +- indexing of blocks contents +- indexation of modules or blocks containing a complex information model +- securing the access to the results +- adding indexing handling for additional modules and plugins adding a php calibrated script +- adding physical filetype handling adding a php calibrated script -delete.php skeleton created for removing deleted documents from the -index. cron.php will contain the logic for running delete.php, -update.php and eventually add.php. +Global Search on NFS Mounted clusters +##################################### -2006/07/11 ----------- -(Warning: It took me 1900 seconds to index the forum, go make coffee -whilst you wait.) [Moodle.org forum data] +This version contains a patched Lucene Zend implementation that allows using the Global Search engine in an NFS mounted shared volume for Web clustering. This implementation +remains highly experimental and not all tests have been processed. Some changes may +occur in the SoftLockManager that was added to the Lucene engine. -Forum search functions changed to use 'get_recordset' instead of -'get_records', for speed reasons. This provides a significant improvement, -but indexing is still slow - getting data from the database and Zend's -tokenising _seem_ to be the prime suspects at the moment. +Future extensions +################# -/search/tests/ added - index.php can be used to see which modules are -ready to be included in the search index, and it informs you of any -errors - should be a prerequisite for indexing. - -Search result pagination added to query.php, will default to 20 until -an admin page for the search module is written. - -2006/07/07 ----------- -Search-enabling functions moved out've the mod's lib.php files and into -/search/documents/mod_document.php - this requires the search module to -operate without requiring modification of lib files. - -SearchDocument base class improved, and the way module documents extend -it. A custom-data field has been added to allow modules to add any custom -data they wish to be stored in the index - this field is serialised into -the index as a binary field. - -Database field 'type' renamed to 'doctype' to match the renaming in the -index, 'type' seems to be a reserved word in Lucene. Several index field -names change to be more descriptive (cid -> course_id). URLs are now -stored in the index, and don't have to be generated on the fly during -display of query results. - -2006/07/05 ------- -Started cleaning and standardising things. - -cvs v1.1 --------- -This is the initial release (prototype) of Moodle's new search module - -so basically watch out for sharp edges. - -The structure has not been finalised, but this is what is working at the -moment, when I start looking at other content to index, it will most likely -change. I don't recommend trying to make your own content modules indexable, -at least not until the whole flow is finalised. I will be implementing the -functions needed to index all of the default content modules on Moodle, so -expect that around mid-August. - -Wiki pages were my goal for this release, they can be indexed and searched, -but not updated or deleted at this stage (was waiting for ZF 0.14 actually). - -I need to check the PostgreSQL sql file, I don't have a PG7 install lying -around to test on, so the script is untested. - -To index for the first time, login as an admin user and browse to /search/index.php -or /search/stats.php - there will be a message and a link telling you to go index. - --- Michael Champanis (mchampan) - email: cynnical@gmail.com - skype: mchampan - Summer of Code 2006 \ No newline at end of file +- Should be added more information to index such as forum and glossary attachements, + so will other standard module contents. +- extending the search capability to a mnet network information space by aggregating remote search responses. \ No newline at end of file diff --git a/search/README_ARCHIVE.txt b/search/README_ARCHIVE.txt new file mode 100644 index 00000000000..64296be2f0b --- /dev/null +++ b/search/README_ARCHIVE.txt @@ -0,0 +1,153 @@ +2006/09/08 +---------- +Google Summer of Code is finished, spent a couple of weeks away from +the project to think about it and also to take a break. Working on it +now I discovered bugs in the query parser (now fixed), and I also +un-convoluted the querylib logic (well slighlty). + +Updated ZFS files to latest SVN. + +2006/08/21 +---------- +Fixed index document count, and created new config variable to store +the size. (Search now has 3 global vars in $CFG, date, size and complete, +see indexer.php for var names). Index size is cached to provide an always +current value for the index - this is to take into account the fact that +deleted documents are in fact not removed from the index, but instead just +marked as deleted and not returned in search results. The actual document +still features in the index, and skews sizes. When the index optimiser is +completed in ZFS, then these deleted documents will be pruned, thus +correctly modifying the index size. + +Additional commenting added. + +Query page logic very slightly modified to clean up GET string a bit (removed +'p' variable). + +Add/delete functions added to other document types. + +A few TODO fields added to source, indicating changes still to come (or at +least to be considered). + +2006/08/16 +---------- +Add/delete/update cron functions finished - can be called seperately +or all at once via cron.php. + +Document date field added to index and database summary. + +Some index db functionality abstracted out to indexlib.php - can +use IndexDBControl class to add/del documents from database, and +to make sure the db table is functioning. + +DB sql files changed to add some extra fields. + +Default 'simple' query modified to search title and author, as well +as contents of document, to provide better results for users. + +2006/08/14 +---------- +First revision of the advanced search page completed. Functional, +but needs a date search field still. + +2006/08/02 +---------- +Added resource search type, and the ability to specify custom 'virtual' +models to search - allowing for non-module specific information to be +indexed. Specify the extra search types to use in lib.php. + +2006/07/28 +---------- +Added delete logic to documents; the moodle database log is checked +and any found delete events are used to remove the referenced documents +from the database table and search index. + +Added database table name constant to lib.php, must change files using +the static table name. + +Changed documents to use 'docid' instead of 'id' to reference the moodle +instance id, since Zend Search adds it's own internal 'id' field. Noticed +this whilst working on deletions. + +Added some additional fields to the permissions checking method, must still +implement it though. + +2006/07/25 +---------- +Query logic moved into the SearchQuery class in querylib.php. Should be able +to include this file in any page and run a query against the index (PHP 5 +checks must be added to those pages then, though). + +Index info can be retrieved using IndexInfo class in indexlib.php. + +Abstracted some stuff away, to reduce rendundancy and decrease the +likelihood of errors. Improved the stats.php page to include some +diagnostics for adminstrators. + +delete.php skeleton created for removing deleted documents from the +index. cron.php will contain the logic for running delete.php, +update.php and eventually add.php. + +2006/07/11 +---------- +(Warning: It took me 1900 seconds to index the forum, go make coffee +whilst you wait.) [Moodle.org forum data] + +Forum search functions changed to use 'get_recordset' instead of +'get_records', for speed reasons. This provides a significant improvement, +but indexing is still slow - getting data from the database and Zend's +tokenising _seem_ to be the prime suspects at the moment. + +/search/tests/ added - index.php can be used to see which modules are +ready to be included in the search index, and it informs you of any +errors - should be a prerequisite for indexing. + +Search result pagination added to query.php, will default to 20 until +an admin page for the search module is written. + +2006/07/07 +---------- +Search-enabling functions moved out've the mod's lib.php files and into +/search/documents/mod_document.php - this requires the search module to +operate without requiring modification of lib files. + +SearchDocument base class improved, and the way module documents extend +it. A custom-data field has been added to allow modules to add any custom +data they wish to be stored in the index - this field is serialised into +the index as a binary field. + +Database field 'type' renamed to 'doctype' to match the renaming in the +index, 'type' seems to be a reserved word in Lucene. Several index field +names change to be more descriptive (cid -> course_id). URLs are now +stored in the index, and don't have to be generated on the fly during +display of query results. + +2006/07/05 +------ +Started cleaning and standardising things. + +cvs v1.1 +-------- +This is the initial release (prototype) of Moodle's new search module - +so basically watch out for sharp edges. + +The structure has not been finalised, but this is what is working at the +moment, when I start looking at other content to index, it will most likely +change. I don't recommend trying to make your own content modules indexable, +at least not until the whole flow is finalised. I will be implementing the +functions needed to index all of the default content modules on Moodle, so +expect that around mid-August. + +Wiki pages were my goal for this release, they can be indexed and searched, +but not updated or deleted at this stage (was waiting for ZF 0.14 actually). + +I need to check the PostgreSQL sql file, I don't have a PG7 install lying +around to test on, so the script is untested. + +To index for the first time, login as an admin user and browse to /search/index.php +or /search/stats.php - there will be a message and a link telling you to go index. + +-- Michael Champanis (mchampan) + email: cynnical@gmail.com + skype: mchampan + Summer of Code 2006 \ No newline at end of file diff --git a/search/add.php b/search/add.php index a1edd3ebd8a..ca5dc7860d2 100644 --- a/search/add.php +++ b/search/add.php @@ -1,25 +1,31 @@ 1.8 -* @date 2008/03/31 -* @license http://www.gnu.org/copyleft/gpl.html GNU Public License -* -* Asynchronous adder for new indexable contents -* -* Major chages in this review is passing the xxxx_db_names return to -* multiple arity to handle multiple document types modules -*/ + /** + * Global Search Engine for Moodle + * + * @package search + * @category core + * @subpackage search_engine + * @author Michael Champanis (mchampan) [cynnical@gmail.com], Valery Fremaux [valery.fremaux@club-internet.fr] > 1.8 + * @date 2008/03/31 + * @license http://www.gnu.org/copyleft/gpl.html GNU Public License + * + * Asynchronous adder for new indexable contents + * + * Major chages in this review is passing the xxxx_db_names return to + * multiple arity to handle multiple document types modules + */ + + /** + * includes and requires + */ + require_once('../config.php'); -/** -* includes and requires -*/ -require_once('../config.php'); -require_once("$CFG->dirroot/search/lib.php"); +/// makes inclusions of the Zend Engine more reliable + $separator = (array_key_exists('WINDIR', $_SERVER)) ? ';' : ':' ; + ini_set('include_path', $CFG->dirroot.'/search'.$separator.ini_get('include_path')); + + require_once("$CFG->dirroot/search/lib.php"); + require_once("$CFG->dirroot/search/indexlib.php"); /// checks global search activation @@ -29,21 +35,18 @@ require_once("$CFG->dirroot/search/lib.php"); error(get_string('globalsearchdisabled', 'search')); } - if (!isadmin()) { + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) { error(get_string('beadmin', 'search'), "$CFG->wwwroot/login/index.php"); } /// check for php5 (lib.php) - if (!search_check_php5()) { - $phpversion = phpversion(); - mtrace("Sorry, global search requires PHP 5.0.0 or later (currently using version ".phpversion().")"); - exit(0); - } - - require_once("$CFG->dirroot/search/indexlib.php"); - - $index = new Zend_Search_Lucene(SEARCH_INDEX_PATH); + try { + $index = new Zend_Search_Lucene(SEARCH_INDEX_PATH); + } catch(LuceneException $e) { + mtrace("Could not construct a valid index. Maybe the first indexation was never made, or files might be corrupted. Run complete indexation again."); + return; + } $dbcontrol = new IndexDBControl(); $addition_count = 0; $startindextime = time(); @@ -76,7 +79,7 @@ require_once("$CFG->dirroot/search/lib.php"); $valuesArray = $db_names_function(); if ($valuesArray){ foreach($valuesArray as $values){ - $where = (isset($values[5])) ? 'AND ('.$values[5].')' : ''; + $where = (!empty($values[5])) ? 'AND ('.$values[5].')' : ''; $itemtypes = ($values[4] != '*' && $values[4] != 'any') ? " AND itemtype = '{$values[4]}' " : '' ; //select records in MODULE table, but not in SEARCH_DATABASE_TABLE @@ -88,10 +91,10 @@ require_once("$CFG->dirroot/search/lib.php"); FROM {$CFG->prefix}{$table} WHERE - doctype = '{$mod->name}' + doctype = '{$values[1]}' $itemtypes "; - $docIds = get_records_sql_menu($query); + $docIds = get_records_sql_menu($query, array($mod->name)); $docIdList = ($docIds) ? implode("','", array_keys($docIds)) : '' ; $query = " @@ -100,7 +103,7 @@ require_once("$CFG->dirroot/search/lib.php"); FROM {$CFG->prefix}{$values[1]} WHERE - id NOT IN ('{$docIdList}') and + id NOT IN ('{$docIdList}') AND {$values[2]} > {$indexdate} $where "; @@ -155,4 +158,4 @@ require_once("$CFG->dirroot/search/lib.php"); mtrace("Added $addition_count documents."); mtrace('Index size after: '.$index->count()); -?> +?> \ No newline at end of file diff --git a/search/cron.php b/search/cron.php index 017c2fd45ae..dd04ae260a5 100644 --- a/search/cron.php +++ b/search/cron.php @@ -18,11 +18,7 @@ require_once("$CFG->dirroot/search/lib.php"); - if (!search_check_php5()) { - $phpversion = phpversion(); - mtrace("Sorry, cannot cron global search as it requires PHP 5.0.0 or later (currently using version $phpversion)"); - } - else if (empty($CFG->enableglobalsearch)) { + if (empty($CFG->enableglobalsearch)) { mtrace('Global searching is not enabled. Nothing performed by search.'); } else{ diff --git a/search/delete.php b/search/delete.php index e4579c60e89..87d155ee891 100644 --- a/search/delete.php +++ b/search/delete.php @@ -1,47 +1,51 @@ 1.8 -* @date 2008/03/31 -* @license http://www.gnu.org/copyleft/gpl.html GNU Public License -* -* Asynchronous index cleaner -* -* Major chages in this review is passing the xxxx_db_names return to -* multiple arity to handle multiple document types modules -*/ + /** + * Global Search Engine for Moodle + * + * @package search + * @category core + * @subpackage search_engine + * @author Michael Champanis (mchampan) [cynnical@gmail.com], Valery Fremaux [valery.fremaux@club-internet.fr] > 1.8 + * @date 2008/03/31 + * @license http://www.gnu.org/copyleft/gpl.html GNU Public License + * + * Asynchronous index cleaner + * + * Major chages in this review is passing the xxxx_db_names return to + * multiple arity to handle multiple document types modules + */ + + /** + * includes and requires + */ + require_once('../config.php'); -/** -* includes and requires -*/ -require_once('../config.php'); -require_once("$CFG->dirroot/search/lib.php"); +/// makes inclusions of the Zend Engine more reliable + $separator = (array_key_exists('WINDIR', $_SERVER)) ? ';' : ':' ; + ini_set('include_path', $CFG->dirroot.'/search'.$separator.ini_get('include_path')); + require_once("$CFG->dirroot/search/lib.php"); + require_once("$CFG->dirroot/search/indexlib.php"); + + +/// checks global search activation require_login(); - + if (empty($CFG->enableglobalsearch)) { error(get_string('globalsearchdisabled', 'search')); } - if (!isadmin()) { + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) { error(get_string('beadmin', 'search'), "$CFG->wwwroot/login/index.php"); } //if - //check for php5 (lib.php) - if (!search_check_php5()) { - $phpversion = phpversion(); - mtrace("Sorry, global search requires PHP 5.0.0 or later (currently using version ".phpversion().")"); - exit(0); + try { + $index = new Zend_Search_Lucene(SEARCH_INDEX_PATH); + } catch(LuceneException $e) { + mtrace("Could not construct a valid index. Maybe the first indexation was never made, or files might be corrupted. Run complete indexation again."); + return; } - - require_once("$CFG->dirroot/search/indexlib.php"); - - $index = new Zend_Search_Lucene(SEARCH_INDEX_PATH); $dbcontrol = new IndexDBControl(); $deletion_count = 0; $startcleantime = time(); @@ -49,6 +53,7 @@ require_once("$CFG->dirroot/search/lib.php"); mtrace('Starting clean-up of removed records...'); mtrace('Index size before: '.$CFG->search_index_size."\n"); +/// check all modules if ($mods = get_records_select('modules')) { $mods = array_merge($mods, search_get_additional_modules()); @@ -68,7 +73,7 @@ require_once("$CFG->dirroot/search/lib.php"); $valuesArray = $db_names_function(); if ($valuesArray){ foreach($valuesArray as $values){ - $where = (isset($values[5])) ? 'WHERE '.$values[5] : ''; + $where = (!empty($values[5])) ? 'WHERE '.$values[5] : ''; $itemtypes = ($values[4] != '*' && $values[4] != 'any') ? " itemtype = '{$values[4]}' AND " : '' ; $query = " SELECT @@ -139,4 +144,4 @@ require_once("$CFG->dirroot/search/lib.php"); mtrace("Finished $deletion_count removals."); mtrace('Index size after: '.$index->count()); -?> +?> \ No newline at end of file diff --git a/search/indexer.php b/search/indexer.php index 293f9c5beab..ff1e322cc96 100644 --- a/search/indexer.php +++ b/search/indexer.php @@ -35,6 +35,10 @@ */ require_once('../config.php'); require_once("$CFG->dirroot/search/lib.php"); +//require_once("debugging.php"); + +$separator = (array_key_exists('WINDIR', $_SERVER)) ? ';' : ':' ; + ini_set('include_path', $CFG->dirroot.'\search'.$separator.ini_get('include_path')); /// only administrators can index the moodle installation, because access to all pages is required @@ -44,7 +48,7 @@ require_once("$CFG->dirroot/search/lib.php"); error(get_string('globalsearchdisabled', 'search')); } - if (!isadmin()) { + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) { error(get_string('beadmin', 'search'), "$CFG->wwwroot/login/index.php"); } @@ -61,12 +65,6 @@ require_once("$CFG->dirroot/search/lib.php"); /// check for php5 (lib.php) - if (!search_check_php5()) { - $phpversion = phpversion(); - mtrace("Sorry, global search requires PHP 5.0.0 or later (currently using version ".phpversion().")"); - exit(0); - } - //php5 found, continue including php5-only files //require_once("$CFG->dirroot/search/Zend/Search/Lucene.php"); require_once("$CFG->dirroot/search/indexlib.php"); @@ -94,13 +92,16 @@ require_once("$CFG->dirroot/search/lib.php"); mtrace("Data directory ($index_path) does not exist, attempting to create."); if (!mkdir($index_path)) { search_pexit("Error creating data directory at: $index_path. Please correct."); - } else { + } + else { mtrace("Directory successfully created."); } - } else { - mtrace("Using $index_path as data directory."); + } + else { + mtrace("Using {$index_path} as data directory."); } + Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8_CaseInsensitive()); $index = new Zend_Search_Lucene($index_path, true); /* @@ -124,31 +125,39 @@ require_once("$CFG->dirroot/search/lib.php"); // * mod_iterator // * mod_get_content_for_index //are the sole basis for including a module in the index at the moment. - $searchables = array(); + + $searchables = search_collect_searchables(); -/// collects modules +/// start indexation - if ($mods = get_records('modules', '', '', '', 'id,name')) { - $searchables = array_merge($searchables, $mods); - } - mtrace(count($searchables).' modules found.'); - - // collects blocks as indexable information may be found in blocks either - if ($blocks = get_records('block', '', '', '', 'id,name')) { - // prepend the "block_" prefix to discriminate document type plugins - foreach(array_keys($blocks) as $aBlockId){ - $blocks[$aBlockId]->name = 'block_'.$blocks[$aBlockId]->name; - } - $searchables = array_merge($searchables, $blocks); - mtrace(count($blocks).' blocks found.'); - } - -/// add virtual modules onto the back of the array - - $searchables = array_merge($searchables, search_get_additional_modules()); if ($searchables){ foreach ($searchables as $mod) { - $class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php'; + + echo "start {$mod->name}"; + + $key = 'search_in_'.$mod->name; + if (isset($CFG->$key) && !$CFG->$key) { + mtrace("module $key has been administratively disabled. Skipping...\n"); + continue; + } + + if ($mod->location == 'internal'){ + $class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php'; + } else { + $class_file = $CFG->dirroot.'/'.$mod->location.'/'.$mod->name.'/search_document.php'; + } + + /* + if (!file_exists($class_file)){ + if (defined("PATH_FOR_SEARCH_TYPE_{$mod->name}")){ + eval("\$pluginpath = PATH_FOR_SEARCH_TYPE_{$mod->name}"); + $class_file = "{$CFG->dirroot}/{$pluginpath}/searchlib.php"; + } else { + mtrace ("No search document found for plugin {$mod->name}. Ignoring."); + continue; + } + } + */ if (file_exists($class_file)) { include_once($class_file); @@ -195,6 +204,8 @@ require_once("$CFG->dirroot/search/lib.php"); mtrace("-- $counter documents indexed"); mtrace("done.\n"); } + } else { + mtrace ("No search document found for plugin {$mod->name}. Ignoring."); } } } diff --git a/search/indexersplash.php b/search/indexersplash.php index 43755ae2412..cf400cb416d 100644 --- a/search/indexersplash.php +++ b/search/indexersplash.php @@ -18,7 +18,11 @@ * includes and requires */ require_once('../config.php'); -require_once("$CFG->dirroot/search/lib.php"); +require_once("{$CFG->dirroot}/search/lib.php"); + +/// makes inclusions of the Zend Engine more reliable +$separator = (array_key_exists('WINDIR', $_SERVER)) ? ';' : ':' ; +ini_set('include_path', $CFG->dirroot.'\search'.$separator.ini_get('include_path')); /// check global search is enabled @@ -28,18 +32,10 @@ require_once("$CFG->dirroot/search/lib.php"); error(get_string('globalsearchdisabled', 'search')); } - if (!isadmin()) { + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) { error(get_string('beadmin', 'search'), "$CFG->wwwroot/login/index.php"); } -/// check for php5 (lib.php) - - if (!search_check_php5()) { - $phpversion = phpversion(); - mtrace("Sorry, global search requires PHP 5.0.0 or later (currently using version ".phpversion().")"); - exit(0); - } - require_once("$CFG->dirroot/search/indexlib.php"); $indexinfo = new IndexInfo(); diff --git a/search/indexlib.php b/search/indexlib.php index be250ca39e5..737c2da4748 100644 --- a/search/indexlib.php +++ b/search/indexlib.php @@ -78,7 +78,8 @@ class IndexInfo { $this->dbcount = count_records(SEARCH_DATABASE_TABLE); //individual document types - $types = search_get_document_types(); + // $types = search_get_document_types(); + $types = search_collect_searchables(true, false); sort($types); foreach($types as $type) { diff --git a/search/lib.php b/search/lib.php index af40f8f1d9f..6e3e98171d6 100644 --- a/search/lib.php +++ b/search/lib.php @@ -16,41 +16,88 @@ * */ -/* -// function reference -function search_get_document_types($prefix = 'SEARCH_TYPE_') { -function search_get_additional_modules() { -function search_shorten_url($url, $length=30) { -function search_escape_string($str) { -function search_check_php5($feedback = false) { -function search_stopwatch($cli = false) { -function search_pexit($str = "") { +/** +* Constants */ - -define('SEARCH_INDEX_PATH', "$CFG->dataroot/search"); +define('SEARCH_INDEX_PATH', "{$CFG->dataroot}/search"); define('SEARCH_DATABASE_TABLE', 'block_search_documents'); -//document types that can be searched -//define('SEARCH_TYPE_NONE', 'none'); -define('SEARCH_TYPE_WIKI', 'wiki'); -define('PATH_FOR_SEARCH_TYPE_WIKI', 'mod/wiki'); -define('SEARCH_TYPE_FORUM', 'forum'); -define('PATH_FOR_SEARCH_TYPE_FORUM', 'mod/forum'); -define('SEARCH_TYPE_GLOSSARY', 'glossary'); -define('PATH_FOR_SEARCH_TYPE_GLOSSARY', 'mod/glossary'); -define('SEARCH_TYPE_RESOURCE', 'resource'); -define('PATH_FOR_SEARCH_TYPE_RESOURCE', 'mod/resource'); -define('SEARCH_TYPE_TECHPROJECT', 'techproject'); -define('PATH_FOR_SEARCH_TYPE_TECHPROJECT', 'mod/techproject'); -define('SEARCH_TYPE_DATA', 'data'); -define('PATH_FOR_SEARCH_TYPE_DATA', 'mod/data'); -define('SEARCH_TYPE_CHAT', 'chat'); -define('PATH_FOR_SEARCH_TYPE_CHAT', 'mod/chat'); -define('SEARCH_TYPE_LESSON', 'lesson'); -define('PATH_FOR_SEARCH_TYPE_LESSON', 'mod/lesson'); +// get document types +include "{$CFG->dirroot}/search/searchtypes.php"; /** -* returns all the document type constants +* collects all searchable items identities +* @param boolean $namelist if true, only returns list of names of searchable items +* @param boolean $verbose if true, prints a discovering status +* @return an array of names or an array of type descriptors +*/ +function search_collect_searchables($namelist=false, $verbose=true){ + global $CFG; + + $searchables = array(); + $searchables_names = array(); + +/// get all installed modules + if ($mods = get_records('modules', '', '', 'name', 'id,name')){ + + $searchabletypes = array_values(search_get_document_types()); + + foreach($mods as $mod){ + if (in_array($mod->name, $searchabletypes)){ + $mod->location = 'internal'; + $searchables[$mod->name] = $mod; + $searchables_names[] = $mod->name; + } else { + $documentfile = $CFG->dirroot."/mod/{$mod->name}/search_document.php"; + $mod->location = 'mod'; + if (file_exists($documentfile)){ + $searchables[$mod->name] = $mod; + $searchables_names[] = $mod->name; + } + } + } + if ($verbose) mtrace(count($searchables).' modules to search in / '.count($mods).' modules found.'); + } + +/// collects blocks as indexable information may be found in blocks either + if ($blocks = get_records('block', '', '', 'name', 'id,name')) { + $blocks_searchables = array(); + // prepend the "block_" prefix to discriminate document type plugins + foreach($blocks as $block){ + $block->dirname = $block->name; + $block->name = 'block_'.$block->name; + if (in_array('SEARCH_TYPE_'.strtoupper($block->name), $searchabletypes)){ + $mod->location = 'internal'; + $blocks_searchables[] = $block; + $searchables_names[] = $block->name; + } else { + $documentfile = $CFG->dirroot."/blocks/{$block->dirname}/search_document.php"; + if (file_exists($documentfile)){ + $mod->location = 'blocks'; + $blocks_searchables[$block->name] = $block; + $searchables_names[] = $block->name; + } + } + } + if ($verbose) mtrace(count($blocks_searchables).' blocks to search in / '.count($blocks).' blocks found.'); + $searchables = array_merge($searchables, $blocks_searchables); + } + +/// add virtual modules onto the back of the array + + $additional = search_get_additional_modules(); + if (!empty($additional)){ + if ($verbose) mtrace(count($additional).' additional to search in.'); + $searchables = array_merge($searchables, $additional); + } + + if ($namelist) + return $searchables_names; + return $searchables; +} + +/** +* returns all the document type constants that are known in core implementation * @param prefix a pattern for recognizing constants * @return an array of type labels */ @@ -63,7 +110,7 @@ function search_get_document_types($prefix = 'SEARCH_TYPE_') { } sort($ret); return $ret; -} //search_get_document_types +} /** * additional virtual modules to index @@ -75,10 +122,16 @@ function search_get_document_types($prefix = 'SEARCH_TYPE_') { */ function search_get_additional_modules() { $extras = array(/* additional keywords go here */); + if (defined('SEARCH_EXTRAS')){ + $extras = explode(',', SEARCH_EXTRAS); + } + $ret = array(); + $temp = new StdClass; foreach($extras as $extra) { $temp->name = $extra; - $ret[] = clone($temp); + $temp->location = 'internal'; + $ret[$temp->name] = clone($temp); } return $ret; } //search_get_additional_modules @@ -113,23 +166,6 @@ function search_escape_string($str) { return $s; } //search_escape_string -/** -* get a real php 5 version number, using 5.0.0 arbitrarily -* @param feedback if true, prints a feedback message to output. -* @return true if version of PHP is high enough -*/ -function search_check_php5($feedback = false) { - if (!check_php_version("5.0.0")) { - if ($feedback) { - print_heading(get_string('versiontoolow', 'search')); - } - return false; - } - else { - return true; - } -} //search_check_php5 - /** * simple timer function, on first call, records a current microtime stamp, outputs result on 2nd call * @param cli an output formatting switch diff --git a/search/query.php b/search/query.php index 0e016139019..efe6a15c83d 100644 --- a/search/query.php +++ b/search/query.php @@ -36,6 +36,7 @@ */ require_once('../config.php'); require_once("$CFG->dirroot/search/lib.php"); + // include "debugging.php"; if ($CFG->forcelogin) { @@ -50,92 +51,105 @@ /// check for php5, but don't die yet (see line 52) - if ($check = search_check_php5()) { - require_once("{$CFG->dirroot}/search/querylib.php"); + require_once("{$CFG->dirroot}/search/querylib.php"); + + $page_number = optional_param('page', -1, PARAM_INT); + $pages = ($page_number == -1) ? false : true; + $advanced = (optional_param('a', '0', PARAM_INT) == '1') ? true : false; + $query_string = stripslashes(optional_param('query_string', '', PARAM_CLEAN)); + +/// discard harmfull searches + + if (!isset($CFG->block_search_utf8dir)){ + set_config('block_search_utf8dir', 1); + } + +/// discard harmfull searches + + if (preg_match("/^[\*\?]+$/", $query_string)){ + $query_string = ''; + $error = get_string('fullwildcardquery','search'); + } - $page_number = optional_param('page', -1, PARAM_INT); - $pages = ($page_number == -1) ? false : true; - $advanced = (optional_param('a', '0', PARAM_INT) == '1') ? true : false; - $query_string = optional_param('query_string', '', PARAM_CLEAN); - - if ($pages && isset($_SESSION['search_advanced_query'])) { - // if both are set, then we are busy browsing through the result pages of an advanced query - $adv = unserialize($_SESSION['search_advanced_query']); - } - else if ($advanced) { - // otherwise we are dealing with a new advanced query - unset($_SESSION['search_advanced_query']); - session_unregister('search_advanced_query'); - - // chars to strip from strings (whitespace) - $chars = " \t\n\r\0\x0B,-+"; - - // retrieve advanced query variables - $adv->mustappear = trim(optional_param('mustappear', '', PARAM_CLEAN), $chars); - $adv->notappear = trim(optional_param('notappear', '', PARAM_CLEAN), $chars); - $adv->canappear = trim(optional_param('canappear', '', PARAM_CLEAN), $chars); - $adv->module = optional_param('module', '', PARAM_CLEAN); - $adv->title = trim(optional_param('title', '', PARAM_CLEAN), $chars); - $adv->author = trim(optional_param('author', '', PARAM_CLEAN), $chars); - } - - if ($advanced) { - //parse the advanced variables into a query string - //TODO: move out to external query class (QueryParse?) - - $query_string = ''; - - // get all available module types - $module_types = array_merge(array('all'), array_values(search_get_document_types())); - $adv->module = in_array($adv->module, $module_types) ? $adv->module : 'all'; - - // convert '1 2' into '+1 +2' for required words field - if (strlen(trim($adv->mustappear)) > 0) { - $query_string = ' +'.implode(' +', preg_split("/[\s,;]+/", $adv->mustappear)); - } - - // convert '1 2' into '-1 -2' for not wanted words field - if (strlen(trim($adv->notappear)) > 0) { - $query_string .= ' -'.implode(' -', preg_split("/[\s,;]+/", $adv->notappear)); - } - - // this field is left untouched, apart from whitespace being stripped - if (strlen(trim($adv->canappear)) > 0) { - $query_string .= ' '.implode(' ', preg_split("/[\s,;]+/", $adv->canappear)); - } - - // add module restriction - $doctypestr = get_string('doctype', 'search'); - $titlestr = get_string('title', 'search'); - $authorstr = get_string('author', 'search'); - if ($adv->module != 'all') { - $query_string .= " +{$doctypestr}:".$adv->module; - } - - // create title search string - if (strlen(trim($adv->title)) > 0) { - $query_string .= " +{$titlestr}:".implode(" +{$titlestr}:", preg_split("/[\s,;]+/", $adv->title)); - } - - // create author search string - if (strlen(trim($adv->author)) > 0) { - $query_string .= " +{$authorstr}:".implode(" +{$authorstr}:", preg_split("/[\s,;]+/", $adv->author)); - } - - // save our options if the query is valid - if (!empty($query_string)) { - $_SESSION['search_advanced_query'] = serialize($adv); - } - } - - // normalise page number - if ($page_number < 1) { - $page_number = 1; - } - - //run the query against the index - $sq = new SearchQuery($query_string, $page_number, 10, false); + + if ($pages && isset($_SESSION['search_advanced_query'])) { + // if both are set, then we are busy browsing through the result pages of an advanced query + $adv = unserialize($_SESSION['search_advanced_query']); + } elseif ($advanced) { + // otherwise we are dealing with a new advanced query + unset($_SESSION['search_advanced_query']); + session_unregister('search_advanced_query'); + + // chars to strip from strings (whitespace) + $chars = " \t\n\r\0\x0B,-+"; + + // retrieve advanced query variables + $adv->mustappear = trim(optional_param('mustappear', '', PARAM_CLEAN), $chars); + $adv->notappear = trim(optional_param('notappear', '', PARAM_CLEAN), $chars); + $adv->canappear = trim(optional_param('canappear', '', PARAM_CLEAN), $chars); + $adv->module = optional_param('module', '', PARAM_CLEAN); + $adv->title = trim(optional_param('title', '', PARAM_CLEAN), $chars); + $adv->author = trim(optional_param('author', '', PARAM_CLEAN), $chars); } + + if ($advanced) { + //parse the advanced variables into a query string + //TODO: move out to external query class (QueryParse?) + + $query_string = ''; + + // get all available module types adding third party modules + $module_types = array_merge(array('all'), array_values(search_get_document_types())); + $module_types = array_merge($module_types, array_values(search_get_document_types('X_SEARCH_TYPE'))); + $adv->module = in_array($adv->module, $module_types) ? $adv->module : 'all'; + + // convert '1 2' into '+1 +2' for required words field + if (strlen(trim($adv->mustappear)) > 0) { + $query_string = ' +'.implode(' +', preg_split("/[\s,;]+/", $adv->mustappear)); + } + + // convert '1 2' into '-1 -2' for not wanted words field + if (strlen(trim($adv->notappear)) > 0) { + $query_string .= ' -'.implode(' -', preg_split("/[\s,;]+/", $adv->notappear)); + } + + // this field is left untouched, apart from whitespace being stripped + if (strlen(trim($adv->canappear)) > 0) { + $query_string .= ' '.implode(' ', preg_split("/[\s,;]+/", $adv->canappear)); + } + + // add module restriction + $doctypestr = get_string('doctype', 'search'); + $titlestr = get_string('title', 'search'); + $authorstr = get_string('author', 'search'); + if ($adv->module != 'all') { + $query_string .= " +{$doctypestr}:".$adv->module; + } + + // create title search string + if (strlen(trim($adv->title)) > 0) { + $query_string .= " +{$titlestr}:".implode(" +{$titlestr}:", preg_split("/[\s,;]+/", $adv->title)); + } + + // create author search string + if (strlen(trim($adv->author)) > 0) { + $query_string .= " +{$authorstr}:".implode(" +{$authorstr}:", preg_split("/[\s,;]+/", $adv->author)); + } + + // save our options if the query is valid + if (!empty($query_string)) { + $_SESSION['search_advanced_query'] = serialize($adv); + } + } + + // normalise page number + if ($page_number < 1) { + $page_number = 1; + } + + //run the query against the index ensuring internal coding works in UTF-8 + Zend_Search_Lucene_Analysis_Analyzer::setDefault(new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8_CaseInsensitive()); + $sq = new SearchQuery($query_string, $page_number, 10, false); if (!$site = get_site()) { redirect("index.php"); @@ -156,12 +170,9 @@ print_header("$strsearch", "$site->fullname" , $navigation, "", "", true, " ", navmenu($site)); } - //keep things pretty, even if php5 isn't available - if (!$check) { - print_heading(search_check_php5(true)); - print_footer(); - exit(0); - } + if (!empty($error)){ + notice ($error); + } print_box_start(); print_heading($strquery); @@ -178,12 +189,11 @@ } } ?> -
' . get_string('noindexmessage', 'search') . '' . get_string('createanindex', 'search')."
\n"; } @@ -296,7 +306,8 @@ is_valid()) { print_box_start(); @@ -305,7 +316,7 @@ print "