diff --git a/lang/en/admin.php b/lang/en/admin.php index a38048bf2f5..1ed26d1dd24 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -1028,7 +1028,8 @@ $string['taskdeleteincompleteusers'] = 'Delete incomplete users'; $string['taskdeleteunconfirmedusers'] = 'Delete unconfirmed users'; $string['taskeventscron'] = 'Background processing for events'; $string['taskfiletrashcleanup'] = 'Cleanup files in trash'; -$string['taskglobalsearch'] = 'Global search indexing'; +$string['taskglobalsearchindex'] = 'Global search indexing'; +$string['taskglobalsearchoptimize'] = 'Global search index optimization'; $string['taskgradecron'] = 'Background processing for gradebook'; $string['tasklegacycron'] = 'Legacy cron processing for plugins'; $string['taskmessagingcleanup'] = 'Background processing for messaging'; diff --git a/lib/classes/task/search_task.php b/lib/classes/task/search_index_task.php similarity index 90% rename from lib/classes/task/search_task.php rename to lib/classes/task/search_index_task.php index f22224a84ff..bdeedf1363a 100644 --- a/lib/classes/task/search_task.php +++ b/lib/classes/task/search_index_task.php @@ -30,7 +30,7 @@ namespace core\task; * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ -class search_task extends scheduled_task { +class search_index_task extends scheduled_task { /** * Get a descriptive name for this task (shown to admins). @@ -38,7 +38,7 @@ class search_task extends scheduled_task { * @return string */ public function get_name() { - return get_string('taskglobalsearch', 'admin'); + return get_string('taskglobalsearchindex', 'admin'); } /** @@ -53,8 +53,5 @@ class search_task extends scheduled_task { // Indexing database records for modules + rich documents of forum. $globalsearch->index(); - - // Optimize index at last. - $globalsearch->optimize_index(); } } diff --git a/lib/classes/task/search_optimize_task.php b/lib/classes/task/search_optimize_task.php new file mode 100644 index 00000000000..9f4b8feb11b --- /dev/null +++ b/lib/classes/task/search_optimize_task.php @@ -0,0 +1,61 @@ +. + +/** + * A scheduled task for global search. + * + * @package core + * @copyright 2016 Eric Merrill {@link https://www.merrilldigital.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace core\task; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Runs search index optimization. + * + * @package core + * @copyright 2016 Eric Merrill {@link https://www.merrilldigital.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class search_optimize_task extends scheduled_task { + + /** + * Get a descriptive name for this task (shown to admins). + * + * @return string + */ + public function get_name() { + return get_string('taskglobalsearchoptimize', 'admin'); + } + + /** + * Do the job. + * Throw exceptions on errors (the job will be retried). + */ + public function execute() { + if (!\core_search\manager::is_global_search_enabled()) { + return; + } + + $globalsearch = \core_search\manager::instance(); + + // Optimize index at last. + $globalsearch->optimize_index(); + } +} diff --git a/lib/db/tasks.php b/lib/db/tasks.php index 8abc3f6d13b..1a564930462 100644 --- a/lib/db/tasks.php +++ b/lib/db/tasks.php @@ -285,7 +285,7 @@ $tasks = array( 'month' => '*' ), array( - 'classname' => 'core\task\search_task', + 'classname' => 'core\task\search_index_task', 'blocking' => 0, 'minute' => '*/30', 'hour' => '*', @@ -293,6 +293,15 @@ $tasks = array( 'dayofweek' => '*', 'month' => '*' ), + array( + 'classname' => 'core\task\search_optimize_task', + 'blocking' => 0, + 'minute' => '15', + 'hour' => '*/12', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*' + ), array( 'classname' => 'core\task\stats_cron_task', 'blocking' => 0, diff --git a/search/classes/engine.php b/search/classes/engine.php index bd77769f59f..9711196b02a 100644 --- a/search/classes/engine.php +++ b/search/classes/engine.php @@ -229,6 +229,56 @@ abstract class engine { return $classname; } + /** + * Run any pre-indexing operations. + * + * Should be overwritten if the search engine needs to do any pre index preparation. + * + * @param bool $fullindex True if a full index will be performed + * @return void + */ + public function index_starting($fullindex = false) { + // Nothing by default. + } + + /** + * Run any post indexing operations. + * + * Should be overwritten if the search engine needs to do any post index cleanup. + * + * @param int $numdocs The number of documents that were added to the index + * @param bool $fullindex True if a full index was performed + * @return void + */ + public function index_complete($numdocs = 0, $fullindex = false) { + // Nothing by default. + } + + /** + * Do anything that may need to be done before an area is indexed. + * + * @param \core_search\area\base $searcharea The search area that was complete + * @param bool $fullindex True if a full index is being performed + * @return void + */ + public function area_index_starting($searcharea, $fullindex = false) { + // Nothing by default. + } + + /** + * Do any area cleanup needed, and do anything to confirm contents. + * + * Return false to prevent the search area completed time and stats from being updated. + * + * @param \core_search\area\base $searcharea The search area that was complete + * @param int $numdocs The number of documents that were added to the index + * @param bool $fullindex True if a full index is being performed + * @return bool True means that data is considered indexed + */ + public function area_index_complete($searcharea, $numdocs = 0, $fullindex = false) { + return true; + } + /** * Optimizes the search engine. * @@ -289,13 +339,6 @@ abstract class engine { */ abstract function add_document($doc); - /** - * Commits changes to the server. - * - * @return void - */ - abstract function commit(); - /** * Executes the query on the engine. * diff --git a/search/classes/manager.php b/search/classes/manager.php index da68eb1fb8d..64863b7e743 100644 --- a/search/classes/manager.php +++ b/search/classes/manager.php @@ -466,7 +466,10 @@ class manager { // Unlimited time. \core_php_time_limit::raise(); - $anyupdate = false; + // Notify the engine that an index starting. + $this->engine->index_starting($fullindex); + + $sumdocs = 0; $searchareas = $this->get_search_areas_list(true); foreach ($searchareas as $areaid => $searcharea) { @@ -475,6 +478,9 @@ class manager { mtrace('Processing ' . $searcharea->get_visible_name() . ' area'); } + // Notify the engine that an area is starting. + $this->engine->area_index_starting($searcharea, $fullindex); + $indexingstart = time(); // This is used to store this component config. @@ -518,38 +524,40 @@ class manager { $numrecords++; } - if ($numdocs > 0) { - $anyupdate = true; - - // Commit all remaining documents. - $this->engine->commit(); - - if (CLI_SCRIPT && !PHPUNIT_TEST) { + if (CLI_SCRIPT && !PHPUNIT_TEST) { + if ($numdocs > 0) { mtrace('Processed ' . $numrecords . ' records containing ' . $numdocs . ' documents for ' . - $searcharea->get_visible_name() . ' area. Commits completed.'); + $searcharea->get_visible_name() . ' area.'); + } else { + mtrace('No new documents to index for ' . $searcharea->get_visible_name() . ' area.'); } - } else if (CLI_SCRIPT && !PHPUNIT_TEST) { - mtrace('No new documents to index for ' . $searcharea->get_visible_name() . ' area.'); } - // Store last index run once documents have been commited to the search engine. - set_config($varname . '_indexingstart', $indexingstart, $componentconfigname); - set_config($varname . '_indexingend', time(), $componentconfigname); - set_config($varname . '_docsignored', $numdocsignored, $componentconfigname); - set_config($varname . '_docsprocessed', $numdocs, $componentconfigname); - set_config($varname . '_recordsprocessed', $numrecords, $componentconfigname); - if ($lastindexeddoc > 0) { - set_config($varname . '_lastindexrun', $lastindexeddoc, $componentconfigname); + // Notify the engine this area is complete, and only mark times if true. + if ($this->engine->area_index_complete($searcharea, $numdocs, $fullindex)) { + $sumdocs += $numdocs; + + // Store last index run once documents have been commited to the search engine. + set_config($varname . '_indexingstart', $indexingstart, $componentconfigname); + set_config($varname . '_indexingend', time(), $componentconfigname); + set_config($varname . '_docsignored', $numdocsignored, $componentconfigname); + set_config($varname . '_docsprocessed', $numdocs, $componentconfigname); + set_config($varname . '_recordsprocessed', $numrecords, $componentconfigname); + if ($lastindexeddoc > 0) { + set_config($varname . '_lastindexrun', $lastindexeddoc, $componentconfigname); + } } } - if ($anyupdate) { + if ($sumdocs > 0) { $event = \core\event\search_indexed::create( array('context' => \context_system::instance())); $event->trigger(); } - return $anyupdate; + $this->engine->index_complete($sumdocs, $fullindex); + + return (bool)$sumdocs; } /** @@ -598,7 +606,6 @@ class manager { $this->engine->delete(); $this->reset_config(); } - $this->engine->commit(); } /** @@ -608,7 +615,6 @@ class manager { */ public function delete_index_by_id($id) { $this->engine->delete_by_id($id); - $this->engine->commit(); } /** diff --git a/search/engine/solr/classes/engine.php b/search/engine/solr/classes/engine.php index b8b88cee1aa..15221dca478 100644 --- a/search/engine/solr/classes/engine.php +++ b/search/engine/solr/classes/engine.php @@ -317,10 +317,26 @@ class engine extends \core_search\engine { * * @return void */ - public function commit() { + protected function commit() { $this->get_search_client()->commit(); } + /** + * Do any area cleanup needed, and do anything to confirm contents. + * + * Return false to prevent the search area completed time and stats from being updated. + * + * @param \core_search\area\base $searcharea The search area that was complete + * @param int $numdocs The number of documents that were added to the index + * @param bool $fullindex True if a full index is being performed + * @return bool True means that data is considered indexed + */ + public function area_index_complete($searcharea, $numdocs = 0, $fullindex = false) { + $this->commit(); + + return true; + } + /** * Defragments the index. * @@ -338,6 +354,7 @@ class engine extends \core_search\engine { */ public function delete_by_id($id) { $this->get_search_client()->deleteById($id); + $this->commit(); } /** @@ -352,6 +369,7 @@ class engine extends \core_search\engine { } else { $this->get_search_client()->deleteByQuery('*:*'); } + $this->commit(); } /** diff --git a/search/tests/fixtures/mock_search_engine.php b/search/tests/fixtures/mock_search_engine.php index a56bda88251..0021d951722 100644 --- a/search/tests/fixtures/mock_search_engine.php +++ b/search/tests/fixtures/mock_search_engine.php @@ -41,10 +41,6 @@ class engine extends \core_search\engine { // No need to implement. } - public function commit() { - // No need to implement. - } - public function execute_query($data, $usercontexts) { // No need to implement. } diff --git a/version.php b/version.php index 725a10533b1..90fc4b020fa 100644 --- a/version.php +++ b/version.php @@ -29,7 +29,7 @@ defined('MOODLE_INTERNAL') || die(); -$version = 2016030400.00; // YYYYMMDD = weekly release date of this DEV branch. +$version = 2016030400.01; // YYYYMMDD = weekly release date of this DEV branch. // RR = release increments - 00 in DEV branches. // .XX = incremental changes.