From 65da6840f84602f4876a1a48c68ce88ceccb473a Mon Sep 17 00:00:00 2001 From: sam marshall Date: Thu, 7 Dec 2017 12:27:31 +0000 Subject: [PATCH] MDL-60981 core_search: UI to gradually reindex a single area Adds a new 'Gradual reindex' link to the search areas page for each area. When clicked, this takes you to a confirm prompt, and then adds each context from that search area to the indexing queue. The search areas page now displays the 'Additional indexing queue' (if it is non-empty). The table shows the first 10 items in the queue, and it also indicates the total number in case there are more. (I don't think people really need to see the entire contents of it, so I didn't implement paging.) --- admin/searchareas.php | 19 +++- admin/searchreindex.php | 87 ++++++++++++++++ lang/en/search.php | 9 ++ search/classes/base.php | 13 +++ search/classes/manager.php | 49 ++++++++- search/classes/output/renderer.php | 11 ++ search/templates/index_requests.mustache | 110 ++++++++++++++++++++ search/upgrade.txt | 4 +- theme/boost/scss/moodle/search.scss | 8 ++ theme/bootstrapbase/less/moodle/search.less | 8 ++ theme/bootstrapbase/style/moodle.css | 7 ++ 11 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 admin/searchreindex.php create mode 100644 search/templates/index_requests.mustache diff --git a/admin/searchareas.php b/admin/searchareas.php index 9ce7af6d032..9cac43969da 100644 --- a/admin/searchareas.php +++ b/admin/searchareas.php @@ -132,7 +132,17 @@ foreach ($searchareas as $area) { $laststatus = ''; } $columns[] = $laststatus; - $columns[] = html_writer::link(admin_searcharea_action_url('delete', $areaid), 'Delete index'); + $accesshide = html_writer::span($area->get_visible_name(), 'accesshide'); + $actions = []; + $actions[] = $OUTPUT->pix_icon('t/delete', '') . + html_writer::link(admin_searcharea_action_url('delete', $areaid), + get_string('deleteindex', 'search', $accesshide)); + if ($area->supports_get_document_recordset()) { + $actions[] = $OUTPUT->pix_icon('i/reload', '') . html_writer::link( + new moodle_url('searchreindex.php', ['areaid' => $areaid]), + get_string('gradualreindex', 'search', $accesshide)); + } + $columns[] = html_writer::alist($actions, ['class' => 'unstyled list-unstyled']); } else { $blankrow = new html_table_cell(get_string('searchnotavailable', 'admin')); @@ -165,6 +175,13 @@ echo $OUTPUT->single_button(admin_searcharea_action_url('deleteall'), get_string echo $OUTPUT->box_end(); echo html_writer::table($table); + +if (empty($searchmanagererror)) { + // Show information about queued index requests for specific contexts. + $searchrenderer = $PAGE->get_renderer('core_search'); + echo $searchrenderer->render_index_requests_info($searchmanager->get_index_requests_info()); +} + echo $OUTPUT->footer(); /** diff --git a/admin/searchreindex.php b/admin/searchreindex.php new file mode 100644 index 00000000000..9fa7a3ccae6 --- /dev/null +++ b/admin/searchreindex.php @@ -0,0 +1,87 @@ +. + +/** + * Adds a search area to the queue for indexing. + * + * @package core_search + * @copyright 2017 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +define('NO_OUTPUT_BUFFERING', true); + +require(__DIR__ . '/../config.php'); + +// Check access. +require_once($CFG->libdir . '/adminlib.php'); + +admin_externalpage_setup('searchareas', '', null, (new moodle_url('/admin/searchreindex.php'))->out(false)); + +// Get area parameter and check it exists. +$areaid = required_param('areaid', PARAM_ALPHAEXT); +$area = \core_search\manager::get_search_area($areaid); +if ($area === false) { + throw new moodle_exception('invalidrequest'); +} +$areaname = $area->get_visible_name(); + +// Start page output. +$heading = get_string('gradualreindex', 'search', ''); +$PAGE->set_title($PAGE->title . ': ' . $heading); +$PAGE->navbar->add($heading); +echo $OUTPUT->header(); +echo $OUTPUT->heading($heading); + +// If sesskey is supplied, actually carry out the action. +if (optional_param('sesskey', '', PARAM_ALPHANUM)) { + require_sesskey(); + + // Get all contexts for search area. This query can take time in large cases. + \core_php_time_limit::raise(0); + $contextiterator = $area->get_contexts_to_reindex(); + + $progress = new \core\progress\display_if_slow(''); + $progress->start_progress($areaname); + + // Request reindexing for each context (with low priority). + $count = 0; + foreach ($contextiterator as $context) { + \core_php_time_limit::raise(30); + \core_search\manager::request_index($context, $area->get_area_id(), + \core_search\manager::INDEX_PRIORITY_REINDEXING); + $progress->progress(); + $count++; + } + + // Unset the iterator which should close the recordset (if there is one). + unset($contextiterator); + + $progress->end_progress(); + + $a = (object)['name' => html_writer::tag('strong', $areaname), 'count' => $count]; + echo $OUTPUT->box(get_string('gradualreindex_queued', 'search', $a)); + + echo $OUTPUT->continue_button(new moodle_url('/admin/searchareas.php')); +} else { + // Display confirmation prompt. + echo $OUTPUT->confirm(get_string('gradualreindex_confirm', 'search', html_writer::tag('strong', $areaname)), + new single_button(new moodle_url('/admin/searchreindex.php', ['areaid' => $areaid, + 'sesskey' => sesskey()]), get_string('continue'), 'post', true), + new single_button(new moodle_url('/admin/searchareas.php'), get_string('cancel'), 'get')); +} + +echo $OUTPUT->footer(); diff --git a/lang/en/search.php b/lang/en/search.php index 552d9b6064f..cba4592d1ce 100644 --- a/lang/en/search.php +++ b/lang/en/search.php @@ -36,6 +36,7 @@ $string['createdon'] = 'Created on'; $string['database'] = 'Database'; $string['databasestate'] = 'Indexing database state'; $string['datadirectory'] = 'Data directory'; +$string['deleteindex'] = 'Delete index {$a}'; $string['deletionsinindex'] = 'Deletions in index'; $string['docmodifiedon'] = 'Last modified on {$a}'; $string['doctype'] = 'Doctype'; @@ -59,6 +60,9 @@ $string['filterheader'] = 'Filter'; $string['fromtime'] = 'Modified after'; $string['globalsearch'] = 'Global search'; $string['globalsearchdisabled'] = 'Global searching is not enabled.'; +$string['gradualreindex'] = 'Gradual reindex {$a}'; +$string['gradualreindex_confirm'] = 'Are you sure you want to reindex {$a}? This may take some time, although existing data will remain available during the reindex.'; +$string['gradualreindex_queued'] = 'Reindexing has been requested for {$a->name} ({$a->count} contexts). This indexing will be carried out by the ‘Global search indexing’ scheduled task.'; $string['checkdb'] = 'Check database'; $string['checkdbadvice'] = 'Check your database for any problems.'; $string['checkdir'] = 'Check dir'; @@ -76,7 +80,12 @@ $string['notitle'] = 'No title'; $string['normalsearch'] = 'Normal search'; $string['openedon'] = 'opened on'; $string['optimize'] = 'Optimize'; +$string['priority'] = 'Priority'; +$string['priority_reindexing'] = 'Reindexing'; +$string['priority_normal'] = 'Normal'; +$string['progress'] = 'Progress'; $string['queryerror'] = 'The query you provided could not be parsed by the search engine: {$a}'; +$string['queueheading'] = 'Additional indexing queue ({$a} items)'; $string['resultsreturnedfor'] = 'results returned for'; $string['runindexer'] = 'Run indexer (real)'; $string['runindexertest'] = 'Run indexer test'; diff --git a/search/classes/base.php b/search/classes/base.php index 2d7e69ef800..97fb8ab8e12 100644 --- a/search/classes/base.php +++ b/search/classes/base.php @@ -302,6 +302,19 @@ abstract class base { return false; } + /** + * Checks if get_document_recordset is supported for this search area. + * + * For many uses you can simply call get_document_recordset and see if it returns false, but + * this function is useful when you don't want to actually call the function right away. + */ + public function supports_get_document_recordset() { + // Easiest way to check this is simply to see if the class has overridden the default + // function. + $method = new \ReflectionMethod($this, 'get_document_recordset'); + return $method->getDeclaringClass()->getName() !== self::class; + } + /** * Returns the document related with the provided record. * diff --git a/search/classes/manager.php b/search/classes/manager.php index 2b131f40e89..e04cf9854d7 100644 --- a/search/classes/manager.php +++ b/search/classes/manager.php @@ -1153,7 +1153,8 @@ class manager { * added to a queue which is processed by the task. * * This is used after a restore to ensure that restored items are indexed, even though their - * modified time will be older than the latest indexed. + * modified time will be older than the latest indexed. It is also used by the 'Gradual reindex' + * admin feature from the search areas screen. * * @param \context $context Context to index within * @param string $areaid Area to index, '' = all areas @@ -1275,6 +1276,52 @@ class manager { } } + /** + * Gets information about the request queue, in the form of a plain object suitable for passing + * to a template for rendering. + * + * @return \stdClass Information about queued index requests + */ + public function get_index_requests_info() { + global $DB; + + $result = new \stdClass(); + + $result->total = $DB->count_records('search_index_requests'); + $result->topten = $DB->get_records('search_index_requests', null, + 'indexpriority DESC, timerequested, contextid, searcharea', + 'id, contextid, timerequested, searcharea, partialarea, partialtime, indexpriority', + 0, 10); + foreach ($result->topten as $item) { + $context = \context::instance_by_id($item->contextid); + $item->contextlink = \html_writer::link($context->get_url(), + s($context->get_context_name())); + if ($item->searcharea) { + $item->areaname = $this->get_search_area($item->searcharea)->get_visible_name(); + } + if ($item->partialarea) { + $item->partialareaname = $this->get_search_area($item->partialarea)->get_visible_name(); + } + switch ($item->indexpriority) { + case self::INDEX_PRIORITY_REINDEXING : + $item->priorityname = get_string('priority_reindexing', 'search'); + break; + case self::INDEX_PRIORITY_NORMAL : + $item->priorityname = get_string('priority_normal', 'search'); + break; + } + } + + // Normalise array indices. + $result->topten = array_values($result->topten); + + if ($result->total > 10) { + $result->ellipsis = true; + } + + return $result; + } + /** * Gets current time for use in search system. * diff --git a/search/classes/output/renderer.php b/search/classes/output/renderer.php index 6f1f51f3d31..97cbe29e110 100644 --- a/search/classes/output/renderer.php +++ b/search/classes/output/renderer.php @@ -103,4 +103,15 @@ class renderer extends \plugin_renderer_base { $content .= $this->output->box_end(); return $content; } + + /** + * Returns information about queued index requests. + * + * @param \stdClass $info Info object from get_index_requests_info + * @return string HTML + * @throws \moodle_exception Any error with template + */ + public function render_index_requests_info(\stdClass $info) { + return $this->output->render_from_template('core_search/index_requests', $info); + } } diff --git a/search/templates/index_requests.mustache b/search/templates/index_requests.mustache new file mode 100644 index 00000000000..784ade70748 --- /dev/null +++ b/search/templates/index_requests.mustache @@ -0,0 +1,110 @@ +{{! + This file is part of Moodle - http://moodle.org/ + + Moodle is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Moodle is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Moodle. If not, see . +}} +{{! + @template core_search/index_requests + + Template to provide admin information about the queue of index requests. + + Classes required for JS: + * none + + Data attributes required for JS: + * none + + Context variables required for this template: + * topten + * count + + Optional context variables for this template: + * ellipsis + + Example context (json): + { + "topten": + [ + { + "id": 42, + "timerequested": 123456789, + "contextid": 123, + "contextlink": "Forum: Tutor group forum", + "searcharea": "mod_forum-activity", + "areaname": "Forum activities", + "partialarea": "mod_forum-activity", + "partialareaname": "Forum activities", + "partialtime": 123400000, + "indexpriority": 100 + } + ], + "total": 1, + "ellipsis": true + } +}} +{{#total}} +
+

+ {{#str}} queueheading, search, {{total}} {{/str}} +

+ + + + + + + + + + + + + + {{#topten}} + + + + + + + + {{/topten}} + + {{#ellipsis}} + + + + {{/ellipsis}} + +
{{#str}} context, role {{/str}}{{#str}} searcharea, search {{/str}}{{#str}} time {{/str}}{{#str}} progress, search {{/str}}{{#str}} priority, search {{/str}}
+ {{{contextlink}}} + + {{#searcharea}} {{areaname}} {{/searcharea}} + {{#userdate}} {{timerequested}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}} + {{#partialarea}} + {{partialareaname}}: + {{/partialarea}} + {{#partialtime}} + {{#userdate}} {{partialtime}}, {{#str}} strftimedatetimeshort {{/str}} {{/userdate}} + {{/partialtime}} + + {{#priorityname}} + {{priorityname}} + {{/priorityname}} + {{^priorityname}} + {{indexpriority}} + {{/priorityname}} +
...
+
+{{/total}} diff --git a/search/upgrade.txt b/search/upgrade.txt index 3a2bbdfa2b2..0371a654b76 100644 --- a/search/upgrade.txt +++ b/search/upgrade.txt @@ -6,7 +6,9 @@ information provided here is intended especially for developers. * Search areas may now optionally implement the get_contexts_to_reindex function (for modules and blocks, see also get_contexts_to_reindex_extra_sql). This allows a search area to customise the order in which it is reindexed when doing a gradual reindex, so as to reindex the most important - contexts first. + contexts first. If not implemented, the default behaviour for modules and blocks is to reindex + the newest items first; for other types of search area it will just index the whole system + context, oldest data first. === 3.4 === diff --git a/theme/boost/scss/moodle/search.scss b/theme/boost/scss/moodle/search.scss index 615d18d0330..bc40c9ab90c 100644 --- a/theme/boost/scss/moodle/search.scss +++ b/theme/boost/scss/moodle/search.scss @@ -68,3 +68,11 @@ margin-right: $spacer; display: inline-block; } + +#core-search-areas .lastcol li { + margin-left: 24px; + text-indent: -24px; +} +#core-search-areas .lastcol li > i { + text-indent: 0; +} diff --git a/theme/bootstrapbase/less/moodle/search.less b/theme/bootstrapbase/less/moodle/search.less index 21344348197..649f0c9c498 100644 --- a/theme/bootstrapbase/less/moodle/search.less +++ b/theme/bootstrapbase/less/moodle/search.less @@ -50,3 +50,11 @@ .search-areas-actions > div { display: inline-block; } + +#core-search-areas .lastcol li { + margin-left: 24px; + text-indent: -24px; +} +#core-search-areas .lastcol li > i { + text-indent: 0; +} diff --git a/theme/bootstrapbase/style/moodle.css b/theme/bootstrapbase/style/moodle.css index 7230efa2bdd..600138f46f4 100644 --- a/theme/bootstrapbase/style/moodle.css +++ b/theme/bootstrapbase/style/moodle.css @@ -10046,6 +10046,13 @@ body.path-question-type .mform fieldset.hidden { .search-areas-actions > div { display: inline-block; } +#core-search-areas .lastcol li { + margin-left: 24px; + text-indent: -24px; +} +#core-search-areas .lastcol li > i { + text-indent: 0; +} .popover-region { float: right; position: relative;