From 0a9a10f029fd5e609f2d0d58f0de90ed74802237 Mon Sep 17 00:00:00 2001
From: Matt Porritt <mattp@catalyst-au.net>
Date: Wed, 12 Apr 2017 12:42:25 +1000
Subject: [PATCH] MDL-58654 Global search: Allow engines to support batch
 processing.

Allow Global Search 3rd party engines to be much faster by supporting batch processing.
Refactor the iterator loop for the documents to add to the search index into its own method.
Move this new method from the manger class to the base engine class.
The move to the base engine class will allow search engine plugins to override this and determine
how they implment it. It will not break the existing interface contract with existing plugins.

Finally, add an elasped time indication to the trace output.
---
 search/classes/engine.php  | 44 ++++++++++++++++++++++++++++++++++++++
 search/classes/manager.php | 41 ++++++++---------------------------
 2 files changed, 53 insertions(+), 32 deletions(-)

diff --git a/search/classes/engine.php b/search/classes/engine.php
index 52c4bb371d6..a1dd10ce955 100644
--- a/search/classes/engine.php
+++ b/search/classes/engine.php
@@ -198,6 +198,50 @@ abstract class engine {
         return $doc;
     }
 
+    /**
+     * Loop through given iterator of search documents
+     * and and have the search engine back end add them
+     * to the index.
+     *
+     * @param iterator $iterator the iterator of documents to index
+     * @param searcharea $searcharea the area for the documents to index
+     * @param array $options document indexing options
+     * @return array Processed document counts
+     */
+    public function add_documents($iterator, $searcharea, $options) {
+        $numrecords = 0;
+        $numdocs = 0;
+        $numdocsignored = 0;
+        $lastindexeddoc = 0;
+
+        foreach ($iterator as $document) {
+            if (!$document instanceof \core_search\document) {
+                continue;
+            }
+
+            if ($options['lastindexedtime'] == 0) {
+                // If we have never indexed this area before, it must be new.
+                $document->set_is_new(true);
+            }
+
+            if ($options['indexfiles']) {
+                // Attach files if we are indexing.
+                $searcharea->attach_files($document);
+            }
+
+            if ($this->add_document($document, $options['indexfiles'])) {
+                $numdocs++;
+            } else {
+                $numdocsignored++;
+            }
+
+            $lastindexeddoc = $document->get('modified');
+            $numrecords++;
+        }
+
+        return array($numrecords, $numdocs, $numdocsignored, $lastindexeddoc);
+    }
+
     /**
      * Returns the plugin name.
      *
diff --git a/search/classes/manager.php b/search/classes/manager.php
index f6f7599abdf..cef7900d1d9 100644
--- a/search/classes/manager.php
+++ b/search/classes/manager.php
@@ -546,15 +546,11 @@ class manager {
             $this->engine->area_index_starting($searcharea, $fullindex);
 
             $indexingstart = time();
+            $elapsed = microtime(true);
 
             // This is used to store this component config.
             list($componentconfigname, $varname) = $searcharea->get_config_var_name();
 
-            $numrecords = 0;
-            $numdocs = 0;
-            $numdocsignored = 0;
-            $lastindexeddoc = 0;
-
             $prevtimestart = intval(get_config($componentconfigname, $varname . '_indexingstart'));
 
             if ($fullindex === true) {
@@ -570,36 +566,17 @@ class manager {
             $fileindexing = $this->engine->file_indexing_enabled() && $searcharea->uses_file_indexing();
             $options = array('indexfiles' => $fileindexing, 'lastindexedtime' => $prevtimestart);
             $iterator = new \core\dml\recordset_walk($recordset, array($searcharea, 'get_document'), $options);
-            foreach ($iterator as $document) {
-                if (!$document instanceof \core_search\document) {
-                    continue;
-                }
-
-                if ($prevtimestart == 0) {
-                    // If we have never indexed this area before, it must be new.
-                    $document->set_is_new(true);
-                }
-
-                if ($fileindexing) {
-                    // Attach files if we are indexing.
-                    $searcharea->attach_files($document);
-                }
-
-                if ($this->engine->add_document($document, $fileindexing)) {
-                    $numdocs++;
-                } else {
-                    $numdocsignored++;
-                }
-
-                $lastindexeddoc = $document->get('modified');
-                $numrecords++;
-            }
+            list($numrecords,
+                 $numdocs,
+                 $numdocsignored,
+                 $lastindexeddoc) = $this->engine->add_documents($iterator, $searcharea, $options);
 
             if (CLI_SCRIPT && !PHPUNIT_TEST) {
                 if ($numdocs > 0) {
+                    $elapsed = round((microtime(true) - $elapsed), 3);
                     mtrace('Processed ' . $numrecords . ' records containing ' . $numdocs . ' documents for ' .
-                            $searcharea->get_visible_name() . ' area.');
-                } else  {
+                            $searcharea->get_visible_name() . ' area, in ' . $elapsed . ' seconds.');
+                } else {
                     mtrace('No new documents to index for ' . $searcharea->get_visible_name() . ' area.');
                 }
             }
@@ -698,7 +675,7 @@ class manager {
 
         $vars = array('indexingstart', 'indexingend', 'lastindexrun', 'docsignored', 'docsprocessed', 'recordsprocessed');
 
-        $configsettings =  array();
+        $configsettings = [];
         foreach ($searchareas as $searcharea) {
 
             $areaid = $searcharea->get_area_id();