From 2ee2f53021bbf98d9e6644ac5116a6e13bf38aa3 Mon Sep 17 00:00:00 2001 From: David Monllao Date: Wed, 2 Mar 2016 00:03:03 +0800 Subject: [PATCH] MDL-53226 search_simpledb: Adding clumsy search --- lib/classes/plugin_manager.php | 2 +- search/engine/simpledb/classes/engine.php | 260 +++++++++++++++++ search/engine/simpledb/db/install.xml | 33 +++ .../simpledb/lang/en/search_simpledb.php | 27 ++ search/engine/simpledb/tests/engine_test.php | 264 ++++++++++++++++++ search/engine/simpledb/version.php | 29 ++ search/engine/solr/lang/en/search_solr.php | 2 +- search/engine/solr/tests/engine_test.php | 4 +- 8 files changed, 617 insertions(+), 4 deletions(-) create mode 100644 search/engine/simpledb/classes/engine.php create mode 100644 search/engine/simpledb/db/install.xml create mode 100644 search/engine/simpledb/lang/en/search_simpledb.php create mode 100644 search/engine/simpledb/tests/engine_test.php create mode 100644 search/engine/simpledb/version.php diff --git a/lib/classes/plugin_manager.php b/lib/classes/plugin_manager.php index 7af9689c66a..fca4ec80603 100644 --- a/lib/classes/plugin_manager.php +++ b/lib/classes/plugin_manager.php @@ -1884,7 +1884,7 @@ class core_plugin_manager { ), 'search' => array( - 'solr' + 'simpledb', 'solr' ), 'scormreport' => array( diff --git a/search/engine/simpledb/classes/engine.php b/search/engine/simpledb/classes/engine.php new file mode 100644 index 00000000000..47d41b15904 --- /dev/null +++ b/search/engine/simpledb/classes/engine.php @@ -0,0 +1,260 @@ +. + +/** + * Simple moodle database engine. + * + * @package search_simpledb + * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +namespace search_simpledb; + +defined('MOODLE_INTERNAL') || die(); + +/** + * Simple moodle database engine. + * + * @package search_simpledb + * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class engine extends \core_search\engine { + + /** + * Prepares a Solr query, applies filters and executes it returning its results. + * + * @throws \core_search\engine_exception + * @param stdClass $filters Containing query and filters. + * @param array $usercontexts Contexts where the user has access. True if the user can access all contexts. + * @return \core_search\document[] Results or false if no results + */ + public function execute_query($filters, $usercontexts) { + global $DB, $USER; + + // Let's keep these changes internal. + $data = clone $filters; + + $serverstatus = $this->is_server_ready(); + if ($serverstatus !== true) { + throw new \core_search\engine_exception('engineserverstatus', 'search'); + } + + $sql = 'SELECT * FROM {search_simpledb_index} WHERE '; + $params = array(); + + // To store all conditions we will add to where. + $ands = array(); + + // Get results only available for the current user. + $ands[] = '(owneruserid = ? OR owneruserid = ?)'; + $params = array_merge($params, array(\core_search\manager::NO_OWNER_ID, $USER->id)); + + // Restrict it to the context where the user can access, we want this one cached. + // If the user can access all contexts $usercontexts value is just true, we don't need to filter + // in that case. + if ($usercontexts && is_array($usercontexts)) { + // Join all area contexts into a single array and implode. + $allcontexts = array(); + foreach ($usercontexts as $areaid => $areacontexts) { + if (!empty($data->areaid) && ($areaid !== $data->areaid)) { + // Skip unused areas. + continue; + } + foreach ($areacontexts as $contextid) { + // Ensure they are unique. + $allcontexts[$contextid] = $contextid; + } + } + if (empty($allcontexts)) { + // This means there are no valid contexts for them, so they get no results. + return array(); + } + + list($contextsql, $contextparams) = $DB->get_in_or_equal($allcontexts); + $ands[] = 'contextid ' . $contextsql; + $params = array_merge($params, $contextparams); + } + + // Course id filter. + if (!empty($data->courseids)) { + list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->courseids); + $ands[] = 'courseid ' . $conditionsql; + $params = array_merge($params, $conditionparams); + } + + // Area id filter. + if (!empty($data->areaid)) { + list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->areaid); + $ands[] = 'areaid ' . $conditionsql; + $params = array_merge($params, $conditionparams); + } + + if (!empty($data->title)) { + list($conditionsql, $conditionparams) = $DB->get_in_or_equal($data->title); + $ands[] = 'title ' . $conditionsql; + $params = array_merge($params, $conditionparams); + } + + if (!empty($data->timestart)) { + $ands[] = 'modified >= ?'; + $params[] = $data->timestart; + } + if (!empty($data->timeend)) { + $ands[] = 'modified <= ?'; + $params[] = $data->timeend; + } + + // And finally the main query after applying all AND filters. + $ands[] = '(' . + $DB->sql_like('title', '?', false, false) . ' OR ' . + $DB->sql_like('content', '?', false, false) . ' OR ' . + $DB->sql_like('description1', '?', false, false) . ' OR ' . + $DB->sql_like('description2', '?', false, false) . + ')'; + $params[] = '%' . $data->q . '%'; + $params[] = '%' . $data->q . '%'; + $params[] = '%' . $data->q . '%'; + $params[] = '%' . $data->q . '%'; + + $recordset = $DB->get_recordset_sql($sql . implode(' AND ', $ands), $params, 0, \core_search\manager::MAX_RESULTS); + + $numgranted = 0; + + if (!$recordset->valid()) { + return array(); + } + + // Iterate through the results checking its availability and whether they are available for the user or not. + $docs = array(); + foreach ($recordset as $docdata) { + if (!$searcharea = $this->get_search_area($docdata->areaid)) { + continue; + } + + // Switch id back to the document id. + $docdata->id = $docdata->docid; + unset($docdata->docid); + + $access = $searcharea->check_access($docdata->itemid); + switch ($access) { + case \core_search\manager::ACCESS_DELETED: + $this->delete_by_id($docdata->id); + break; + case \core_search\manager::ACCESS_DENIED: + break; + case \core_search\manager::ACCESS_GRANTED: + $numgranted++; + $docs[] = $this->to_document($searcharea, (array)$docdata); + break; + } + + // This should never happen. + if ($numgranted >= \core_search\manager::MAX_RESULTS) { + $docs = array_slice($docs, 0, \core_search\manager::MAX_RESULTS, true); + break; + } + } + $recordset->close(); + + return $docs; + } + + /** + * Adds a document to the search engine. + * + * This does not commit to the search engine. + * + * @param \core_search\document $document + * @param bool $fileindexing True if file indexing is to be used + * @return bool False if the file was skipped or failed, true on success + */ + public function add_document($document, $fileindexing = false) { + global $DB; + + $doc = (object)$document->export_for_engine(); + + // Moodle's ids using DML are always autoincremented. + $doc->docid = $doc->id; + unset($doc->id); + + $id = $DB->get_field('search_simpledb_index', 'id', array('docid' => $doc->docid)); + try { + if ($id) { + $doc->id = $id; + $DB->update_record('search_simpledb_index', $doc); + } else { + $DB->insert_record('search_simpledb_index', $doc); + } + + } catch (dml_exception $ex) { + debugging('dml error while trying to insert document with id ' . $doc->docid . ': ' . $e->getMessage(), + DEBUG_DEVELOPER); + } + + return true; + } + + /** + * Deletes the specified document. + * + * @param string $id The document id to delete + * @return void + */ + public function delete_by_id($id) { + global $DB; + $DB->delete_records('search_simpledb_index', array('docid' => $id)); + } + + /** + * Delete all area's documents. + * + * @param string $areaid + * @return void + */ + public function delete($areaid = null) { + global $DB; + if ($areaid) { + $DB->delete_records('search_simpledb_index', array('areaid' => $areaid)); + } else { + $DB->delete_records('search_simpledb_index'); + } + } + + /** + * Checks that the required table was installed. + * + * @return true|string Returns true if all good or an error string. + */ + public function is_server_ready() { + global $DB; + if (!$DB->get_manager()->table_exists('search_simpledb_index')) { + return 'search_simpledb_index table does not exist'; + } + + return true; + } + + /** + * It is always installed. + * + * @return true + */ + public function is_installed() { + return true; + } +} diff --git a/search/engine/simpledb/db/install.xml b/search/engine/simpledb/db/install.xml new file mode 100644 index 00000000000..21393311b31 --- /dev/null +++ b/search/engine/simpledb/db/install.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + \ No newline at end of file diff --git a/search/engine/simpledb/lang/en/search_simpledb.php b/search/engine/simpledb/lang/en/search_simpledb.php new file mode 100644 index 00000000000..5063a88843b --- /dev/null +++ b/search/engine/simpledb/lang/en/search_simpledb.php @@ -0,0 +1,27 @@ +. + +/** + * Strings for component 'search_simpledb'. + * + * @package search_simpledb + * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +$string['pluginname'] = 'Simple search'; +$string['searchinfo'] = 'Search queries'; +$string['searchinfo_help'] = 'Enter the search query.'; diff --git a/search/engine/simpledb/tests/engine_test.php b/search/engine/simpledb/tests/engine_test.php new file mode 100644 index 00000000000..fa37e19eb0b --- /dev/null +++ b/search/engine/simpledb/tests/engine_test.php @@ -0,0 +1,264 @@ +. + +/** + * Simple db search engine tests. + * + * @package search_simpledb + * @category phpunit + * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php'); +require_once($CFG->dirroot . '/search/tests/fixtures/mock_search_area.php'); + +/** + * Simple search engine base unit tests. + * + * @package search_simpledb + * @category phpunit + * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class search_simpledb_engine_testcase extends advanced_testcase { + + /** + * @var \core_search::manager + */ + protected $search = null; + + public function setUp() { + $this->resetAfterTest(); + set_config('enableglobalsearch', true); + + // Inject search_simpledb engine into the testable core search as we need to add the mock + // search component to it. + $searchengine = new \search_simpledb\engine(); + $this->search = testable_core_search::instance($searchengine); + $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); + $this->search->add_search_area($areaid, new core_mocksearch\search\role_capabilities()); + } + + public function test_index() { + global $DB; + + $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher')); + + // Data gets into the search engine. + $this->assertTrue($this->search->index()); + + // Not anymore as everything was already added. + sleep(1); + $this->assertFalse($this->search->index()); + + assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id); + accesslib_clear_all_caches_for_unit_testing(); + + // Indexing again once there is new data. + $this->assertTrue($this->search->index()); + } + + /** + * Test search filters. + * + * @return void + */ + public function test_search() { + global $USER, $DB; + + $this->setAdminUser(); + + $noneditingteacherid = $DB->get_field('role', 'id', array('shortname' => 'teacher')); + + $this->search->index(); + + // Check that docid - id is respected. + $rolecaps = $DB->get_records('role_capabilities', array('capability' => 'moodle/course:renameroles')); + $rolecap = reset($rolecaps); + $rolecap->timemodified = time(); + $DB->update_record('role_capabilities', $rolecap); + + $this->search->index(); + + $querydata = new stdClass(); + $querydata->q = 'message'; + $results = $this->search->search($querydata); + $this->assertCount(2, $results); + + // Based on core_mocksearch\search\indexer. + $this->assertEquals($USER->id, $results[0]->get('userid')); + $this->assertEquals(\context_system::instance()->id, $results[0]->get('contextid')); + + // Do a test to make sure we aren't searching non-query fields, like areaid. + $querydata->q = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); + $this->assertCount(0, $this->search->search($querydata)); + $querydata->q = 'message'; + + sleep(1); + $beforeadding = time(); + sleep(1); + assign_capability('moodle/course:renameroles', CAP_ALLOW, $noneditingteacherid, context_system::instance()->id); + accesslib_clear_all_caches_for_unit_testing(); + $this->search->index(); + + // Timestart. + $querydata->timestart = $beforeadding; + $this->assertCount(2, $this->search->search($querydata)); + + // Timeend. + unset($querydata->timestart); + $querydata->timeend = $beforeadding; + $this->assertCount(1, $this->search->search($querydata)); + + // Title. + unset($querydata->timeend); + $querydata->title = 'moodle/course:renameroles roleid 1'; + $this->assertCount(1, $this->search->search($querydata)); + + // Course IDs. + unset($querydata->title); + $querydata->courseids = array(SITEID + 1); + $this->assertCount(0, $this->search->search($querydata)); + + $querydata->courseids = array(SITEID); + $this->assertCount(3, $this->search->search($querydata)); + + // Check that index contents get updated. + $DB->delete_records('role_capabilities', array('capability' => 'moodle/course:renameroles')); + $this->search->index(true); + unset($querydata->title); + $querydata->q = '*renameroles*'; + $this->assertCount(0, $this->search->search($querydata)); + } + + public function test_delete() { + $this->search->index(); + + $querydata = new stdClass(); + $querydata->q = 'message'; + + $this->assertCount(2, $this->search->search($querydata)); + + $areaid = \core_search\manager::generate_areaid('core_mocksearch', 'role_capabilities'); + $this->search->delete_index($areaid); + $this->assertCount(0, $this->search->search($querydata)); + } + + public function test_alloweduserid() { + $engine = $this->search->get_engine(); + $area = new core_mocksearch\search\role_capabilities(); + + // Get the first record for the recordset. + $recordset = $area->get_recordset_by_timestamp(); + foreach ($recordset as $r) { + $record = $r; + break; + } + $recordset->close(); + + // Get the doc and insert the default doc. + $doc = $area->get_document($record); + $engine->add_document($doc); + + $users = array(); + $users[] = $this->getDataGenerator()->create_user(); + $users[] = $this->getDataGenerator()->create_user(); + $users[] = $this->getDataGenerator()->create_user(); + + // Add a record that only user 100 can see. + $originalid = $doc->get('id'); + + // Now add a custom doc for each user. + foreach ($users as $user) { + $doc = $area->get_document($record); + $doc->set('id', $originalid.'-'.$user->id); + $doc->set('owneruserid', $user->id); + $engine->add_document($doc); + } + + $engine->area_index_complete($area->get_area_id()); + + $querydata = new stdClass(); + $querydata->q = 'message'; + $querydata->title = $doc->get('title'); + + // We are going to go through each user and see if they get the original and the owned doc. + foreach ($users as $user) { + $this->setUser($user); + + $results = $this->search->search($querydata); + $this->assertCount(2, $results); + + $owned = 0; + $notowned = 0; + + // We don't know what order we will get the results in, so we are doing this. + foreach ($results as $result) { + $owneruserid = $result->get('owneruserid'); + if (empty($owneruserid)) { + $notowned++; + $this->assertEquals(0, $owneruserid); + $this->assertEquals($originalid, $result->get('id')); + } else { + $owned++; + $this->assertEquals($user->id, $owneruserid); + $this->assertEquals($originalid.'-'.$user->id, $result->get('id')); + } + } + + $this->assertEquals(1, $owned); + $this->assertEquals(1, $notowned); + } + + // Now test a user with no owned results. + $otheruser = $this->getDataGenerator()->create_user(); + $this->setUser($otheruser); + + $results = $this->search->search($querydata); + $this->assertCount(1, $results); + + $this->assertEquals(0, $results[0]->get('owneruserid')); + $this->assertEquals($originalid, $results[0]->get('id')); + } + + public function test_delete_by_id() { + // First get files in the index. + $this->search->index(); + $engine = $this->search->get_engine(); + + $querydata = new stdClass(); + + // Then search to make sure they are there. + $querydata->q = 'moodle/course:renameroles'; + $results = $this->search->search($querydata); + $this->assertCount(2, $results); + + $first = reset($results); + $deleteid = $first->get('id'); + + $engine->delete_by_id($deleteid); + + // Check that we don't get a result for it anymore. + $results = $this->search->search($querydata); + $this->assertCount(1, $results); + $result = reset($results); + $this->assertNotEquals($deleteid, $result->get('id')); + } +} diff --git a/search/engine/simpledb/version.php b/search/engine/simpledb/version.php new file mode 100644 index 00000000000..10e5a8c40b9 --- /dev/null +++ b/search/engine/simpledb/version.php @@ -0,0 +1,29 @@ +. + +/** + * Version info. + * + * @package search_simpledb + * @copyright 2016 David Monllao {@link http://www.davidmonllao.com} + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +$plugin->version = 2016030100; +$plugin->requires = 2015111000; +$plugin->component = 'search_simpledb'; diff --git a/search/engine/solr/lang/en/search_solr.php b/search/engine/solr/lang/en/search_solr.php index 6addfea4c24..e87576c1c60 100644 --- a/search/engine/solr/lang/en/search_solr.php +++ b/search/engine/solr/lang/en/search_solr.php @@ -17,7 +17,7 @@ /** * Strings for component 'search_solr'. * - * @package core_search + * @package search_solr * @copyright Prateek Sachan {@link http://prateeksachan.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ diff --git a/search/engine/solr/tests/engine_test.php b/search/engine/solr/tests/engine_test.php index 7a4bcde943d..f9b76c56ac1 100644 --- a/search/engine/solr/tests/engine_test.php +++ b/search/engine/solr/tests/engine_test.php @@ -30,7 +30,7 @@ * - define('TEST_SEARCH_SOLR_KEYPASSWORD', ''); * - define('TEST_SEARCH_SOLR_CAINFOCERT', ''); * - * @package core_search + * @package search_solr * @category phpunit * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later @@ -46,7 +46,7 @@ require_once($CFG->dirroot . '/search/engine/solr/tests/fixtures/testable_engine /** * Solr search engine base unit tests. * - * @package core_search + * @package search_solr * @category phpunit * @copyright 2015 David Monllao {@link http://www.davidmonllao.com} * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later