From b56ccc31929a13eb3875409852e434c9e91aab56 Mon Sep 17 00:00:00 2001 From: diml Date: Wed, 17 Dec 2008 20:52:07 +0000 Subject: [PATCH] Big global search update for 1.9. Will resolve major bug points in Global Search component section. --- blocks/search/block_search.php | 2 +- blocks/search/config_global.html | 209 +++++- lang/en_utf8/block_search.php | 23 +- search/Zend/Search/Exception.php | 6 +- search/Zend/Search/Lucene.php | 503 +++++++++++-- .../Zend/Search/Lucene/Analysis/Analyzer.php | 33 +- .../Lucene/Analysis/Analyzer/Common.php | 8 +- .../Lucene/Analysis/Analyzer/Common/Text.php | 12 +- .../Analyzer/Common/Text/CaseInsensitive.php | 8 +- .../Analysis/Analyzer/Common/TextNum.php | 8 +- .../Common/TextNum/CaseInsensitive.php | 8 +- .../Lucene/Analysis/Analyzer/Common/Utf8.php | 124 ++-- .../Analysis/Analyzer/Common/Utf8Num.php | 120 +--- search/Zend/Search/Lucene/Analysis/Token.php | 4 +- .../Search/Lucene/Analysis/TokenFilter.php | 6 +- .../Lucene/Analysis/TokenFilter/LowerCase.php | 6 +- .../Analysis/TokenFilter/LowerCaseUtf8.php | 5 +- .../Analysis/TokenFilter/ShortWords.php | 6 +- .../Lucene/Analysis/TokenFilter/StopWords.php | 11 +- search/Zend/Search/Lucene/Document.php | 20 +- search/Zend/Search/Lucene/Document/Html.php | 8 +- search/Zend/Search/Lucene/Exception.php | 6 +- search/Zend/Search/Lucene/FSM.php | 8 +- search/Zend/Search/Lucene/FSMAction.php | 4 +- search/Zend/Search/Lucene/Field.php | 4 +- .../Search/Lucene/Index/DictionaryLoader.php | 16 +- search/Zend/Search/Lucene/Index/FieldInfo.php | 4 +- .../Zend/Search/Lucene/Index/SegmentInfo.php | 674 ++++++++++++++---- .../Lucene/Index/SegmentInfoPriorityQueue.php | 8 +- .../Search/Lucene/Index/SegmentMerger.php | 14 +- .../Search/Lucene/Index/SegmentWriter.php | 66 +- .../Index/SegmentWriter/DocumentWriter.php | 17 +- .../Index/SegmentWriter/StreamWriter.php | 17 +- search/Zend/Search/Lucene/Index/Term.php | 39 +- search/Zend/Search/Lucene/Index/TermInfo.php | 4 +- search/Zend/Search/Lucene/Index/Writer.php | 585 ++++++++++----- search/Zend/Search/Lucene/Interface.php | 59 +- search/Zend/Search/Lucene/PriorityQueue.php | 4 +- search/Zend/Search/Lucene/Proxy.php | 82 ++- .../Search/BooleanExpressionRecognizer.php | 12 +- search/Zend/Search/Lucene/Search/Query.php | 6 +- .../Search/Lucene/Search/Query/Boolean.php | 136 +++- .../Zend/Search/Lucene/Search/Query/Empty.php | 8 +- .../Search/Lucene/Search/Query/MultiTerm.php | 124 +++- .../Search/Lucene/Search/Query/Phrase.php | 47 +- .../Zend/Search/Lucene/Search/Query/Term.php | 10 +- .../Zend/Search/Lucene/Search/QueryEntry.php | 16 +- .../Lucene/Search/QueryEntry/Phrase.php | 20 +- .../Lucene/Search/QueryEntry/Subquery.php | 12 +- .../Search/Lucene/Search/QueryEntry/Term.php | 81 ++- search/Zend/Search/Lucene/Search/QueryHit.php | 4 +- .../Zend/Search/Lucene/Search/QueryLexer.php | 12 +- .../Zend/Search/Lucene/Search/QueryParser.php | 259 +++++-- .../Lucene/Search/QueryParserContext.php | 30 +- .../Lucene/Search/QueryParserException.php | 6 +- .../Zend/Search/Lucene/Search/QueryToken.php | 6 +- .../Zend/Search/Lucene/Search/Similarity.php | 6 +- .../Lucene/Search/Similarity/Default.php | 8 +- search/Zend/Search/Lucene/Search/Weight.php | 4 +- .../Search/Lucene/Search/Weight/Boolean.php | 6 +- .../Search/Lucene/Search/Weight/Empty.php | 6 +- .../Search/Lucene/Search/Weight/MultiTerm.php | 6 +- .../Search/Lucene/Search/Weight/Phrase.php | 6 +- .../Zend/Search/Lucene/Search/Weight/Term.php | 6 +- search/Zend/Search/Lucene/SoftLockManager.php | 225 ++++++ .../Zend/Search/Lucene/Storage/Directory.php | 15 +- .../Lucene/Storage/Directory/Filesystem.php | 85 ++- search/Zend/Search/Lucene/Storage/File.php | 6 +- .../Search/Lucene/Storage/File/Filesystem.php | 23 +- .../Search/Lucene/Storage/File/Memory.php | 8 +- search/documents/assignment_document.php | 362 ++++++++++ search/documents/chat_document.php | 27 +- search/documents/data_document.php | 19 +- search/documents/document.php | 12 +- search/documents/forum_document.php | 18 +- search/documents/glossary_document.php | 15 +- search/documents/label_document.php | 184 +++++ search/documents/lesson_document.php | 16 +- search/documents/physical_doc.php | 27 +- search/documents/physical_htm.php | 16 +- search/documents/physical_html.php | 4 +- search/documents/physical_odt.php | 55 ++ search/documents/physical_pdf.php | 23 +- search/documents/physical_ppt.php | 14 +- search/documents/physical_swf.php | 69 ++ search/documents/physical_txt.php | 15 +- search/documents/physical_xml.php | 6 +- search/documents/resource_document.php | 31 +- search/documents/techproject_document.php | 301 -------- search/documents/user_document.php | 366 ++++++++++ search/documents/wiki_document.php | 9 +- search/tests/index.php | 200 ++++-- 92 files changed, 4297 insertions(+), 1432 deletions(-) create mode 100644 search/Zend/Search/Lucene/SoftLockManager.php create mode 100644 search/documents/assignment_document.php create mode 100644 search/documents/label_document.php create mode 100644 search/documents/physical_odt.php create mode 100644 search/documents/physical_swf.php delete mode 100644 search/documents/techproject_document.php create mode 100644 search/documents/user_document.php diff --git a/blocks/search/block_search.php b/blocks/search/block_search.php index d8ab6064d03..7fdddbfb88f 100644 --- a/blocks/search/block_search.php +++ b/blocks/search/block_search.php @@ -36,7 +36,7 @@ global $CFG; if (empty($CFG->enableglobalsearch)) { - return ''; + return get_string('disabledsearch', 'search'); } //cache block contents diff --git a/blocks/search/config_global.html b/blocks/search/config_global.html index be8041c74c2..fd42b64b23d 100644 --- a/blocks/search/config_global.html +++ b/blocks/search/config_global.html @@ -1,5 +1,22 @@ +dirroot."/search/lib.php"; + +$defaultfiletypes = "PDF,TXT,HTML,PPT,XML,DOC,HTM"; + +if (!isset($CFG->block_search_utf8dir)){ + $CFG->block_search_utf8dir = 0; +} +if (!isset($CFG->block_search_softlock)){ + $CFG->block_search_softlock = 0; +} + +?>
- +
+ + + + + + + + @@ -69,6 +107,24 @@

+ + + + +
: @@ -26,6 +43,26 @@ } ?>"/>
+ : + + block_search_utf8dir) echo "checked=\"checked\"" ?> /> + block_search_utf8dir) echo "checked=\"checked\"" ?> /> +

+
+ : + + block_search_softlock) echo "checked=\"checked\"" ?> /> + block_search_softlock) echo "checked=\"checked\"" ?> /> +

+
: @@ -48,11 +85,12 @@ "/>

+ if(!isset($CFG->block_search_filetypes)) { + $CFG->block_search_filetypes = $defaultfiletypes; + } + p($CFG->block_search_filetypes); + ?>" /> +

+ : + +

+
+ +
+ + +
: @@ -88,6 +144,12 @@ } ?>"/>
+
+ +
+ + +
: @@ -124,19 +186,140 @@ } ?>"/>
+
+ +block_search_filetypes); +if (!empty($types)){ + foreach($types as $type) { + $utype = strtoupper($type); + $type = strtolower($type); + $type = trim($type); + if (preg_match("/\\b$type\\b/i", $defaultfiletypes)) continue; +?> +
+ + + + + + +
- : + : -

+ $propname)) { + p($CFG->$propname); + } + ?>"/> +

+ : + + $propname)) { + echo stripslashes($CFG->$propname); + } ?>"/>

+
+
+ + +
+ + + + + +
+ 
+            
+
+
+
+ +
+ + + + + +
+name}"; + $checkedup = (!isset($CFG->$keyname) || $CFG->$keyname) ? 'checked="checked"' : '' ; + $checkeddown = !(!isset($CFG->$keyname) || $CFG->$keyname) ? 'checked="checked"' : '' ; + echo "name}\" value=\"0\" {$checkeddown} /> " . get_string('no'); + echo " - name}\" value=\"1\" {$checkedup} /> " . get_string('yes'); + echo " - ".get_string('modulename', $module->name).'
'; + if ($i % 20 == 0){ + echo "		"; + } + $found_searchable_modules = 1; + } +} +if (!$found_searchable_modules) { + print_string('nosearchablemodules', 'block_search'); +} +?> + +
+
+ +
+ + + + + +
+name}"; + $checked = (!isset($CFG->$keyname) || $CFG->$keyname) ? 'checked="checked"' : '' ; + echo "name}\" value=\"1\" {$checked} />"; + + // multiple fallback strategy to get the name of the block + $blocklabel = get_string('blockname', 'block_'.$block->name); + if ($blocklabel == '[[blockname]]') $blocklabel = get_string($block->name, 'block_'.$block->name); + if ($blocklabel == "[[{$block->name}]]") $blocklabel = "$block->name"; + echo " - ".$blocklabel.'
'; + if ($i % 20 == 0){ + echo "		"; + } + $found_searchable_blocks = 1; + } +} +if (!$found_searchable_blocks) { + print_string('nosearchableblocks', 'block_search'); +} +?> + +
+
+ +
diff --git a/lang/en_utf8/block_search.php b/lang/en_utf8/block_search.php index 0dc84730148..edb48e7ee2b 100644 --- a/lang/en_utf8/block_search.php +++ b/lang/en_utf8/block_search.php @@ -1,15 +1,34 @@ \ No newline at end of file diff --git a/search/Zend/Search/Exception.php b/search/Zend/Search/Exception.php index 291cc43ed50..63bdf565b25 100644 --- a/search/Zend/Search/Exception.php +++ b/search/Zend/Search/Exception.php @@ -14,7 +14,7 @@ * * @category Zend * @package Zend_Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -22,13 +22,13 @@ /** * Framework base exception */ -require_once $CFG->dirroot.'/search/Zend/Exception.php'; +require_once "Zend/Exception.php"; /** * @category Zend * @package Zend_Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Exception extends Zend_Exception diff --git a/search/Zend/Search/Lucene.php b/search/Zend/Search/Lucene.php index 1f15c9a0ba1..361b85f5c6f 100644 --- a/search/Zend/Search/Lucene.php +++ b/search/Zend/Search/Lucene.php @@ -14,65 +14,72 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ - /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once "Zend/Search/Lucene/Exception.php"; /** Zend_Search_Lucene_Document */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document.php'; +require_once "Zend/Search/Lucene/Document.php"; /** Zend_Search_Lucene_Document_Html */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document/Html.php'; +require_once "Zend/Search/Lucene/Document/Html.php"; -/** Zend_Search_Lucene_Storage_Directory */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php'; +/** Zend_Search_Lucene_Storage_Directory_Filesystem */ +require_once "Zend/Search/Lucene/Storage/Directory/Filesystem.php"; /** Zend_Search_Lucene_Storage_File_Memory */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File/Memory.php'; +require_once "Zend/Search/Lucene/Storage/File/Memory.php"; /** Zend_Search_Lucene_Index_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; +require_once "Zend/Search/Lucene/Index/Term.php"; /** Zend_Search_Lucene_Index_TermInfo */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/TermInfo.php'; +require_once "Zend/Search/Lucene/Index/TermInfo.php"; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once "Zend/Search/Lucene/Index/SegmentInfo.php"; /** Zend_Search_Lucene_Index_FieldInfo */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/FieldInfo.php'; +require_once "Zend/Search/Lucene/Index/FieldInfo.php"; /** Zend_Search_Lucene_Index_Writer */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Writer.php'; +require_once "Zend/Search/Lucene/Index/Writer.php"; /** Zend_Search_Lucene_Search_QueryParser */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParser.php'; +require_once "Zend/Search/Lucene/Search/QueryParser.php"; /** Zend_Search_Lucene_Search_QueryHit */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryHit.php'; +require_once "Zend/Search/Lucene/Search/QueryHit.php"; /** Zend_Search_Lucene_Search_Similarity */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity.php'; +require_once "Zend/Search/Lucene/Search/Similarity.php"; /** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php'; +require_once "Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php"; + +/** Zend_Search_Lucene_LockManager */ +if (@$CFG->block_search_softlock){ + require_once "Zend/Search/Lucene/SoftLockManager.php"; +} else { + require_once "Zend/Search/Lucene/LockManager.php"; +} + /** Zend_Search_Lucene_Interface */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Interface.php'; +require_once "Zend/Search/Lucene/Interface.php"; /** Zend_Search_Lucene_Proxy */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Proxy.php'; +require_once "Zend/Search/Lucene/Proxy.php"; /** * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface @@ -86,6 +93,15 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface */ private static $_defaultSearchField = null; + /** + * Result set limit + * + * 0 means no limit + * + * @var integer + */ + private static $_resultSetLimit = 0; + /** * File system adapter. * @@ -129,13 +145,6 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface private $_hasChanges = false; - /** - * Index lock object - * - * @var Zend_Search_Lucene_Storage_File - */ - private $_lock; - /** * Signal, that index is already closed, changes are fixed and resources are cleaned up * @@ -150,7 +159,14 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface */ private $_refCount = 0; + /** + * Current segment generation + * + * @var integer + */ + private $_generation; + /** * Create index * @@ -173,56 +189,101 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface return new Zend_Search_Lucene_Proxy(new Zend_Search_Lucene($directory, false)); } + /** Generation retrieving counter */ + const GENERATION_RETRIEVE_COUNT = 10; + + /** Pause between generation retrieving attempts in milliseconds */ + const GENERATION_RETRIEVE_PAUSE = 50; + /** - * Opens the index. + * Get current generation number * - * IndexReader constructor needs Directory as a parameter. It should be - * a string with a path to the index folder or a Directory object. + * Returns generation number + * 0 means pre-2.1 index format + * -1 means there are no segments files. * - * @param mixed $directory + * @param Zend_Search_Lucene_Storage_Directory $directory + * @return integer * @throws Zend_Search_Lucene_Exception */ - public function __construct($directory = null, $create = false) + public static function getActualGeneration(Zend_Search_Lucene_Storage_Directory $directory) { - if ($directory === null) { - throw new Zend_Search_Exception('No index directory specified'); - } + /** + * Zend_Search_Lucene uses segments.gen file to retrieve current generation number + * + * Apache Lucene index format documentation mentions this method only as a fallback method + * + * Nevertheless we use it according to the performance considerations + * + * @todo check if we can use some modification of Apache Lucene generation determination algorithm + * without performance problems + */ - if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) { - $this->_directory = $directory; - $this->_closeDirOnExit = false; - } else { - $this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory); - $this->_closeDirOnExit = true; - } + try { + for ($count = 0; $count < self::GENERATION_RETRIEVE_COUNT; $count++) { + // Try to get generation file + $genFile = $directory->getFileObject('segments.gen', false); + $format = $genFile->readInt(); + if ($format != (int)0xFFFFFFFE) { + throw new Zend_Search_Lucene_Exception('Wrong segments.gen file format'); + } - // Get a shared lock to the index - $this->_lock = $this->_directory->createFile('index.lock'); + $gen1 = $genFile->readLong(); + $gen2 = $genFile->readLong(); - $this->_segmentInfos = array(); + if ($gen1 == $gen2) { + return $gen1; + } - if ($create) { - // Throw an exception if index is under processing now - if (!$this->_lock->lock(LOCK_EX, true)) { - throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now'); + usleep(self::GENERATION_RETRIEVE_PAUSE * 1000); } - // Writer will create segments file for empty segments list - $this->_writer = new Zend_Search_Lucene_Index_Writer($this->_directory, $this->_segmentInfos, true); + // All passes are failed + throw new Zend_Search_Lucene_Exception('Index is under processing now'); + } catch (Zend_Search_Lucene_Exception $e) { + if (strpos($e->getMessage(), 'is not readable') !== false) { + try { + // Try to open old style segments file + $segmentsFile = $directory->getFileObject('segments', false); - if (!$this->_lock->lock(LOCK_SH)) { - throw new Zend_Search_Lucene_Exception('Can\'t reduce lock level from Exclusive to Shared'); + // It's pre-2.1 index + return 0; + } catch (Zend_Search_Lucene_Exception $e) { + if (strpos($e->getMessage(), 'is not readable') !== false) { + return -1; + } else { + throw $e; + } + } + } else { + throw $e; } - } else { - // Wait if index is under switching from one set of segments to another (Index_Writer::_updateSegments()) - if (!$this->_lock->lock(LOCK_SH)) { - throw new Zend_Search_Lucene_Exception('Can\'t obtain shared index lock'); - } - $this->_writer = null; } + return -1; + } + /** + * Get segments file name + * + * @param integer $generation + * @return string + */ + public static function getSegmentFileName($generation) + { + if ($generation == 0) { + return 'segments'; + } + + return 'segments_' . base_convert($generation, 10, 36); + } + + /** + * Read segments file for pre-2.1 Lucene index format + */ + private function _readPre21SegmentsFile() + { $segmentsFile = $this->_directory->getFileObject('segments'); $format = $segmentsFile->readInt(); @@ -248,13 +309,156 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface $segSize = $segmentsFile->readInt(); $this->_docCount += $segSize; - $this->_segmentInfos[] = - new Zend_Search_Lucene_Index_SegmentInfo($segName, - $segSize, - $this->_directory); + $this->_segmentInfos[$segName] = + new Zend_Search_Lucene_Index_SegmentInfo($this->_directory, + $segName, + $segSize); } } + /** + * Read segments file + * + * @throws Zend_Search_Lucene_Exception + */ + private function _readSegmentsFile() + { + $segmentsFile = $this->_directory->getFileObject(self::getSegmentFileName($this->_generation)); + + $format = $segmentsFile->readInt(); + + if ($format != (int)0xFFFFFFFD) { + throw new Zend_Search_Lucene_Exception('Wrong segments file format'); + } + + // read version + // $segmentsFile->readLong(); + $segmentsFile->readInt(); $segmentsFile->readInt(); + + // read segment name counter + $segmentsFile->readInt(); + + $segments = $segmentsFile->readInt(); + + $this->_docCount = 0; + + // read segmentInfos + for ($count = 0; $count < $segments; $count++) { + $segName = $segmentsFile->readString(); + $segSize = $segmentsFile->readInt(); + + // 2.1+ specific properties + //$delGen = $segmentsFile->readLong(); + $delGenHigh = $segmentsFile->readInt(); + $delGenLow = $segmentsFile->readInt(); + if ($delGenHigh == (int)0xFFFFFFFF && $delGenLow == (int)0xFFFFFFFF) { + $delGen = -1; // There are no deletes + } else { + $delGen = ($delGenHigh << 32) | $delGenLow; + } + + $hasSingleNormFile = $segmentsFile->readByte(); + $numField = $segmentsFile->readInt(); + + $normGens = array(); + if ($numField != (int)0xFFFFFFFF) { + for ($count1 = 0; $count1 < $numField; $count1++) { + $normGens[] = $segmentsFile->readLong(); + } + + throw new Zend_Search_Lucene_Exception('Separate norm files are not supported. Optimize index to use it with Zend_Search_Lucene.'); + } + + $isCompound = $segmentsFile->readByte(); + + + $this->_docCount += $segSize; + + $this->_segmentInfos[$segName] = + new Zend_Search_Lucene_Index_SegmentInfo($this->_directory, + $segName, + $segSize, + $delGen, + $hasSingleNormFile, + $isCompound); + } + } + + /** + * Opens the index. + * + * IndexReader constructor needs Directory as a parameter. It should be + * a string with a path to the index folder or a Directory object. + * + * @param mixed $directory + * @throws Zend_Search_Lucene_Exception + */ + public function __construct($directory = null, $create = false) + { + if ($directory === null) { + throw new Zend_Search_Exception('No index directory specified'); + } + + if ($directory instanceof Zend_Search_Lucene_Storage_Directory_Filesystem) { + $this->_directory = $directory; + $this->_closeDirOnExit = false; + } else { + $this->_directory = new Zend_Search_Lucene_Storage_Directory_Filesystem($directory); + $this->_closeDirOnExit = true; + } + + $this->_segmentInfos = array(); + + // Mark index as "under processing" to prevent other processes from premature index cleaning + Zend_Search_Lucene_LockManager::obtainReadLock($this->_directory); + + // Escalate read lock to prevent current generation index files to be deleted while opening process is not done + Zend_Search_Lucene_LockManager::escalateReadLock($this->_directory); + + + $this->_generation = self::getActualGeneration($this->_directory); + + if ($create) { + try { + Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory); + } catch (Zend_Search_Lucene_Exception $e) { + if (strpos($e->getMessage(), 'Can\'t obtain exclusive index lock') === false) { + throw $e; + } else { + throw new Zend_Search_Lucene_Exception('Can\'t create index. It\'s under processing now'); + } + } + + if ($this->_generation == -1) { + // Directory doesn't contain existing index, start from 1 + $this->_generation = 1; + $nameCounter = 0; + } else { + // Directory contains existing index + $segmentsFile = $this->_directory->getFileObject(self::getSegmentFileName($this->_generation)); + $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) + + $nameCounter = $segmentsFile->readInt(); + $this->_generation++; + } + + Zend_Search_Lucene_Index_Writer::createIndex($this->_directory, $this->_generation, $nameCounter); + + Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); + } + + if ($this->_generation == -1) { + throw new Zend_Search_Lucene_Exception('Index doesn\'t exists in the specified directory.'); + } else if ($this->_generation == 0) { + $this->_readPre21SegmentsFile(); + } else { + $this->_readSegmentsFile(); + } + + // De-escalate read lock to prevent current generation index files to be deleted while opening process is not done + Zend_Search_Lucene_LockManager::deEscalateReadLock($this->_directory); + } + /** * Close current index and free resources */ @@ -267,9 +471,9 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface $this->commit(); - // Free shared lock - $this->_lock->unlock(); - + // Release "under processing" flag + Zend_Search_Lucene_LockManager::releaseReadLock($this->_directory); + if ($this->_closeDirOnExit) { $this->_directory->close(); } @@ -431,6 +635,30 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface return self::$_defaultSearchField; } + /** + * Set result set limit. + * + * 0 (default) means no limit + * + * @param integer $limit + */ + public static function setResultSetLimit($limit) + { + self::$_resultSetLimit = $limit; + } + + /** + * Set result set limit. + * + * 0 means no limit + * + * @return integer + */ + public static function getResultSetLimit() + { + return self::$_resultSetLimit; + } + /** * Retrieve index maxBufferedDocs option * @@ -585,11 +813,15 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface $topScore = $docScore; } } + + if (self::$_resultSetLimit != 0 && count($hits) >= self::$_resultSetLimit) { + break; + } } if (count($hits) == 0) { // skip sorting, which may cause a error on empty index - return array(); + return array(); } if ($topScore > 1) { @@ -977,6 +1209,8 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface { $this->getIndexWriter()->addDocument($document); $this->_docCount++; + + $this->_hasChanges = true; } @@ -1002,14 +1236,12 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface foreach ($this->_segmentInfos as $segInfo) { $segInfo->writeChanges(); } - - $this->_hasChanges = false; - } - - if ($this->_writer !== null) { - $this->_writer->commit(); - + + $this->getIndexWriter()->commit(); + $this->_updateDocCount(); + + $this->_hasChanges = false; } } @@ -1059,9 +1291,7 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface $result[] = $segmentInfo->currentTerm(); } - $segmentInfo->nextTerm(); - // check, if segment dictionary is finished - if ($segmentInfo->currentTerm() !== null) { + if ($segmentInfo->nextTerm() !== null) { // Put segment back into the priority queue $segmentInfoQueue->put($segmentInfo); } @@ -1071,6 +1301,125 @@ class Zend_Search_Lucene implements Zend_Search_Lucene_Interface } + /** + * Terms stream queue + * + * @var Zend_Search_Lucene_Index_SegmentInfoPriorityQueue + */ + private $_termsStreamQueue = null; + + /** + * Last Term in a terms stream + * + * @var Zend_Search_Lucene_Index_Term + */ + private $_lastTerm = null; + + /** + * Reset terms stream. + */ + public function resetTermsStream() + { + $this->_termsStreamQueue = new Zend_Search_Lucene_Index_SegmentInfoPriorityQueue(); + + foreach ($this->_segmentInfos as $segmentInfo) { + $segmentInfo->reset(); + + // Skip "empty" segments + if ($segmentInfo->currentTerm() !== null) { + $this->_termsStreamQueue->put($segmentInfo); + } + } + + $this->nextTerm(); + } + + /** + * Skip terms stream up to specified term preffix. + * + * Prefix contains fully specified field info and portion of searched term + * + * @param Zend_Search_Lucene_Index_Term $prefix + */ + public function skipTo(Zend_Search_Lucene_Index_Term $prefix) + { + $segments = array(); + + while (($segmentInfo = $this->_termsStreamQueue->pop()) !== null) { + $segments[] = $segmentInfo; + } + + foreach ($segments as $segmentInfo) { + $segmentInfo->skipTo($prefix); + + if ($segmentInfo->currentTerm() !== null) { + $this->_termsStreamQueue->put($segmentInfo); + } + } + + $this->nextTerm(); + } + + /** + * Scans terms dictionary and returns next term + * + * @return Zend_Search_Lucene_Index_Term|null + */ + public function nextTerm() + { + while (($segmentInfo = $this->_termsStreamQueue->pop()) !== null) { + if ($this->_termsStreamQueue->top() === null || + $this->_termsStreamQueue->top()->currentTerm()->key() != + $segmentInfo->currentTerm()->key()) { + // We got new term + $this->_lastTerm = $segmentInfo->currentTerm(); + + if ($segmentInfo->nextTerm() !== null) { + // Put segment back into the priority queue + $this->_termsStreamQueue->put($segmentInfo); + } + + return $this->_lastTerm; + } + + if ($segmentInfo->nextTerm() !== null) { + // Put segment back into the priority queue + $this->_termsStreamQueue->put($segmentInfo); + } + } + + // End of stream + $this->_lastTerm = null; + + return null; + } + + /** + * Returns term in current position + * + * @return Zend_Search_Lucene_Index_Term|null + */ + public function currentTerm() + { + return $this->_lastTerm; + } + + /** + * Close terms stream + * + * Should be used for resources clean up if stream is not read up to the end + */ + public function closeTermsStream() + { + while (($segmentInfo = $this->_termsStreamQueue->pop()) !== null) { + $segmentInfo->closeTermsStream(); + } + + $this->_termsStreamQueue = null; + $this->_lastTerm = null; + } + + /************************************************************************* @todo UNIMPLEMENTED *************************************************************************/ diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer.php b/search/Zend/Search/Lucene/Analysis/Analyzer.php index 7a29c763d90..def78ac5906 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer.php @@ -15,37 +15,43 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Token */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Token.php"; /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php"; + +/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8_CaseInsensitive */ +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8/CaseInsensitive.php"; /** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php"; + +/** Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive */ +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num/CaseInsensitive.php"; /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php"; /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php"; /** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php"; /** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php"; /** Zend_Search_Lucene_Analysis_TokenFilter_StopWords */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php'; +require_once 'Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php'; /** Zend_Search_Lucene_Analysis_TokenFilter_ShortWords */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php"; /** @@ -61,7 +67,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/Shor * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -97,9 +103,10 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer * @param string $data * @return array */ - public function tokenize($data, $encoding = 'UTF-8') + public function tokenize($data, $encoding = '') { $this->setInput($data, $encoding); + $tokenList = array(); while (($nextToken = $this->nextToken()) !== null) { $tokenList[] = $nextToken; @@ -160,7 +167,7 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer public static function getDefault() { if (!self::$_defaultImpl instanceof Zend_Search_Lucene_Analysis_Analyzer) { - self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8(); + self::$_defaultImpl = new Zend_Search_Lucene_Analysis_Analyzer_Common_Text_CaseInsensitive(); } return self::$_defaultImpl; diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php index c518e930717..01164d8874f 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer.php'; /** @@ -34,7 +34,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_Lucene_Analysis_Analyzer @@ -68,7 +68,7 @@ abstract class Zend_Search_Lucene_Analysis_Analyzer_Common extends Zend_Search_L foreach ($this->_filters as $filter) { $token = $filter->normalize($token); - // resulting token can be null if the filter removed it + // resulting token can be null if the filter removes it if (is_null($token)) { return null; } diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php index d084ebc4c5f..46bf196e12b 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -53,7 +53,9 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucen } // convert input into ascii - $this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input); + //$this->_input = iconv($this->_encoding, 'ASCII//TRANSLIT', $this->_input); + $this->_input = mb_convert_encoding($this->_input, 'ASCII', 'auto'); + $this->_encoding = 'ASCII'; } @@ -75,7 +77,7 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucen if (! preg_match('/[a-zA-Z]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) { // It covers both cases a) there are no matches (preg_match(...) === 0) // b) error occured (preg_match(...) === FALSE) - return null; + return null; } $str = $match[0][0]; diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php index d9f786a8703..4e3dd662a73 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text/CaseInsensitive.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common_Text */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/Text.php'; /** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php'; +require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php index d68b594a077..b8aca5d95f7 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -74,7 +74,7 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum extends Zend_Search_Lu if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) { // It covers both cases a) there are no matches (preg_match(...) === 0) // b) error occured (preg_match(...) === FALSE) - return null; + return null; } $str = $match[0][0]; diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php index 6eab4372291..89873f10f17 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum/CaseInsensitive.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common/TextNum.php'; /** Zend_Search_Lucene_Analysis_TokenFilter_LowerCase */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php'; +require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php index 674a3d9e643..768e7847c6f 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -47,13 +47,20 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 extends Zend_Search_Lucen * @var integer */ private $_bytePosition; - + /** - * Stream length + * Object constructor * - * @var integer + * @throws Zend_Search_Lucene_Exception */ - private $_streamLength; + public function __construct() + { + if (@preg_match('/\pL/u', 'a') != 1) { + // PCRE unicode support is turned off + require_once 'Zend/Search/Lucene/Exception.php'; + throw new Zend_Search_Lucene_Exception('Utf8 analyzer needs PCRE unicode support to be enabled.'); + } + } /** * Reset token stream @@ -66,56 +73,9 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 extends Zend_Search_Lucen // convert input into UTF-8 if (strcasecmp($this->_encoding, 'utf8' ) != 0 && strcasecmp($this->_encoding, 'utf-8') != 0 ) { - $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input); + $this->_input = @iconv($this->_encoding, 'UTF-8', $this->_input); $this->_encoding = 'UTF-8'; } - - // Get UTF-8 string length. - // It also checks if it's a correct utf-8 string - $this->_streamLength = iconv_strlen($this->_input, 'UTF-8'); - } - - /** - * Check, that character is a letter - * - * @param string $char - * @return boolean - */ - private static function _isAlpha($char) - { - if (strlen($char) > 1) { - // It's an UTF-8 character - return true; - } - - return ctype_alpha($char); - } - - /** - * Get next UTF-8 char - * - * @param string $char - * @return boolean - */ - private function _nextChar() - { - $char = $this->_input[$this->_bytePosition++]; - - if (( ord($char) & 0xC0 ) == 0xC0) { - $addBytes = 1; - if (ord($char) & 0x20 ) { - $addBytes++; - if (ord($char) & 0x10 ) { - $addBytes++; - } - } - $char .= substr($this->_input, $this->_bytePosition, $addBytes); - $this->_bytePosition += $addBytes; - } - - $this->_position++; - - return $char; } /** @@ -131,39 +91,35 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8 extends Zend_Search_Lucen return null; } - while ($this->_position < $this->_streamLength) { - // skip white space - while ($this->_position < $this->_streamLength && - !self::_isAlpha($char = $this->_nextChar())) { - $char = ''; - } - - $termStartPosition = $this->_position - 1; - $termText = $char; - - // read token - while ($this->_position < $this->_streamLength && - self::_isAlpha($char = $this->_nextChar())) { - $termText .= $char; - } - - // Empty token, end of stream. - if ($termText == '') { + do { + if (! preg_match('/[\p{L}]+/u', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_bytePosition)) { + // It covers both cases a) there are no matches (preg_match(...) === 0) + // b) error occured (preg_match(...) === FALSE) return null; } - $token = new Zend_Search_Lucene_Analysis_Token( - $termText, - $termStartPosition, - $this->_position - 1); - $token = $this->normalize($token); - if ($token !== null) { - return $token; - } - // Continue if token is skipped - } + // matched string + $matchedWord = $match[0][0]; + + // binary position of the matched word in the input stream + $binStartPos = $match[0][1]; + + // character position of the matched word in the input stream + $startPos = $this->_position + + iconv_strlen(substr($this->_input, + $this->_bytePosition, + $binStartPos - $this->_bytePosition), + 'UTF-8'); + // character postion of the end of matched word in the input stream + $endPos = $startPos + iconv_strlen($matchedWord, 'UTF-8'); - return null; + $this->_bytePosition = $binStartPos + strlen($matchedWord); + $this->_position = $endPos; + + $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($matchedWord, $startPos, $endPos)); + } while ($token === null); // try again if token is skipped + + return $token; } } diff --git a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php index 982b55418fd..74c89be0c21 100644 --- a/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php +++ b/search/Zend/Search/Lucene/Analysis/Analyzer/Common/Utf8Num.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Analyzer_Common */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer/Common.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -49,11 +49,18 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num extends Zend_Search_Lu private $_bytePosition; /** - * Stream length + * Object constructor * - * @var integer + * @throws Zend_Search_Lucene_Exception */ - private $_streamLength; + public function __construct() + { + if (@preg_match('/\pL/u', 'a') != 1) { + // PCRE unicode support is turned off + require_once 'Zend/Search/Lucene/Exception.php'; + throw new Zend_Search_Lucene_Exception('Utf8Num analyzer needs PCRE unicode support to be enabled.'); + } + } /** * Reset token stream @@ -69,53 +76,6 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num extends Zend_Search_Lu $this->_input = iconv($this->_encoding, 'UTF-8', $this->_input); $this->_encoding = 'UTF-8'; } - - // Get UTF-8 string length. - // It also checks if it's a correct utf-8 string - $this->_streamLength = iconv_strlen($this->_input, 'UTF-8'); - } - - /** - * Check, that character is a letter - * - * @param string $char - * @return boolean - */ - private static function _isAlNum($char) - { - if (strlen($char) > 1) { - // It's an UTF-8 character - return true; - } - - return ctype_alnum($char); - } - - /** - * Get next UTF-8 char - * - * @param string $char - * @return boolean - */ - private function _nextChar() - { - $char = $this->_input[$this->_bytePosition++]; - - if (( ord($char) & 0xC0 ) == 0xC0) { - $addBytes = 1; - if (ord($char) & 0x20 ) { - $addBytes++; - if (ord($char) & 0x10 ) { - $addBytes++; - } - } - $char .= substr($this->_input, $this->_bytePosition, $addBytes); - $this->_bytePosition += $addBytes; - } - - $this->_position++; - - return $char; } /** @@ -131,39 +91,35 @@ class Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num extends Zend_Search_Lu return null; } - while ($this->_position < $this->_streamLength) { - // skip white space - while ($this->_position < $this->_streamLength && - !self::_isAlNum($char = $this->_nextChar())) { - $char = ''; - } - - $termStartPosition = $this->_position - 1; - $termText = $char; - - // read token - while ($this->_position < $this->_streamLength && - self::_isAlNum($char = $this->_nextChar())) { - $termText .= $char; - } - - // Empty token, end of stream. - if ($termText == '') { + do { + if (! preg_match('/[\p{L}\p{N}]+/u', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_bytePosition)) { + // It covers both cases a) there are no matches (preg_match(...) === 0) + // b) error occured (preg_match(...) === FALSE) return null; } - $token = new Zend_Search_Lucene_Analysis_Token( - $termText, - $termStartPosition, - $this->_position - 1); - $token = $this->normalize($token); - if ($token !== null) { - return $token; - } - // Continue if token is skipped - } + // matched string + $matchedWord = $match[0][0]; + + // binary position of the matched word in the input stream + $binStartPos = $match[0][1]; + + // character position of the matched word in the input stream + $startPos = $this->_position + + iconv_strlen(substr($this->_input, + $this->_bytePosition, + $binStartPos - $this->_bytePosition), + 'UTF-8'); + // character postion of the end of matched word in the input stream + $endPos = $startPos + iconv_strlen($matchedWord, 'UTF-8'); - return null; + $this->_bytePosition = $binStartPos + strlen($matchedWord); + $this->_position = $endPos; + + $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($matchedWord, $startPos, $endPos)); + } while ($token === null); // try again if token is skipped + + return $token; } } diff --git a/search/Zend/Search/Lucene/Analysis/Token.php b/search/Zend/Search/Lucene/Analysis/Token.php index 91586b01f72..3097992a9b8 100644 --- a/search/Zend/Search/Lucene/Analysis/Token.php +++ b/search/Zend/Search/Lucene/Analysis/Token.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,7 +24,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Analysis_Token diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter.php b/search/Zend/Search/Lucene/Analysis/TokenFilter.php index 4d559a93b7f..5c582024a15 100644 --- a/search/Zend/Search/Lucene/Analysis/TokenFilter.php +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_Token */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php'; +require_once 'Zend/Search/Lucene/Analysis/Token.php'; /** @@ -30,7 +30,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Token.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php b/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php index 01d25c67452..9fb545cb36f 100644 --- a/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_TokenFilter */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php'; +require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php'; /** @@ -30,7 +30,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php' * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCaseUtf8.php b/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCaseUtf8.php index 90d0bc117a7..78cb5e680f8 100644 --- a/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCaseUtf8.php +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter/LowerCaseUtf8.php @@ -21,7 +21,7 @@ /** Zend_Search_Lucene_Analysis_TokenFilter */ -require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Analysis/TokenFilter.php"; /** @@ -41,9 +41,10 @@ class Zend_Search_Lucene_Analysis_TokenFilter_LowerCaseUtf8 extends Zend_Search_ */ public function __construct() { + global $CFG; if (!function_exists('mb_strtolower')) { // mbstring extension is disabled - require_once 'Zend/Search/Lucene/Exception.php'; + require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Exception.php"; throw new Zend_Search_Lucene_Exception('Utf8 compatible lower case filter needs mbstring extension to be enabled.'); } } diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php b/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php index 83abfb22c3f..50c3f50d60a 100644 --- a/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter/ShortWords.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_TokenFilter */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php'; +require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php'; /** @@ -30,7 +30,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php' * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ diff --git a/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php b/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php index f85d5d55d8e..55b7567c1bf 100644 --- a/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php +++ b/search/Zend/Search/Lucene/Analysis/TokenFilter/StopWords.php @@ -15,14 +15,14 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Analysis_TokenFilter */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/TokenFilter.php'; -require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; +require_once 'Zend/Search/Lucene/Analysis/TokenFilter.php'; +require_once 'Zend/Search/Exception.php'; /** @@ -34,14 +34,14 @@ require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Analysis - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Analysis_TokenFilter_StopWords extends Zend_Search_Lucene_Analysis_TokenFilter { /** - * Minimum allowed term length + * Stop Words * @var array */ private $_stopSet; @@ -63,7 +63,6 @@ class Zend_Search_Lucene_Analysis_TokenFilter_StopWords extends Zend_Search_Luce */ public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) { if (array_key_exists($srcToken->getTermText(), $this->_stopSet)) { - $t = $srcToken->getTermText(); return null; } else { return $srcToken; diff --git a/search/Zend/Search/Lucene/Document.php b/search/Zend/Search/Lucene/Document.php index 6309719568d..de4281efa7c 100644 --- a/search/Zend/Search/Lucene/Document.php +++ b/search/Zend/Search/Lucene/Document.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Field */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Field.php'; +require_once "Zend/Search/Lucene/Field.php"; /** @@ -30,7 +30,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Field.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Document @@ -54,10 +54,10 @@ class Zend_Search_Lucene_Document * @param $offset * @return string */ - public function __get($offset) - { - return $this->getFieldValue($offset); - } + public function __get($offset) + { + return $this->getFieldValue($offset); + } /** @@ -78,7 +78,7 @@ class Zend_Search_Lucene_Document */ public function getFieldNames() { - return array_keys($this->_fields); + return array_keys($this->_fields); } @@ -105,7 +105,7 @@ class Zend_Search_Lucene_Document */ public function getFieldValue($fieldName) { - return $this->getField($fieldName)->value; + return $this->getField($fieldName)->value; } /** @@ -116,6 +116,6 @@ class Zend_Search_Lucene_Document */ public function getFieldUtf8Value($fieldName) { - return $this->getField($fieldName)->getUtf8Value(); + return $this->getField($fieldName)->getUtf8Value(); } } diff --git a/search/Zend/Search/Lucene/Document/Html.php b/search/Zend/Search/Lucene/Document/Html.php index c10c8236c5c..e7cd2f90f4f 100644 --- a/search/Zend/Search/Lucene/Document/Html.php +++ b/search/Zend/Search/Lucene/Document/Html.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Document */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document.php'; +require_once 'Zend/Search/Lucene/Document.php'; /** @@ -30,7 +30,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Document_Html extends Zend_Search_Lucene_Document @@ -290,7 +290,7 @@ class Zend_Search_Lucene_Document_Html extends Zend_Search_Lucene_Document $xpath = new DOMXPath($this->_doc); - $matchedNodes = $xpath->query("/html/body/*"); + $matchedNodes = $xpath->query("/html/body"); foreach ($matchedNodes as $matchedNode) { $this->_highlightNode($matchedNode, $wordsToHighlightFlipped, $color); } diff --git a/search/Zend/Search/Lucene/Exception.php b/search/Zend/Search/Lucene/Exception.php index 9d06e89522a..d08b30dbb1d 100644 --- a/search/Zend/Search/Lucene/Exception.php +++ b/search/Zend/Search/Lucene/Exception.php @@ -14,7 +14,7 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -22,13 +22,13 @@ /** * Framework base exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; +require_once "Zend/Search/Exception.php"; /** * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Exception extends Zend_Search_Exception diff --git a/search/Zend/Search/Lucene/FSM.php b/search/Zend/Search/Lucene/FSM.php index 31c9069fe4c..55e758fe7da 100644 --- a/search/Zend/Search/Lucene/FSM.php +++ b/search/Zend/Search/Lucene/FSM.php @@ -14,16 +14,16 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_FSMAction */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSMAction.php'; +require_once 'Zend/Search/Lucene/FSMAction.php'; /** Zend_Search_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; +require_once 'Zend/Search/Exception.php'; /** @@ -37,7 +37,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Exception.php'; * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_FSM diff --git a/search/Zend/Search/Lucene/FSMAction.php b/search/Zend/Search/Lucene/FSMAction.php index 606de7b613f..fdbd6ff659a 100644 --- a/search/Zend/Search/Lucene/FSMAction.php +++ b/search/Zend/Search/Lucene/FSMAction.php @@ -14,7 +14,7 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -25,7 +25,7 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_FSMAction diff --git a/search/Zend/Search/Lucene/Field.php b/search/Zend/Search/Lucene/Field.php index 86cd22ccea1..d02f1d94979 100644 --- a/search/Zend/Search/Lucene/Field.php +++ b/search/Zend/Search/Lucene/Field.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -31,7 +31,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Document - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Field diff --git a/search/Zend/Search/Lucene/Index/DictionaryLoader.php b/search/Zend/Search/Lucene/Index/DictionaryLoader.php index d3f0669c814..f01be7b4b13 100644 --- a/search/Zend/Search/Lucene/Index/DictionaryLoader.php +++ b/search/Zend/Search/Lucene/Index/DictionaryLoader.php @@ -15,13 +15,13 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** @@ -35,7 +35,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_DictionaryLoader @@ -61,11 +61,12 @@ class Zend_Search_Lucene_Index_DictionaryLoader // $tiVersion = $tiiFile->readInt(); $tiVersion = ord($data[0]) << 24 | ord($data[1]) << 16 | ord($data[2]) << 8 | ord($data[3]); $pos += 4; - if ($tiVersion != (int)0xFFFFFFFE) { + if ($tiVersion != (int)0xFFFFFFFE /* pre-2.1 format */ && + $tiVersion != (int)0xFFFFFFFD /* 2.1+ format */) { throw new Zend_Search_Lucene_Exception('Wrong TermInfoIndexFile file format'); } - // $indexTermCount = = $tiiFile->readLong(); + // $indexTermCount = $tiiFile->readLong(); if (PHP_INT_SIZE > 4) { $indexTermCount = ord($data[$pos]) << 56 | ord($data[$pos+1]) << 48 | @@ -101,6 +102,11 @@ class Zend_Search_Lucene_Index_DictionaryLoader throw new Zend_Search_Lucene_Exception('Wrong number of terms in a term dictionary index'); } + if ($tiVersion == (int)0xFFFFFFFD /* 2.1+ format */) { + /* Skip MaxSkipLevels value */ + $pos += 4; + } + $prevTerm = ''; $freqPointer = 0; $proxPointer = 0; diff --git a/search/Zend/Search/Lucene/Index/FieldInfo.php b/search/Zend/Search/Lucene/Index/FieldInfo.php index 1d138b6575f..0a91872e3e5 100644 --- a/search/Zend/Search/Lucene/Index/FieldInfo.php +++ b/search/Zend/Search/Lucene/Index/FieldInfo.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,7 +24,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_FieldInfo diff --git a/search/Zend/Search/Lucene/Index/SegmentInfo.php b/search/Zend/Search/Lucene/Index/SegmentInfo.php index c6f7868fd50..d35d7f91b50 100644 --- a/search/Zend/Search/Lucene/Index/SegmentInfo.php +++ b/search/Zend/Search/Lucene/Index/SegmentInfo.php @@ -15,23 +15,30 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Index_DictionaryLoader */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/DictionaryLoader.php'; +require_once 'Zend/Search/Lucene/Index/DictionaryLoader.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; + +/** Zend_Search_Lucene_LockManager */ +if (@$CFG->block_search_softlock){ + require_once "Zend/Search/Lucene/SoftLockManager.php"; +} else { + require_once "Zend/Search/Lucene/LockManager.php"; +} /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_SegmentInfo @@ -110,6 +117,35 @@ class Zend_Search_Lucene_Index_SegmentInfo */ private $_segFileSizes; + /** + * Delete file generation number + * + * -1 means 'there is no delete file' + * 0 means pre-2.1 format delete file + * X specifies used delete file + * + * @var integer + */ + private $_delGen; + + /** + * Segment has single norms file + * + * If true then one .nrm file is used for all fields + * Otherwise .fN files are used + * + * @var boolean + */ + private $_hasSingleNormFile; + + /** + * Use compound segment file (*.cfs) to collect all other segment files + * (excluding .del files) + * + * @var boolean + */ + private $_isCompound; + /** * File system adapter. @@ -136,7 +172,7 @@ class Zend_Search_Lucene_Index_SegmentInfo * * @var mixed */ - private $_deleted; + private $_deleted = null; /** * $this->_deleted update flag @@ -147,22 +183,45 @@ class Zend_Search_Lucene_Index_SegmentInfo /** - * Zend_Search_Lucene_Index_SegmentInfo constructor needs Segmentname, - * Documents count and Directory as a parameter. + * Zend_Search_Lucene_Index_SegmentInfo constructor * + * @param Zend_Search_Lucene_Storage_Directory $directory * @param string $name * @param integer $docCount - * @param Zend_Search_Lucene_Storage_Directory $directory + * @param integer $delGen + * @param boolean $isCompound */ - public function __construct($name, $docCount, $directory) + public function __construct(Zend_Search_Lucene_Storage_Directory $directory, $name, $docCount, $delGen = 0, $hasSingleNormFile = false, $isCompound = null) { - $this->_name = $name; - $this->_docCount = $docCount; $this->_directory = $directory; - $this->_termDictionary = null; + $this->_name = $name; + $this->_docCount = $docCount; + $this->_hasSingleNormFile = $hasSingleNormFile; + $this->_delGen = $delGen; + $this->_termDictionary = null; + + if (!is_null($isCompound)) { + $this->_isCompound = $isCompound; + } else { + // It's a pre-2.1 segment + // detect if it uses compond file + $this->_isCompound = true; + + try { + // Try to open compound file + $this->_directory->getFileObject($name . '.cfs'); + } catch (Zend_Search_Lucene_Exception $e) { + if (strpos($e->getMessage(), 'is not readable') !== false) { + // Compound file is not found or is not readable + $this->_isCompound = false; + } else { + throw $e; + } + } + } $this->_segFiles = array(); - if ($this->_directory->fileExists($name . '.cfs')) { + if ($this->_isCompound) { $cfsFile = $this->_directory->getFileObject($name . '.cfs'); $segFilesCount = $cfsFile->readVInt(); @@ -202,37 +261,83 @@ class Zend_Search_Lucene_Index_SegmentInfo array_multisort($fieldNames, SORT_ASC, SORT_REGULAR, $fieldNums); $this->_fieldsDicPositions = array_flip($fieldNums); - try { - $delFile = $this->openCompoundFile('.del'); + if ($this->_delGen == -1) { + // There is no delete file for this segment + // Do nothing + } else if ($this->_delGen == 0) { + // It's a segment with pre-2.1 format delete file + // Try to find delete file + try { + // '.del' files always stored in a separate file + // Segment compound is not used + $delFile = $this->_directory->getFileObject($this->_name . '.del'); - $byteCount = $delFile->readInt(); - $byteCount = ceil($byteCount/8); - $bitCount = $delFile->readInt(); + $byteCount = $delFile->readInt(); + $byteCount = ceil($byteCount/8); + $bitCount = $delFile->readInt(); - if ($bitCount == 0) { - $delBytes = ''; - } else { - $delBytes = $delFile->readBytes($byteCount); - } + if ($bitCount == 0) { + $delBytes = ''; + } else { + $delBytes = $delFile->readBytes($byteCount); + } - if (extension_loaded('bitset')) { - $this->_deleted = $delBytes; - } else { - $this->_deleted = array(); - for ($count = 0; $count < $byteCount; $count++) { - $byte = ord($delBytes{$count}); - for ($bit = 0; $bit < 8; $bit++) { - if ($byte & (1<<$bit)) { - $this->_deleted[$count*8 + $bit] = 1; + if (extension_loaded('bitset')) { + $this->_deleted = $delBytes; + } else { + $this->_deleted = array(); + for ($count = 0; $count < $byteCount; $count++) { + $byte = ord($delBytes{$count}); + for ($bit = 0; $bit < 8; $bit++) { + if ($byte & (1<<$bit)) { + $this->_deleted[$count*8 + $bit] = 1; + } } } } + } catch(Zend_Search_Exception $e) { + if (strpos($e->getMessage(), 'is not readable') === false ) { + throw $e; + } + // There is no delete file + // Do nothing } - } catch(Zend_Search_Exception $e) { - if (strpos($e->getMessage(), 'compound file doesn\'t contain') !== false ) { - $this->_deleted = null; + } else { + // It's 2.1+ format delete file + $delFile = $this->_directory->getFileObject($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del'); + + $format = $delFile->readInt(); + + if ($format == (int)0xFFFFFFFF) { + /** + * @todo Implement support of DGaps delete file format. + * See Lucene file format for details - http://lucene.apache.org/java/docs/fileformats.html#Deleted%20Documents + */ + throw new Zend_Search_Lucene_Exception('DGaps delete file format is not supported. Optimize index to use it with Zend_Search_Lucene'); } else { - throw $e; + // $format is actually byte count + $byteCount = ceil($format/8); + $bitCount = $delFile->readInt(); + + if ($bitCount == 0) { + $delBytes = ''; + } else { + $delBytes = $delFile->readBytes($byteCount); + } + + if (extension_loaded('bitset')) { + $this->_deleted = $delBytes; + } else { + $this->_deleted = array(); + for ($count = 0; $count < $byteCount; $count++) { + $byte = ord($delBytes{$count}); + for ($bit = 0; $bit < 8; $bit++) { + if ($byte & (1<<$bit)) { + $this->_deleted[$count*8 + $bit] = 1; + } + } + } + } } } } @@ -249,13 +354,12 @@ class Zend_Search_Lucene_Index_SegmentInfo { $filename = $this->_name . $extension; - // Try to open common file first - if ($this->_directory->fileExists($filename)) { + if (!$this->_isCompound) { return $this->_directory->getFileObject($filename, $shareHandler); } if( !isset($this->_segFiles[$filename]) ) { - throw new Zend_Search_Lucene_Exception('Index compound file doesn\'t contain ' + throw new Zend_Search_Lucene_Exception('Segment compound file doesn\'t contain ' . $filename . ' file.' ); } @@ -343,6 +447,16 @@ class Zend_Search_Lucene_Index_SegmentInfo return $this->_fields; } + /** + * Returns actual deletions file generation number. + * + * @return integer + */ + public function getDelGen() + { + return $this->_delGen; + } + /** * Returns the total number of documents in this segment (including deleted documents). * @@ -431,6 +545,41 @@ class Zend_Search_Lucene_Index_SegmentInfo } } + /** + * Load terms dictionary index + * + * @throws Zend_Search_Lucene_Exception + */ + private function _loadDictionaryIndex() + { + // Check, if index is already serialized + if ($this->_directory->fileExists($this->_name . '.sti')) { + // Load serialized dictionary index data + $stiFile = $this->_directory->getFileObject($this->_name . '.sti'); + $stiFileData = $stiFile->readBytes($this->_directory->fileLength($this->_name . '.sti')); + + // Load dictionary index data + if (($unserializedData = @unserialize($stiFileData)) !== false) { + list($this->_termDictionary, $this->_termDictionaryInfos) = $unserializedData; + return; + } + } + + // Load data from .tii file and generate .sti file + + // Prefetch dictionary index data + $tiiFile = $this->openCompoundFile('.tii'); + $tiiFileData = $tiiFile->readBytes($this->compoundFileLength('.tii')); + + // Load dictionary index data + list($this->_termDictionary, $this->_termDictionaryInfos) = + Zend_Search_Lucene_Index_DictionaryLoader::load($tiiFileData); + + $stiFileData = serialize(array($this->_termDictionary, $this->_termDictionaryInfos)); + $stiFile = $this->_directory->createFile($this->_name . '.sti'); + $stiFile->writeBytes($stiFileData); + } + /** * Scans terms dictionary and returns term info * @@ -452,32 +601,9 @@ class Zend_Search_Lucene_Index_SegmentInfo if ($this->_termDictionary === null) { - // Check, if index is already serialized - if ($this->_directory->fileExists($this->_name . '.sti')) { - // Prefetch dictionary index data - $stiFile = $this->_directory->getFileObject($this->_name . '.sti'); - $stiFileData = $stiFile->readBytes($this->_directory->fileLength($this->_name . '.sti')); - - // Load dictionary index data - list($this->_termDictionary, $this->_termDictionaryInfos) = unserialize($stiFileData); - } else { - // Prefetch dictionary index data - $tiiFile = $this->openCompoundFile('.tii'); - $tiiFileData = $tiiFile->readBytes($this->compoundFileLength('.tii')); - - // Load dictionary index data - list($this->_termDictionary, $this->_termDictionaryInfos) = - Zend_Search_Lucene_Index_DictionaryLoader::load($tiiFileData); - - $stiFileData = serialize(array($this->_termDictionary, $this->_termDictionaryInfos)); - $stiFile = $this->_directory->createFile($this->_name . '.sti'); - $stiFile->writeBytes($stiFileData); - } - + $this->_loadDictionaryIndex(); } - - $searchField = $this->getFieldNum($term->field); if ($searchField == -1) { @@ -526,15 +652,19 @@ class Zend_Search_Lucene_Index_SegmentInfo $tisFile = $this->openCompoundFile('.tis'); $tiVersion = $tisFile->readInt(); - if ($tiVersion != (int)0xFFFFFFFE) { + if ($tiVersion != (int)0xFFFFFFFE /* pre-2.1 format */ && + $tiVersion != (int)0xFFFFFFFD /* 2.1+ format */) { throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format'); } $termCount = $tisFile->readLong(); $indexInterval = $tisFile->readInt(); $skipInterval = $tisFile->readInt(); + if ($tiVersion == (int)0xFFFFFFFD /* 2.1+ format */) { + $maxSkipLevels = $tisFile->readInt(); + } - $tisFile->seek($prevTermInfo[4] /* indexPointer */ - 20 /* header size*/, SEEK_CUR); + $tisFile->seek($prevTermInfo[4] /* indexPointer */ - (($tiVersion == (int)0xFFFFFFFD)? 24 : 20) /* header size*/, SEEK_CUR); $termValue = $prevTerm[1] /* text */; $termFieldNum = $prevTerm[0] /* field */; @@ -666,11 +796,29 @@ class Zend_Search_Lucene_Index_SegmentInfo * Load normalizatin factors from an index file * * @param integer $fieldNum + * @throws Zend_Search_Lucene_Exception */ private function _loadNorm($fieldNum) { - $fFile = $this->openCompoundFile('.f' . $fieldNum); - $this->_norms[$fieldNum] = $fFile->readBytes($this->_docCount); + if ($this->_hasSingleNormFile) { + $normfFile = $this->openCompoundFile('.nrm'); + + $header = $normfFile->readBytes(3); + $headerFormatVersion = $normfFile->readByte(); + + if ($header != 'NRM' || $headerFormatVersion != (int)0xFF) { + throw new Zend_Search_Lucene_Exception('Wrong norms file format.'); + } + + foreach ($this->_fields as $fieldNum => $fieldInfo) { + if ($fieldInfo->isIndexed) { + $this->_norms[$fieldNum] = $normfFile->readBytes($this->_docCount); + } + } + } else { + $fFile = $this->openCompoundFile('.f' . $fieldNum); + $this->_norms[$fieldNum] = $fFile->readBytes($this->_docCount); + } } /** @@ -731,6 +879,26 @@ class Zend_Search_Lucene_Index_SegmentInfo } + /** + * Returns true if segment has single norms file. + * + * @return boolean + */ + public function hasSingleNormFile() + { + return $this->_hasSingleNormFile ? 1 : 0; + } + + /** + * Returns true if segment is stored using compound segment file. + * + * @return boolean + */ + public function isCompound() + { + return $this->_isCompound ? 1 : 0; + } + /** * Deletes a document from the index segment. * $id is an internal document id @@ -803,7 +971,35 @@ class Zend_Search_Lucene_Index_SegmentInfo } - $delFile = $this->_directory->createFile($this->_name . '.del'); + // Get new generation number + Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory); + + $delFileList = array(); + foreach ($this->_directory->fileList() as $file) { + if ($file == $this->_name . '.del') { + // Matches .del file name + $delFileList[] = 0; + } else if (preg_match('/^' . $this->_name . '_([a-zA-Z0-9]+)\.del$/i', $file, $matches)) { + // Matches _NNN.del file names + $delFileList[] = (int)base_convert($matches[1], 36, 10); + } + } + + if (count($delFileList) == 0) { + // There is no deletions file for current segment in the directory + // Set detetions file generation number to 1 + $this->_delGen = 1; + } else { + // There are some deletions files for current segment in the directory + // Set detetions file generation number to the highest + 1 + $this->_delGen = max($delFileList) + 1; + } + + $delFile = $this->_directory->createFile($this->_name . '_' . base_convert($this->_delGen, 10, 36) . '.del'); + + Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); + + $delFile->writeInt($this->_docCount); $delFile->writeInt($bitCount); $delFile->writeBytes($delBytes); @@ -820,6 +1016,13 @@ class Zend_Search_Lucene_Index_SegmentInfo */ private $_tisFile = null; + /** + * Actual offset of the .tis file data + * + * @var integer + */ + private $_tisFileOffset; + /** * Frequencies File object for stream like terms reading * @@ -828,7 +1031,7 @@ class Zend_Search_Lucene_Index_SegmentInfo private $_frqFile = null; /** - * Offset of the .frq file in the compound file + * Actual offset of the .frq file data * * @var integer */ @@ -842,7 +1045,7 @@ class Zend_Search_Lucene_Index_SegmentInfo private $_prxFile = null; /** - * Offset of the .prx file in the compound file + * Actual offset of the .prx file in the compound file * * @var integer */ @@ -850,12 +1053,26 @@ class Zend_Search_Lucene_Index_SegmentInfo /** - * Number of terms in term stream + * Actual number of terms in term stream * * @var integer */ private $_termCount = 0; + /** + * Overall number of terms in term stream + * + * @var integer + */ + private $_termNum = 0; + + /** + * Segment index interval + * + * @var integer + */ + private $_indexInterval; + /** * Segment skip interval * @@ -891,10 +1108,32 @@ class Zend_Search_Lucene_Index_SegmentInfo * An array of all term positions in the documents. * Array structure: array( docId => array( pos1, pos2, ...), ...) * - * @var array + * Is set to null if term positions loading has to be skipped + * + * @var array|null */ private $_lastTermPositions; + + /** + * Terms scan mode + * + * Values: + * + * self::SM_TERMS_ONLY - terms are scanned, no additional info is retrieved + * self::SM_MERGE_INFO - terms are scanned, frequency and position info is retrieved + * document numbers are compacted (shifted if segment has deleted documents) + * + * @var integer + */ + private $_termsScanMode; + + /** Scan modes */ + const SM_TERMS_ONLY = 0; // terms are scanned, no additional info is retrieved + const SM_FULL_INFO = 1; // terms are scanned, frequency and position info is retrieved + const SM_MERGE_INFO = 2; // terms are scanned, frequency and position info is retrieved + // document numbers are compacted (shifted if segment contains deleted documents) + /** * Reset terms stream * @@ -904,50 +1143,223 @@ class Zend_Search_Lucene_Index_SegmentInfo * Returns start document id for the next segment * * @param integer $startId - * @param boolean $compact + * @param integer $mode * @throws Zend_Search_Lucene_Exception * @return integer */ - public function reset($startId = 0, $compact = false) + public function reset($startId = 0, $mode = self::SM_TERMS_ONLY) { if ($this->_tisFile !== null) { $this->_tisFile = null; } $this->_tisFile = $this->openCompoundFile('.tis', false); + $this->_tisFileOffset = $this->_tisFile->tell(); + $tiVersion = $this->_tisFile->readInt(); - if ($tiVersion != (int)0xFFFFFFFE) { + if ($tiVersion != (int)0xFFFFFFFE /* pre-2.1 format */ && + $tiVersion != (int)0xFFFFFFFD /* 2.1+ format */) { throw new Zend_Search_Lucene_Exception('Wrong TermInfoFile file format'); } - $this->_termCount = $this->_tisFile->readLong(); - $this->_tisFile->readInt(); // Read Index interval - $this->_skipInterval = $this->_tisFile->readInt(); // Read skip interval + $this->_termCount = + $this->_termNum = $this->_tisFile->readLong(); // Read terms count + $this->_indexInterval = $this->_tisFile->readInt(); // Read Index interval + $this->_skipInterval = $this->_tisFile->readInt(); // Read skip interval + if ($tiVersion == (int)0xFFFFFFFD /* 2.1+ format */) { + $maxSkipLevels = $this->_tisFile->readInt(); + } if ($this->_frqFile !== null) { $this->_frqFile = null; } - $this->_frqFile = $this->openCompoundFile('.frq', false); - $this->_frqFileOffset = $this->_frqFile->tell(); - if ($this->_prxFile !== null) { $this->_prxFile = null; } - $this->_prxFile = $this->openCompoundFile('.prx', false); - $this->_prxFileOffset = $this->_prxFile->tell(); - - $this->_lastTerm = new Zend_Search_Lucene_Index_Term('', -1); - $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo(0, 0, 0, 0); - $this->_docMap = array(); - for ($count = 0; $count < $this->_docCount; $count++) { - if (!$this->isDeleted($count)) { - $this->_docMap[$count] = $startId + ($compact ? count($this->_docMap) : $count); + + $this->_lastTerm = new Zend_Search_Lucene_Index_Term('', -1); + $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo(0, 0, 0, 0); + $this->_lastTermPositions = null; + + $this->_termsScanMode = $mode; + + switch ($mode) { + case self::SM_TERMS_ONLY: + // Do nothing + break; + + case self::SM_FULL_INFO: + // break intentionally omitted + case self::SM_MERGE_INFO: + $this->_frqFile = $this->openCompoundFile('.frq', false); + $this->_frqFileOffset = $this->_frqFile->tell(); + + $this->_prxFile = $this->openCompoundFile('.prx', false); + $this->_prxFileOffset = $this->_prxFile->tell(); + + for ($count = 0; $count < $this->_docCount; $count++) { + if (!$this->isDeleted($count)) { + $this->_docMap[$count] = $startId + (($mode == self::SM_MERGE_INFO) ? count($this->_docMap) : $count); + } + } + break; + + default: + throw new Zend_Search_Lucene_Exception('Wrong terms scaning mode specified.'); + break; + } + + + $this->nextTerm(); + return $startId + (($mode == self::SM_MERGE_INFO) ? count($this->_docMap) : $this->_docCount); + } + + + /** + * Skip terms stream up to specified term preffix. + * + * Prefix contains fully specified field info and portion of searched term + * + * @param Zend_Search_Lucene_Index_Term $prefix + * @throws Zend_Search_Lucene_Exception + */ + public function skipTo(Zend_Search_Lucene_Index_Term $prefix) + { + if ($this->_termDictionary === null) { + $this->_loadDictionaryIndex(); + } + + $searchField = $this->getFieldNum($prefix->field); + + if ($searchField == -1) { + /** + * Field is not presented in this segment + * Go to the end of dictionary + */ + $this->_tisFile = null; + $this->_frqFile = null; + $this->_prxFile = null; + + $this->_lastTerm = null; + $this->_lastTermInfo = null; + $this->_lastTermPositions = null; + + return; + } + $searchDicField = $this->_getFieldPosition($searchField); + + // search for appropriate value in dictionary + $lowIndex = 0; + $highIndex = count($this->_termDictionary)-1; + while ($highIndex >= $lowIndex) { + // $mid = ($highIndex - $lowIndex)/2; + $mid = ($highIndex + $lowIndex) >> 1; + $midTerm = $this->_termDictionary[$mid]; + + $fieldNum = $this->_getFieldPosition($midTerm[0] /* field */); + $delta = $searchDicField - $fieldNum; + if ($delta == 0) { + $delta = strcmp($prefix->text, $midTerm[1] /* text */); + } + + if ($delta < 0) { + $highIndex = $mid-1; + } elseif ($delta > 0) { + $lowIndex = $mid+1; + } else { + // We have reached term we are looking for + break; } } - $this->nextTerm(); - return $startId + ($compact ? count($this->_docMap) : $this->_docCount); + if ($highIndex == -1) { + // Term is out of the dictionary range + $this->_tisFile = null; + $this->_frqFile = null; + $this->_prxFile = null; + + $this->_lastTerm = null; + $this->_lastTermInfo = null; + $this->_lastTermPositions = null; + + return; + } + + $prevPosition = $highIndex; + $prevTerm = $this->_termDictionary[$prevPosition]; + $prevTermInfo = $this->_termDictionaryInfos[$prevPosition]; + + if ($this->_tisFile === null) { + // The end of terms stream is reached and terms dictionary file is closed + // Perform mini-reset operation + $this->_tisFile = $this->openCompoundFile('.tis', false); + + if ($this->_termsScanMode == self::SM_FULL_INFO || $this->_termsScanMode == self::SM_MERGE_INFO) { + $this->_frqFile = $this->openCompoundFile('.frq', false); + $this->_prxFile = $this->openCompoundFile('.prx', false); + } + } + $this->_tisFile->seek($this->_tisFileOffset + $prevTermInfo[4], SEEK_SET); + + $this->_lastTerm = new Zend_Search_Lucene_Index_Term($prevTerm[1] /* text */, + ($prevTerm[0] == -1) ? '' : $this->_fields[$prevTerm[0] /* field */]->name); + $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo($prevTermInfo[0] /* docFreq */, + $prevTermInfo[1] /* freqPointer */, + $prevTermInfo[2] /* proxPointer */, + $prevTermInfo[3] /* skipOffset */); + $this->_termCount = $this->_termNum - $prevPosition*$this->_indexInterval; + + if ($highIndex == 0) { + // skip start entry + $this->nextTerm(); + } else if ($prefix->field == $this->_lastTerm->field && $prefix->text == $this->_lastTerm->text) { + // We got exact match in the dictionary index + + if ($this->_termsScanMode == self::SM_FULL_INFO || $this->_termsScanMode == self::SM_MERGE_INFO) { + $this->_lastTermPositions = array(); + + $this->_frqFile->seek($this->_lastTermInfo->freqPointer + $this->_frqFileOffset, SEEK_SET); + $freqs = array(); $docId = 0; + for( $count = 0; $count < $this->_lastTermInfo->docFreq; $count++ ) { + $docDelta = $this->_frqFile->readVInt(); + if( $docDelta % 2 == 1 ) { + $docId += ($docDelta-1)/2; + $freqs[ $docId ] = 1; + } else { + $docId += $docDelta/2; + $freqs[ $docId ] = $this->_frqFile->readVInt(); + } + } + + $this->_prxFile->seek($this->_lastTermInfo->proxPointer + $this->_prxFileOffset, SEEK_SET); + foreach ($freqs as $docId => $freq) { + $termPosition = 0; $positions = array(); + + for ($count = 0; $count < $freq; $count++ ) { + $termPosition += $this->_prxFile->readVInt(); + $positions[] = $termPosition; + } + + if (isset($this->_docMap[$docId])) { + $this->_lastTermPositions[$this->_docMap[$docId]] = $positions; + } + } + } + + return; + } + + // Search term matching specified prefix + while ($this->_lastTerm !== null) { + if ( strcmp($this->_lastTerm->field, $prefix->field) > 0 || + ($prefix->field == $this->_lastTerm->field && strcmp($this->_lastTerm->text, $prefix->text) >= 0) ) { + // Current term matches or greate than the pattern + return; + } + + $this->nextTerm(); + } } @@ -959,8 +1371,10 @@ class Zend_Search_Lucene_Index_SegmentInfo public function nextTerm() { if ($this->_tisFile === null || $this->_termCount == 0) { - $this->_lastTerm = null; - $this->_lastTermInfo = null; + $this->_lastTerm = null; + $this->_lastTermInfo = null; + $this->_lastTermPositions = null; + $this->_docMap = null; // may be necessary for "empty" segment $this->_tisFile = null; @@ -989,36 +1403,37 @@ class Zend_Search_Lucene_Index_SegmentInfo $this->_lastTermInfo = new Zend_Search_Lucene_Index_TermInfo($docFreq, $freqPointer, $proxPointer, $skipOffset); - $this->_lastTermPositions = array(); + if ($this->_termsScanMode == self::SM_FULL_INFO || $this->_termsScanMode == self::SM_MERGE_INFO) { + $this->_lastTermPositions = array(); - $this->_frqFile->seek($this->_lastTermInfo->freqPointer + $this->_frqFileOffset, SEEK_SET); - $freqs = array(); $docId = 0; - for( $count = 0; $count < $this->_lastTermInfo->docFreq; $count++ ) { - $docDelta = $this->_frqFile->readVInt(); - if( $docDelta % 2 == 1 ) { - $docId += ($docDelta-1)/2; - $freqs[ $docId ] = 1; - } else { - $docId += $docDelta/2; - $freqs[ $docId ] = $this->_frqFile->readVInt(); + $this->_frqFile->seek($this->_lastTermInfo->freqPointer + $this->_frqFileOffset, SEEK_SET); + $freqs = array(); $docId = 0; + for( $count = 0; $count < $this->_lastTermInfo->docFreq; $count++ ) { + $docDelta = $this->_frqFile->readVInt(); + if( $docDelta % 2 == 1 ) { + $docId += ($docDelta-1)/2; + $freqs[ $docId ] = 1; + } else { + $docId += $docDelta/2; + $freqs[ $docId ] = $this->_frqFile->readVInt(); + } + } + + $this->_prxFile->seek($this->_lastTermInfo->proxPointer + $this->_prxFileOffset, SEEK_SET); + foreach ($freqs as $docId => $freq) { + $termPosition = 0; $positions = array(); + + for ($count = 0; $count < $freq; $count++ ) { + $termPosition += $this->_prxFile->readVInt(); + $positions[] = $termPosition; + } + + if (isset($this->_docMap[$docId])) { + $this->_lastTermPositions[$this->_docMap[$docId]] = $positions; + } } } - $this->_prxFile->seek($this->_lastTermInfo->proxPointer + $this->_prxFileOffset, SEEK_SET); - foreach ($freqs as $docId => $freq) { - $termPosition = 0; $positions = array(); - - for ($count = 0; $count < $freq; $count++ ) { - $termPosition += $this->_prxFile->readVInt(); - $positions[] = $termPosition; - } - - if (isset($this->_docMap[$docId])) { - $this->_lastTermPositions[$this->_docMap[$docId]] = $positions; - } - } - - $this->_termCount--; if ($this->_termCount == 0) { $this->_tisFile = null; @@ -1029,11 +1444,28 @@ class Zend_Search_Lucene_Index_SegmentInfo return $this->_lastTerm; } + /** + * Close terms stream + * + * Should be used for resources clean up if stream is not read up to the end + */ + public function closeTermsStream() + { + $this->_tisFile = null; + $this->_frqFile = null; + $this->_prxFile = null; + + $this->_lastTerm = null; + $this->_lastTermInfo = null; + $this->_lastTermPositions = null; + + $this->_docMap = null; + } + /** * Returns term in current position * - * @param Zend_Search_Lucene_Index_Term $term * @return Zend_Search_Lucene_Index_Term|null */ public function currentTerm() diff --git a/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php b/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php index 4d0f346ceb2..233d7987545 100644 --- a/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php +++ b/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/PriorityQueue.php'; +require_once 'Zend/Search/Lucene/PriorityQueue.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_SegmentInfoPriorityQueue extends Zend_Search_Lucene_PriorityQueue diff --git a/search/Zend/Search/Lucene/Index/SegmentMerger.php b/search/Zend/Search/Lucene/Index/SegmentMerger.php index 157489c492e..97ce4bf6d80 100644 --- a/search/Zend/Search/Lucene/Index/SegmentMerger.php +++ b/search/Zend/Search/Lucene/Index/SegmentMerger.php @@ -15,29 +15,29 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; /** Zend_Search_Lucene_Index_SegmentWriter_StreamWriter */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php'; +require_once 'Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php'; /** Zend_Search_Lucene_Index_SegmentInfoPriorityQueue */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php'; +require_once 'Zend/Search/Lucene/Index/SegmentInfoPriorityQueue.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_SegmentMerger @@ -232,7 +232,7 @@ class Zend_Search_Lucene_Index_SegmentMerger $segmentStartId = 0; foreach ($this->_segmentInfos as $segName => $segmentInfo) { - $segmentStartId = $segmentInfo->reset($segmentStartId, true); + $segmentStartId = $segmentInfo->reset($segmentStartId, Zend_Search_Lucene_Index_SegmentInfo::SM_MERGE_INFO); // Skip "empty" segments if ($segmentInfo->currentTerm() !== null) { diff --git a/search/Zend/Search/Lucene/Index/SegmentWriter.php b/search/Zend/Search/Lucene/Index/SegmentWriter.php index 2f1a05e3229..21e9a04d0de 100644 --- a/search/Zend/Search/Lucene/Index/SegmentWriter.php +++ b/search/Zend/Search/Lucene/Index/SegmentWriter.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Index_SegmentWriter @@ -47,19 +47,33 @@ abstract class Zend_Search_Lucene_Index_SegmentWriter */ public static $indexInterval = 128; - /** Expert: The fraction of TermDocs entries stored in skip tables. + /** + * Expert: The fraction of TermDocs entries stored in skip tables. * Larger values result in smaller indexes, greater acceleration, but fewer * accelerable cases, while smaller values result in bigger indexes, * less acceleration and more * accelerable cases. More detailed experiments would be useful here. * - * 0x0x7FFFFFFF indicates that we don't use skip data - * Default value is 16 + * 0x7FFFFFFF indicates that we don't use skip data + * + * Note: not used in current implementation * * @var integer */ public static $skipInterval = 0x7FFFFFFF; + /** + * Expert: The maximum number of skip levels. Smaller values result in + * slightly smaller indexes, but slower skipping in big posting lists. + * + * 0 indicates that we don't use skip data + * + * Note: not used in current implementation + * + * @var integer + */ + public static $maxSkipLevels = 0; + /** * Number of docs in a segment * @@ -245,6 +259,16 @@ abstract class Zend_Search_Lucene_Index_SegmentWriter return $this->_docCount; } + /** + * Return segment name + * + * @return string + */ + public function getName() + { + return $this->_name; + } + /** * Dump Field Info (.fnm) segment file */ @@ -253,6 +277,12 @@ abstract class Zend_Search_Lucene_Index_SegmentWriter $fnmFile = $this->_directory->createFile($this->_name . '.fnm'); $fnmFile->writeVInt(count($this->_fields)); + $nrmFile = $this->_directory->createFile($this->_name . '.nrm'); + // Write header + $nrmFile->writeBytes('NRM'); + // Write format specifier + $nrmFile->writeByte((int)0xFF); + foreach ($this->_fields as $field) { $fnmFile->writeString($field->name); $fnmFile->writeByte(($field->isIndexed ? 0x01 : 0x00) | @@ -262,14 +292,18 @@ abstract class Zend_Search_Lucene_Index_SegmentWriter ); if ($field->isIndexed) { - $normFileName = $this->_name . '.f' . $field->number; - $fFile = $this->_directory->createFile($normFileName); - $fFile->writeBytes($this->_norms[$field->name]); - $this->_files[] = $normFileName; + // pre-2.1 index mode (not used now) + // $normFileName = $this->_name . '.f' . $field->number; + // $fFile = $this->_directory->createFile($normFileName); + // $fFile->writeBytes($this->_norms[$field->name]); + // $this->_files[] = $normFileName; + + $nrmFile->writeBytes($this->_norms[$field->name]); } } $this->_files[] = $this->_name . '.fnm'; + $this->_files[] = $this->_name . '.nrm'; } @@ -351,16 +385,18 @@ abstract class Zend_Search_Lucene_Index_SegmentWriter public function initializeDictionaryFiles() { $this->_tisFile = $this->_directory->createFile($this->_name . '.tis'); - $this->_tisFile->writeInt((int)0xFFFFFFFE); + $this->_tisFile->writeInt((int)0xFFFFFFFD); $this->_tisFile->writeLong(0 /* dummy data for terms count */); $this->_tisFile->writeInt(self::$indexInterval); $this->_tisFile->writeInt(self::$skipInterval); + $this->_tisFile->writeInt(self::$maxSkipLevels); $this->_tiiFile = $this->_directory->createFile($this->_name . '.tii'); - $this->_tiiFile->writeInt((int)0xFFFFFFFE); + $this->_tiiFile->writeInt((int)0xFFFFFFFD); $this->_tiiFile->writeLong(0 /* dummy data for terms count */); $this->_tiiFile->writeInt(self::$indexInterval); $this->_tiiFile->writeInt(self::$skipInterval); + $this->_tiiFile->writeInt(self::$maxSkipLevels); /** Dump dictionary header */ $this->_tiiFile->writeVInt(0); // preffix length @@ -370,7 +406,7 @@ abstract class Zend_Search_Lucene_Index_SegmentWriter $this->_tiiFile->writeVInt(0); // DocFreq $this->_tiiFile->writeVInt(0); // FreqDelta $this->_tiiFile->writeVInt(0); // ProxDelta - $this->_tiiFile->writeVInt(20); // IndexDelta + $this->_tiiFile->writeVInt(24); // IndexDelta $this->_frqFile = $this->_directory->createFile($this->_name . '.frq'); $this->_prxFile = $this->_directory->createFile($this->_name . '.prx'); @@ -384,7 +420,7 @@ abstract class Zend_Search_Lucene_Index_SegmentWriter $this->_prevTermInfo = null; $this->_prevIndexTerm = null; $this->_prevIndexTermInfo = null; - $this->_lastIndexPosition = 20; + $this->_lastIndexPosition = 24; $this->_termCount = 0; } diff --git a/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php b/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php index 7dd2bf91587..56d226512a8 100644 --- a/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php +++ b/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php @@ -15,26 +15,26 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Analysis_Analyzer */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer.php'; /** Zend_Search_Lucene_Index_SegmentWriter */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter.php'; +require_once 'Zend/Search/Lucene/Index/SegmentWriter.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter extends Zend_Search_Lucene_Index_SegmentWriter @@ -204,9 +204,12 @@ class Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter extends Zend_Search_ $this->_generateCFS(); - return new Zend_Search_Lucene_Index_SegmentInfo($this->_name, + return new Zend_Search_Lucene_Index_SegmentInfo($this->_directory, + $this->_name, $this->_docCount, - $this->_directory); + -1, + true, + true); } } diff --git a/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php b/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php index ba0e202522a..318a3a39889 100644 --- a/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php +++ b/search/Zend/Search/Lucene/Index/SegmentWriter/StreamWriter.php @@ -15,26 +15,26 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; /** Zend_Search_Lucene_Index_SegmentWriter */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter.php'; +require_once 'Zend/Search/Lucene/Index/SegmentWriter.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_SegmentWriter_StreamWriter extends Zend_Search_Lucene_Index_SegmentWriter @@ -86,9 +86,12 @@ class Zend_Search_Lucene_Index_SegmentWriter_StreamWriter extends Zend_Search_Lu $this->_dumpFNM(); $this->_generateCFS(); - return new Zend_Search_Lucene_Index_SegmentInfo($this->_name, + return new Zend_Search_Lucene_Index_SegmentInfo($this->_directory, + $this->_name, $this->_docCount, - $this->_directory); + -1, + true, + true); } } diff --git a/search/Zend/Search/Lucene/Index/Term.php b/search/Zend/Search/Lucene/Index/Term.php index 465b4ef41ba..1a5ac62caf9 100644 --- a/search/Zend/Search/Lucene/Index/Term.php +++ b/search/Zend/Search/Lucene/Index/Term.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -31,7 +31,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_Term @@ -74,6 +74,7 @@ class Zend_Search_Lucene_Index_Term /** * Get term prefix * + * @param string $str * @param integer $length * @return string */ @@ -104,5 +105,39 @@ class Zend_Search_Lucene_Index_Term return substr($str, 0, $prefixBytes); } + + /** + * Get UTF-8 string length + * + * @param string $str + * @return string + */ + public static function getLength($str) + { + $bytes = 0; + $chars = 0; + while ($bytes < strlen($str)) { + $charBytes = 1; + if ((ord($str[$bytes]) & 0xC0) == 0xC0) { + $charBytes++; + if (ord($str[$bytes]) & 0x20 ) { + $charBytes++; + if (ord($str[$bytes]) & 0x10 ) { + $charBytes++; + } + } + } + + if ($bytes + $charBytes > strlen($str)) { + // wrong character + break; + } + + $chars++; + $bytes += $charBytes; + } + + return $chars; + } } diff --git a/search/Zend/Search/Lucene/Index/TermInfo.php b/search/Zend/Search/Lucene/Index/TermInfo.php index 95f7cfc65f4..2d46724bd9f 100644 --- a/search/Zend/Search/Lucene/Index/TermInfo.php +++ b/search/Zend/Search/Lucene/Index/TermInfo.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -26,7 +26,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_TermInfo diff --git a/search/Zend/Search/Lucene/Index/Writer.php b/search/Zend/Search/Lucene/Index/Writer.php index 8e32f4e1f6e..c5569d7408f 100644 --- a/search/Zend/Search/Lucene/Index/Writer.php +++ b/search/Zend/Search/Lucene/Index/Writer.php @@ -15,27 +15,33 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ -/** Zend_Search_Lucene_Index_SegmentWriter_ */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php'; +/** Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter */ +require_once 'Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php'; /** Zend_Search_Lucene_Index_SegmentInfo */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentInfo.php'; +require_once 'Zend/Search/Lucene/Index/SegmentInfo.php'; /** Zend_Search_Lucene_Index_SegmentMerger */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/SegmentMerger.php'; +require_once 'Zend/Search/Lucene/Index/SegmentMerger.php'; +/** Zend_Search_Lucene_LockManager */ +if (@$CFG->block_search_softlock){ + require_once "Zend/Search/Lucene/SoftLockManager.php"; +} else { + require_once "Zend/Search/Lucene/LockManager.php"; +} /** * @category Zend * @package Zend_Search_Lucene * @subpackage Index - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Index_Writer @@ -154,33 +160,29 @@ class Zend_Search_Lucene_Index_Writer '.del' => '.del', '.sti' => '.sti' ); + /** - * Opens the index for writing - * - * IndexWriter constructor needs Directory as a parameter. It should be - * a string with a path to the index folder or a Directory object. - * Second constructor parameter create is optional - true to create the - * index or overwrite the existing one. + * Create empty index * * @param Zend_Search_Lucene_Storage_Directory $directory - * @param array $segmentInfos - * @param boolean $create + * @param integer $generation + * @param integer $nameCount */ - public function __construct(Zend_Search_Lucene_Storage_Directory $directory, &$segmentInfos, $create = false) + public static function createIndex(Zend_Search_Lucene_Storage_Directory $directory, $generation, $nameCount) { - $this->_directory = $directory; - $this->_segmentInfos = &$segmentInfos; + if ($generation == 0) { + // Create index in pre-2.1 mode - if ($create) { - foreach ($this->_directory->fileList() as $file) { + foreach ($directory->fileList() as $file) { if ($file == 'deletable' || $file == 'segments' || isset(self::$_indexExtensions[ substr($file, strlen($file)-4)]) || preg_match('/\.f\d+$/i', $file) /* matches .f file names */) { - $this->_directory->deleteFile($file); + $directory->deleteFile($file); } } - $segmentsFile = $this->_directory->createFile('segments'); + + $segmentsFile = $directory->createFile('segments'); $segmentsFile->writeInt((int)0xFFFFFFFF); // write version (is initialized by current time @@ -190,22 +192,55 @@ class Zend_Search_Lucene_Index_Writer $segmentsFile->writeInt((int)($version & 0xFFFFFFFF)); // write name counter - $segmentsFile->writeInt(0); + $segmentsFile->writeInt($nameCount); // write segment counter $segmentsFile->writeInt(0); - $deletableFile = $this->_directory->createFile('deletable'); + $deletableFile = $directory->createFile('deletable'); // write counter $deletableFile->writeInt(0); } else { - $segmentsFile = $this->_directory->getFileObject('segments'); - $format = $segmentsFile->readInt(); - if ($format != (int)0xFFFFFFFF) { - throw new Zend_Search_Lucene_Exception('Wrong segments file format'); - } + $genFile = $directory->createFile('segments.gen'); + + $genFile->writeInt((int)0xFFFFFFFE); + // Write generation two times + $genFile->writeLong($generation); + $genFile->writeLong($generation); + + $segmentsFile = $directory->createFile(Zend_Search_Lucene::getSegmentFileName($generation)); + $segmentsFile->writeInt((int)0xFFFFFFFD); + + // write version (is initialized by current time + // $segmentsFile->writeLong((int)microtime(true)); + $version = microtime(true); + $segmentsFile->writeInt((int)($version/((double)0xFFFFFFFF + 1))); + $segmentsFile->writeInt((int)($version & 0xFFFFFFFF)); + + // write name counter + $segmentsFile->writeInt($nameCount); + // write segment counter + $segmentsFile->writeInt(0); } } + /** + * Open the index for writing + * + * IndexWriter constructor needs Directory as a parameter. It should be + * a string with a path to the index folder or a Directory object. + * Second constructor parameter create is optional - true to create the + * index or overwrite the existing one. + * + * @param Zend_Search_Lucene_Storage_Directory $directory + * @param array $segmentInfos + * @param Zend_Search_Lucene_Storage_File $cleanUpLock + */ + public function __construct(Zend_Search_Lucene_Storage_Directory $directory, &$segmentInfos) + { + $this->_directory = $directory; + $this->_segmentInfos = &$segmentInfos; + } + /** * Adds a document to this index. * @@ -223,27 +258,86 @@ class Zend_Search_Lucene_Index_Writer $this->commit(); } - $this->_versionUpdate++; - $this->_maybeMergeSegments(); + + $this->_versionUpdate++; } /** - * Merge segments if necessary + * Check if we have anything to merge + * + * @return boolean */ - private function _maybeMergeSegments() + private function _hasAnythingToMerge() { $segmentSizes = array(); - foreach ($this->_segmentInfos as $segId => $segmentInfo) { - $segmentSizes[$segId] = $segmentInfo->count(); + foreach ($this->_segmentInfos as $segName => $segmentInfo) { + $segmentSizes[$segName] = $segmentInfo->count(); } $mergePool = array(); $poolSize = 0; $sizeToMerge = $this->maxBufferedDocs; asort($segmentSizes, SORT_NUMERIC); - foreach ($segmentSizes as $segId => $size) { + foreach ($segmentSizes as $segName => $size) { + // Check, if segment comes into a new merging block + while ($size >= $sizeToMerge) { + // Merge previous block if it's large enough + if ($poolSize >= $sizeToMerge) { + return true; + } + $mergePool = array(); + $poolSize = 0; + + $sizeToMerge *= $this->mergeFactor; + + if ($sizeToMerge > $this->maxMergeDocs) { + return false; + } + } + + $mergePool[] = $this->_segmentInfos[$segName]; + $poolSize += $size; + } + + if ($poolSize >= $sizeToMerge) { + return true; + } + + return false; + } + + /** + * Merge segments if necessary + */ + private function _maybeMergeSegments() + { + if (Zend_Search_Lucene_LockManager::obtainOptimizationLock($this->_directory) === false) { + return; + } + + + if (!$this->_hasAnythingToMerge()) { + Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory); + return; + } + + // Update segments list to be sure all segments are not merged yet by other process + $this->_updateSegments(); + + + // Perform standard auto-optimization procedure + $segmentSizes = array(); + foreach ($this->_segmentInfos as $segName => $segmentInfo) { + $segmentSizes[$segName] = $segmentInfo->count(); + } + + $mergePool = array(); + $poolSize = 0; + $sizeToMerge = $this->maxBufferedDocs; + asort($segmentSizes, SORT_NUMERIC); + foreach ($segmentSizes as $segName => $size) { // Check, if segment comes into a new merging block while ($size >= $sizeToMerge) { // Merge previous block if it's large enough @@ -256,17 +350,20 @@ class Zend_Search_Lucene_Index_Writer $sizeToMerge *= $this->mergeFactor; if ($sizeToMerge > $this->maxMergeDocs) { + Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory); return; } } - $mergePool[] = $this->_segmentInfos[$segId]; + $mergePool[] = $this->_segmentInfos[$segName]; $poolSize += $size; } if ($poolSize >= $sizeToMerge) { $this->_mergeSegments($mergePool); } + + Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory); } /** @@ -278,13 +375,6 @@ class Zend_Search_Lucene_Index_Writer */ private function _mergeSegments($segments) { - // Try to get exclusive non-blocking lock to the 'index.optimization.lock' - // Skip optimization if it's performed by other process right now - $optimizationLock = $this->_directory->createFile('index.optimization.lock'); - if (!$optimizationLock->lock(LOCK_EX,true)) { - return; - } - $newName = $this->_newSegmentName(); $merger = new Zend_Search_Lucene_Index_SegmentMerger($this->_directory, $newName); @@ -299,9 +389,6 @@ class Zend_Search_Lucene_Index_Writer } $this->commit(); - - // optimization is finished - $optimizationLock->unlock(); } /** @@ -312,119 +399,299 @@ class Zend_Search_Lucene_Index_Writer private function _updateSegments() { // Get an exclusive index lock - // Wait, until all parallel searchers or indexers won't stop - // and stop all next searchers, while we are updating segments file - $lock = $this->_directory->getFileObject('index.lock'); - if (!$lock->lock(LOCK_EX)) { - throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock'); - } + Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory); + $generation = Zend_Search_Lucene::getActualGeneration($this->_directory); + $segmentsFile = $this->_directory->getFileObject(Zend_Search_Lucene::getSegmentFileName($generation), false); + $newSegmentFile = $this->_directory->createFile(Zend_Search_Lucene::getSegmentFileName(++$generation), false); - // Do not share file handlers to get file updates from other sessions. - $segmentsFile = $this->_directory->getFileObject('segments', false); - $newSegmentFile = $this->_directory->createFile('segments.new', false); - - // Write format marker - $newSegmentFile->writeInt((int)0xFFFFFFFF); - - // Write index version - $segmentsFile->seek(4, SEEK_CUR); - // $version = $segmentsFile->readLong() + $this->_versionUpdate; - // Process version on 32-bit platforms - $versionHigh = $segmentsFile->readInt(); - $versionLow = $segmentsFile->readInt(); - $version = $versionHigh * ((double)0xFFFFFFFF + 1) + - (($versionLow < 0)? (double)0xFFFFFFFF - (-1 - $versionLow) : $versionLow); - $version += $this->_versionUpdate; - $this->_versionUpdate = 0; - $newSegmentFile->writeInt((int)($version/((double)0xFFFFFFFF + 1))); - $newSegmentFile->writeInt((int)($version & 0xFFFFFFFF)); - - // Write segment name counter - $newSegmentFile->writeInt($segmentsFile->readInt()); - - // Get number of segments offset - $numOfSegmentsOffset = $newSegmentFile->tell(); - // Write number of segemnts - $segmentsCount = $segmentsFile->readInt(); - $newSegmentFile->writeInt(0); // Write dummy data (segment counter) - - $segments = array(); - for ($count = 0; $count < $segmentsCount; $count++) { - $segName = $segmentsFile->readString(); - $segSize = $segmentsFile->readInt(); - - if (!in_array($segName, $this->_segmentsToDelete)) { - $newSegmentFile->writeString($segName); - $newSegmentFile->writeInt($segSize); - - $segments[$segName] = $segSize; - } - } - $segmentsFile->close(); - - $segmentsCount = count($segments) + count($this->_newSegments); - - // Remove segments, not listed in $segments (deleted) - // Load segments, not listed in $this->_segmentInfos - foreach ($this->_segmentInfos as $segId => $segInfo) { - if (isset($segments[$segInfo->getName()])) { - // Segment is already included into $this->_segmentInfos - unset($segments[$segInfo->getName()]); + try { + $genFile = $this->_directory->getFileObject('segments.gen', false); + } catch (Zend_Search_Lucene_Exception $e) { + if (strpos($e->getMessage(), 'is not readable') !== false) { + $genFile = $this->_directory->createFile('segments.gen'); } else { - // remove deleted segment from a list - unset($this->_segmentInfos[$segId]); + throw $e; } } - // $segments contains a list of segments to load - // do it later + + $genFile->writeInt((int)0xFFFFFFFE); + // Write generation (first copy) + $genFile->writeLong($generation); - foreach ($this->_newSegments as $segName => $segmentInfo) { - $newSegmentFile->writeString($segName); - $newSegmentFile->writeInt($segmentInfo->count()); + try { + // Write format marker + $newSegmentFile->writeInt((int)0xFFFFFFFD); + + // Skip format identifier + $segmentsFile->seek(4, SEEK_CUR); + // $version = $segmentsFile->readLong() + $this->_versionUpdate; + // Process version on 32-bit platforms + $versionHigh = $segmentsFile->readInt(); + $versionLow = $segmentsFile->readInt(); + $version = $versionHigh * ((double)0xFFFFFFFF + 1) + + (($versionLow < 0)? (double)0xFFFFFFFF - (-1 - $versionLow) : $versionLow); + $version += $this->_versionUpdate; + $this->_versionUpdate = 0; + $newSegmentFile->writeInt((int)($version/((double)0xFFFFFFFF + 1))); + $newSegmentFile->writeInt((int)($version & 0xFFFFFFFF)); + + // Write segment name counter + $newSegmentFile->writeInt($segmentsFile->readInt()); + + // Get number of segments offset + $numOfSegmentsOffset = $newSegmentFile->tell(); + // Write dummy data (segment counter) + $newSegmentFile->writeInt(0); + + // Read number of segemnts + $segmentsCount = $segmentsFile->readInt(); - $this->_segmentInfos[] = $segmentInfo; + $segments = array(); + for ($count = 0; $count < $segmentsCount; $count++) { + $segName = $segmentsFile->readString(); + $segSize = $segmentsFile->readInt(); + + if ($generation == 1 /* retrieved generation is 0 */) { + // pre-2.1 index format + $delGenHigh = 0; + $delGenLow = 0; + $hasSingleNormFile = false; + $numField = (int)0xFFFFFFFF; + $isCompound = 1; + } else { + //$delGen = $segmentsFile->readLong(); + $delGenHigh = $segmentsFile->readInt(); + $delGenLow = $segmentsFile->readInt(); + $hasSingleNormFile = $segmentsFile->readByte(); + $numField = $segmentsFile->readInt(); + + $normGens = array(); + if ($numField != (int)0xFFFFFFFF) { + for ($count1 = 0; $count1 < $numField; $count1++) { + $normGens[] = $segmentsFile->readLong(); + } + } + $isCompound = $segmentsFile->readByte(); + } + + if (!in_array($segName, $this->_segmentsToDelete)) { + // Load segment if necessary + if (!isset($this->_segmentInfos[$segName])) { + $delGen = $delGenHigh * ((double)0xFFFFFFFF + 1) + + (($delGenLow < 0)? (double)0xFFFFFFFF - (-1 - $delGenLow) : $delGenLow); + $this->_segmentInfos[$segName] = + new Zend_Search_Lucene_Index_SegmentInfo($this->_directory, + $segName, + $segSize, + $delGen, + $hasSingleNormFile, + $isCompound); + } else { + // Retrieve actual detetions file generation number + $delGen = $this->_segmentInfos[$segName]->getDelGen(); + + if ($delGen >= 0) { + $delGenHigh = (int)($delGen/((double)0xFFFFFFFF + 1)); + $delGenLow =(int)($delGen & 0xFFFFFFFF); + } else { + $delGenHigh = $delGenLow = (int)0xFFFFFFFF; + } + } + + $newSegmentFile->writeString($segName); + $newSegmentFile->writeInt($segSize); + $newSegmentFile->writeInt($delGenHigh); + $newSegmentFile->writeInt($delGenLow); + $newSegmentFile->writeByte($hasSingleNormFile); + $newSegmentFile->writeInt($numField); + if ($numField != (int)0xFFFFFFFF) { + foreach ($normGens as $normGen) { + $newSegmentFile->writeLong($normGen); + } + } + $newSegmentFile->writeByte($isCompound); + + $segments[$segName] = $segSize; + } + } + $segmentsFile->close(); + + $segmentsCount = count($segments) + count($this->_newSegments); + + foreach ($this->_newSegments as $segName => $segmentInfo) { + $newSegmentFile->writeString($segName); + $newSegmentFile->writeInt($segmentInfo->count()); + + // delete file generation: -1 (there is no delete file yet) + $newSegmentFile->writeInt((int)0xFFFFFFFF);$newSegmentFile->writeInt((int)0xFFFFFFFF); + // HasSingleNormFile + $newSegmentFile->writeByte($segmentInfo->hasSingleNormFile()); + // NumField + $newSegmentFile->writeInt((int)0xFFFFFFFF); + // IsCompoundFile + $newSegmentFile->writeByte($segmentInfo->isCompound()); + + $segments[$segmentInfo->getName()] = $segmentInfo->count(); + $this->_segmentInfos[$segName] = $segmentInfo; + } + $this->_newSegments = array(); + + $newSegmentFile->seek($numOfSegmentsOffset); + $newSegmentFile->writeInt($segmentsCount); // Update segments count + $newSegmentFile->close(); + } catch (Exception $e) { + /** Restore previous index generation */ + $generation--; + $genFile->seek(4, SEEK_SET); + // Write generation number twice + $genFile->writeLong($generation); $genFile->writeLong($generation); + + // Release index write lock + Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); + + // Throw the exception + throw $e; } - $this->_newSegments = array(); - $newSegmentFile->seek($numOfSegmentsOffset); - $newSegmentFile->writeInt($segmentsCount); // Update segments count - $newSegmentFile->close(); - $this->_directory->renameFile('segments.new', 'segments'); + // Write generation (second copy) + $genFile->writeLong($generation); - - // Segments file update is finished - // Switch back to shared lock mode - $lock->lock(LOCK_SH); - - - $fileList = $this->_directory->fileList(); - foreach ($this->_segmentsToDelete as $nameToDelete) { - foreach (self::$_indexExtensions as $ext) { - if ($this->_directory->fileExists($nameToDelete . $ext)) { - $this->_directory->deleteFile($nameToDelete . $ext); + + // Check if another update process is not running now + // If yes, skip clean-up procedure + if (Zend_Search_Lucene_LockManager::escalateReadLock($this->_directory)) { + /** + * Clean-up directory + */ + $filesToDelete = array(); + $filesTypes = array(); + $filesNumbers = array(); + + // list of .del files of currently used segments + // each segment can have several generations of .del files + // only last should not be deleted + $delFiles = array(); + + foreach ($this->_directory->fileList() as $file) { + if ($file == 'deletable') { + // 'deletable' file + $filesToDelete[] = $file; + $filesTypes[] = 0; // delete this file first, since it's not used starting from Lucene v2.1 + $filesNumbers[] = 0; + } else if ($file == 'segments') { + // 'segments' file + + $filesToDelete[] = $file; + $filesTypes[] = 1; // second file to be deleted "zero" version of segments file (Lucene pre-2.1) + $filesNumbers[] = 0; + } else if (preg_match('/^segments_[a-zA-Z0-9]+$/i', $file)) { + // 'segments_xxx' file + // Check if it's not a just created generation file + if ($file != Zend_Search_Lucene::getSegmentFileName($generation)) { + $filesToDelete[] = $file; + $filesTypes[] = 2; // first group of files for deletions + $filesNumbers[] = (int)base_convert(substr($file, 9), 36, 10); // ordered by segment generation numbers + } + } else if (preg_match('/(^_([a-zA-Z0-9]+))\.f\d+$/i', $file, $matches)) { + // one of per segment files ('.f') + // Check if it's not one of the segments in the current segments set + if (!isset($segments[$matches[1]])) { + $filesToDelete[] = $file; + $filesTypes[] = 3; // second group of files for deletions + $filesNumbers[] = (int)base_convert($matches[2], 36, 10); // order by segment number + } + } else if (preg_match('/(^_([a-zA-Z0-9]+))(_([a-zA-Z0-9]+))\.del$/i', $file, $matches)) { + // one of per segment files ('_.del' where is '_') + // Check if it's not one of the segments in the current segments set + if (!isset($segments[$matches[1]])) { + $filesToDelete[] = $file; + $filesTypes[] = 3; // second group of files for deletions + $filesNumbers[] = (int)base_convert($matches[2], 36, 10); // order by segment number + } else { + $segmentNumber = (int)base_convert($matches[2], 36, 10); + $delGeneration = (int)base_convert($matches[4], 36, 10); + if (!isset($delFiles[$segmentNumber])) { + $delFiles[$segmentNumber] = array(); + } + $delFiles[$segmentNumber][$delGeneration] = $file; + } + } else if (isset(self::$_indexExtensions[substr($file, strlen($file)-4)])) { + // one of per segment files ('.') + $segmentName = substr($file, 0, strlen($file) - 4); + // Check if it's not one of the segments in the current segments set + if (!isset($segments[$segmentName]) && + ($this->_currentSegment === null || $this->_currentSegment->getName() != $segmentName)) { + $filesToDelete[] = $file; + $filesTypes[] = 3; // second group of files for deletions + $filesNumbers[] = (int)base_convert(substr($file, 1 /* skip '_' */, strlen($file)-5), 36, 10); // order by segment number + } } } - foreach ($fileList as $file) { - if (substr($file, 0, strlen($nameToDelete) + 2) == ($nameToDelete . '.f') && - ctype_digit( substr($file, strlen($nameToDelete) + 2) )) { - $this->_directory->deleteFile($file); + $maxGenNumber = 0; + // process .del files of currently used segments + foreach ($delFiles as $segmentNumber => $segmentDelFiles) { + ksort($delFiles[$segmentNumber], SORT_NUMERIC); + array_pop($delFiles[$segmentNumber]); // remove last delete file generation from candidates for deleting + + end($delFiles[$segmentNumber]); + $lastGenNumber = key($delFiles[$segmentNumber]); + if ($lastGenNumber > $maxGenNumber) { + $maxGenNumber = $lastGenNumber; + } + } + foreach ($delFiles as $segmentNumber => $segmentDelFiles) { + foreach ($segmentDelFiles as $delGeneration => $file) { + $filesToDelete[] = $file; + $filesTypes[] = 4; // third group of files for deletions + $filesNumbers[] = $segmentNumber*$maxGenNumber + $delGeneration; // order by , pair + } + } + + // Reorder files for deleting + array_multisort($filesTypes, SORT_ASC, SORT_NUMERIC, + $filesNumbers, SORT_ASC, SORT_NUMERIC, + $filesToDelete, SORT_ASC, SORT_STRING); + + foreach ($filesToDelete as $file) { + try { + $this->_directory->deleteFile($file); + } catch (Zend_Search_Lucene_Exception $e) { + if (strpos($e->getMessage(), 'Can\'t delete file') === false) { + // That's not "file is under processing or already deleted" exception + // Pass it through + throw $e; } + } + } + + // Return read lock into the previous state + Zend_Search_Lucene_LockManager::deEscalateReadLock($this->_directory); + } else { + // Only release resources if another index reader is running now + foreach ($this->_segmentsToDelete as $segName) { + foreach (self::$_indexExtensions as $ext) { + $this->_directory->purgeFile($segName . $ext); + } } } - $this->_segmentsToDelete = array(); - // Load segments, created by other process - foreach ($segments as $segName => $segSize) { - // Load new segments - $this->_segmentInfos[] = new Zend_Search_Lucene_Index_SegmentInfo($segName, - $segSize, - $this->_directory); + // Clean-up _segmentsToDelete container + $this->_segmentsToDelete = array(); + + + // Release index write lock + Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); + + // Remove unused segments from segments list + foreach ($this->_segmentInfos as $segName => $segmentInfo) { + if (!isset($segments[$segName])) { + unset($this->_segmentInfos[$segName]); + } } } - /** * Commit current changes */ @@ -438,10 +705,7 @@ class Zend_Search_Lucene_Index_Writer $this->_currentSegment = null; } - if (count($this->_newSegments) != 0 || - count($this->_segmentsToDelete) != 0) { - $this->_updateSegments(); - } + $this->_updateSegments(); } @@ -459,15 +723,24 @@ class Zend_Search_Lucene_Index_Writer } /** - * Merges all segments together into a single segment, optimizing - * an index for search. - * Input is an array of Zend_Search_Lucene_Index_SegmentInfo objects + * Merges all segments together into new one + * + * Returns true on success and false if another optimization or auto-optimization process + * is running now * - * @throws Zend_Search_Lucene_Exception + * @return boolean */ public function optimize() { + if (Zend_Search_Lucene_LockManager::obtainOptimizationLock($this->_directory) === false) { + return false; + } + $this->_mergeSegments($this->_segmentInfos); + + Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory); + + return true; } /** @@ -477,16 +750,10 @@ class Zend_Search_Lucene_Index_Writer */ private function _newSegmentName() { - // Do not share file handler to get file updates from other sessions. - $segmentsFile = $this->_directory->getFileObject('segments', false); - - // Get exclusive segments file lock - // We have guarantee, that we will not intersect with _updateSegments() call - // of other process, because it needs exclusive index lock and waits - // until all other searchers won't stop - if (!$segmentsFile->lock(LOCK_EX)) { - throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock'); - } + Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory); + + $generation = Zend_Search_Lucene::getActualGeneration($this->_directory); + $segmentsFile = $this->_directory->getFileObject(Zend_Search_Lucene::getSegmentFileName($generation), false); $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version) $segmentNameCounter = $segmentsFile->readInt(); @@ -498,8 +765,8 @@ class Zend_Search_Lucene_Index_Writer // return (which calls $segmentsFile destructor) $segmentsFile->flush(); - $segmentsFile->unlock(); - + Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory); + return '_' . base_convert($segmentNameCounter, 10, 36); } diff --git a/search/Zend/Search/Lucene/Interface.php b/search/Zend/Search/Lucene/Interface.php index 58c75b644f3..4e9dea5ea75 100644 --- a/search/Zend/Search/Lucene/Interface.php +++ b/search/Zend/Search/Lucene/Interface.php @@ -14,7 +14,7 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -22,7 +22,7 @@ /** * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ interface Zend_Search_Lucene_Interface @@ -86,6 +86,24 @@ interface Zend_Search_Lucene_Interface */ public static function getDefaultSearchField(); + /** + * Set result set limit. + * + * 0 (default) means no limit + * + * @param integer $limit + */ + public static function setResultSetLimit($limit); + + /** + * Set result set limit. + * + * 0 means no limit + * + * @return integer + */ + public static function getResultSetLimit(); + /** * Retrieve index maxBufferedDocs option * @@ -306,6 +324,43 @@ interface Zend_Search_Lucene_Interface */ public function terms(); + + /** + * Reset terms stream. + */ + public function resetTermsStream(); + + /** + * Skip terms stream up to specified term preffix. + * + * Prefix contains fully specified field info and portion of searched term + * + * @param Zend_Search_Lucene_Index_Term $prefix + */ + public function skipTo(Zend_Search_Lucene_Index_Term $prefix); + + /** + * Scans terms dictionary and returns next term + * + * @return Zend_Search_Lucene_Index_Term|null + */ + public function nextTerm(); + + /** + * Returns term in current position + * + * @return Zend_Search_Lucene_Index_Term|null + */ + public function currentTerm(); + + /** + * Close terms stream + * + * Should be used for resources clean up if stream is not read up to the end + */ + public function closeTermsStream(); + + /** * Undeletes all documents currently marked as deleted in this index. */ diff --git a/search/Zend/Search/Lucene/PriorityQueue.php b/search/Zend/Search/Lucene/PriorityQueue.php index 4e844b18f60..4712e559aca 100644 --- a/search/Zend/Search/Lucene/PriorityQueue.php +++ b/search/Zend/Search/Lucene/PriorityQueue.php @@ -14,7 +14,7 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -31,7 +31,7 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_PriorityQueue diff --git a/search/Zend/Search/Lucene/Proxy.php b/search/Zend/Search/Lucene/Proxy.php index 53fb150bd49..74610877862 100644 --- a/search/Zend/Search/Lucene/Proxy.php +++ b/search/Zend/Search/Lucene/Proxy.php @@ -14,12 +14,12 @@ * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Interface */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Interface.php'; +require_once 'Zend/Search/Lucene/Interface.php'; /** @@ -29,7 +29,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Interface.php'; * * @category Zend * @package Zend_Search_Lucene - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Proxy implements Zend_Search_Lucene_Interface @@ -144,6 +144,30 @@ class Zend_Search_Lucene_Proxy implements Zend_Search_Lucene_Interface return Zend_Search_Lucene::getDefaultSearchField(); } + /** + * Set result set limit. + * + * 0 (default) means no limit + * + * @param integer $limit + */ + public static function setResultSetLimit($limit) + { + Zend_Search_Lucene::setResultSetLimit($limit); + } + + /** + * Set result set limit. + * + * 0 means no limit + * + * @return integer + */ + public static function getResultSetLimit() + { + return Zend_Search_Lucene::getResultSetLimit(); + } + /** * Retrieve index maxBufferedDocs option * @@ -436,6 +460,58 @@ class Zend_Search_Lucene_Proxy implements Zend_Search_Lucene_Interface return $this->_index->terms(); } + + /** + * Reset terms stream. + */ + public function resetTermsStream() + { + $this->_index->resetTermsStream(); + } + + /** + * Skip terms stream up to specified term preffix. + * + * Prefix contains fully specified field info and portion of searched term + * + * @param Zend_Search_Lucene_Index_Term $prefix + */ + public function skipTo(Zend_Search_Lucene_Index_Term $prefix) + { + return $this->_index->skipTo($prefix); + } + + /** + * Scans terms dictionary and returns next term + * + * @return Zend_Search_Lucene_Index_Term|null + */ + public function nextTerm() + { + return $this->_index->nextTerm(); + } + + /** + * Returns term in current position + * + * @return Zend_Search_Lucene_Index_Term|null + */ + public function currentTerm() + { + return $this->_index->currentTerm(); + } + + /** + * Close terms stream + * + * Should be used for resources clean up if stream is not read up to the end + */ + public function closeTermsStream() + { + $this->_index->closeTermsStream(); + } + + /** * Undeletes all documents currently marked as deleted in this index. */ diff --git a/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php b/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php index 1801bc1cf3a..649c222b9b5 100644 --- a/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php +++ b/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php @@ -15,29 +15,29 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_FSM */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php'; +require_once 'Zend/Search/Lucene/FSM.php'; /** Zend_Search_Lucene_Search_QueryToken */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php'; +require_once 'Zend/Search/Lucene/Search/QueryToken.php'; /** Zend_Search_Lucene_Search_QueryParser */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParser.php'; +require_once 'Zend/Search/Lucene/Search/QueryParser.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_BooleanExpressionRecognizer extends Zend_Search_Lucene_FSM diff --git a/search/Zend/Search/Lucene/Search/Query.php b/search/Zend/Search/Lucene/Search/Query.php index e55c22ce013..e230eba268b 100644 --- a/search/Zend/Search/Lucene/Search/Query.php +++ b/search/Zend/Search/Lucene/Search/Query.php @@ -15,19 +15,19 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Document_Html */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Document/Html.php'; +require_once 'Zend/Search/Lucene/Document/Html.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Search_Query diff --git a/search/Zend/Search/Lucene/Search/Query/Boolean.php b/search/Zend/Search/Lucene/Search/Query/Boolean.php index 710a41f381f..3846aa15633 100644 --- a/search/Zend/Search/Lucene/Search/Query/Boolean.php +++ b/search/Zend/Search/Lucene/Search/Query/Boolean.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Query */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; +require_once 'Zend/Search/Lucene/Search/Query.php'; /** Zend_Search_Lucene_Search_Weight_Boolean */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Boolean.php'; +require_once 'Zend/Search/Lucene/Search/Weight/Boolean.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_Query @@ -39,7 +39,7 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ /** * Subqueries - * Array of Zend_Search_Lucene_Query + * Array of Zend_Search_Lucene_Search_Query * * @var array */ @@ -105,7 +105,7 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ /** - * Add a $subquery (Zend_Search_Lucene_Query) to this query. + * Add a $subquery (Zend_Search_Lucene_Search_Query) to this query. * * The sign is specified as: * TRUE - subquery is required @@ -119,6 +119,7 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ public function addSubquery(Zend_Search_Lucene_Search_Query $subquery, $sign=null) { if ($sign !== true || $this->_signs !== null) { // Skip, if all subqueries are required if ($this->_signs === null) { // Check, If all previous subqueries are required + $this->_signs = array(); foreach ($this->_subqueries as $prevSubquery) { $this->_signs[] = true; } @@ -165,6 +166,31 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ $signs[] = ($this->_signs === null)? true : $this->_signs[$id]; } + // Remove insignificant subqueries + foreach ($subqueries as $id => $subquery) { + if ($subquery instanceof Zend_Search_Lucene_Search_Query_Insignificant) { + // Insignificant subquery has to be removed anyway + unset($subqueries[$id]); + unset($signs[$id]); + } + } + if (count($subqueries) == 0) { + // Boolean query doesn't has non-insignificant subqueries + return new Zend_Search_Lucene_Search_Query_Insignificant(); + } + // Check if all non-insignificant subqueries are prohibited + $allProhibited = true; + foreach ($signs as $sign) { + if ($sign !== false) { + $allProhibited = false; + break; + } + } + if ($allProhibited) { + return new Zend_Search_Lucene_Search_Query_Insignificant(); + } + + // Check for empty subqueries foreach ($subqueries as $id => $subquery) { if ($subquery instanceof Zend_Search_Lucene_Search_Query_Empty) { @@ -180,6 +206,10 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ } } + // Check, if reduced subqueries list is empty + if (count($subqueries) == 0) { + return new Zend_Search_Lucene_Search_Query_Empty(); + } // Check if all non-empty subqueries are prohibited $allProhibited = true; @@ -210,12 +240,6 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ } - // Check, if reduced subqueries list is empty - if (count($subqueries) == 0) { - return new Zend_Search_Lucene_Search_Query_Empty(); - } - - // Prepare first candidate for optimized query $optimizedQuery = new Zend_Search_Lucene_Search_Query_Boolean($subqueries, $signs); $optimizedQuery->setBoost($this->getBoost()); @@ -456,11 +480,35 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ $this->_resVector = array(); } - foreach ($this->_subqueries as $subquery) { + $resVectors = array(); + $resVectorsSizes = array(); + $resVectorsIds = array(); // is used to prevent arrays comparison + foreach ($this->_subqueries as $subqueryId => $subquery) { + $resVectors[] = $subquery->matchedDocs(); + $resVectorsSizes[] = count(end($resVectors)); + $resVectorsIds[] = $subqueryId; + } + // sort resvectors in order of subquery cardinality increasing + array_multisort($resVectorsSizes, SORT_ASC, SORT_NUMERIC, + $resVectorsIds, SORT_ASC, SORT_NUMERIC, + $resVectors); + + foreach ($resVectors as $nextResVector) { if($this->_resVector === null) { - $this->_resVector = $subquery->matchedDocs(); + $this->_resVector = $nextResVector; } else { - $this->_resVector = array_intersect_key($this->_resVector, $subquery->matchedDocs()); + //$this->_resVector = array_intersect_key($this->_resVector, $nextResVector); + + /** + * This code is used as workaround for array_intersect_key() slowness problem. + */ + $updatedVector = array(); + foreach ($this->_resVector as $id => $value) { + if (isset($nextResVector[$id])) { + $updatedVector[$id] = $value; + } + } + $this->_resVector = $updatedVector; } if (count($this->_resVector) == 0) { @@ -469,7 +517,8 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ } } - ksort($this->_resVector, SORT_NUMERIC); + // ksort($this->_resVector, SORT_NUMERIC); + // Used algorithm doesn't change elements order } @@ -479,30 +528,61 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ */ private function _calculateNonConjunctionResult() { - $required = null; - $optional = array(); + $requiredVectors = array(); + $requiredVectorsSizes = array(); + $requiredVectorsIds = array(); // is used to prevent arrays comparison + + $optional = array(); foreach ($this->_subqueries as $subqueryId => $subquery) { - $docs = $subquery->matchedDocs(); - if ($this->_signs[$subqueryId] === true) { // required - if ($required !== null) { - // array intersection - $required = array_intersect_key($required, $docs); - } else { - $required = $docs; - } + $requiredVectors[] = $subquery->matchedDocs(); + $requiredVectorsSizes[] = count(end($requiredVectors)); + $requiredVectorsIds[] = $subqueryId; } elseif ($this->_signs[$subqueryId] === false) { // prohibited // Do nothing. matchedDocs() may include non-matching id's + // Calculating prohibited vector may take significant time, but do not affect the result + // Skipped. } else { // neither required, nor prohibited // array union - $optional += $docs; + $optional += $subquery->matchedDocs(); } } + // sort resvectors in order of subquery cardinality increasing + array_multisort($requiredVectorsSizes, SORT_ASC, SORT_NUMERIC, + $requiredVectorsIds, SORT_ASC, SORT_NUMERIC, + $requiredVectors); + + $required = null; + foreach ($requiredVectors as $nextResVector) { + if($required === null) { + $required = $nextResVector; + } else { + //$required = array_intersect_key($required, $nextResVector); + + /** + * This code is used as workaround for array_intersect_key() slowness problem. + */ + $updatedVector = array(); + foreach ($required as $id => $value) { + if (isset($nextResVector[$id])) { + $updatedVector[$id] = $value; + } + } + $required = $updatedVector; + } + + if (count($required) == 0) { + // Empty result set, we don't need to check other terms + break; + } + } + + if ($required !== null) { $this->_resVector = &$required; } else { @@ -705,7 +785,7 @@ class Zend_Search_Lucene_Search_Query_Boolean extends Zend_Search_Lucene_Search_ $query .= '(' . $subquery->__toString() . ')'; if ($subquery->getBoost() != 1) { - $query .= '^' . $subquery->getBoost(); + $query .= '^' . round($subquery->getBoost(), 4); } } diff --git a/search/Zend/Search/Lucene/Search/Query/Empty.php b/search/Zend/Search/Lucene/Search/Query/Empty.php index 738e33232fd..2b67d1bfb25 100644 --- a/search/Zend/Search/Lucene/Search/Query/Empty.php +++ b/search/Zend/Search/Lucene/Search/Query/Empty.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Query */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; +require_once 'Zend/Search/Lucene/Search/Query.php'; /** Zend_Search_Lucene_Search_Weight_Empty */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Empty.php'; +require_once 'Zend/Search/Lucene/Search/Weight/Empty.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_Empty extends Zend_Search_Lucene_Search_Query diff --git a/search/Zend/Search/Lucene/Search/Query/MultiTerm.php b/search/Zend/Search/Lucene/Search/Query/MultiTerm.php index 9258279a003..e3f79acedd1 100644 --- a/search/Zend/Search/Lucene/Search/Query/MultiTerm.php +++ b/search/Zend/Search/Lucene/Search/Query/MultiTerm.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Query */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; +require_once 'Zend/Search/Lucene/Search/Query.php'; /** Zend_Search_Lucene_Search_Weight_MultiTerm */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php'; +require_once 'Zend/Search/Lucene/Search/Weight/MultiTerm.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Search_Query @@ -103,7 +103,6 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc * * @param array $terms Array of Zend_Search_Lucene_Index_Term objects * @param array $signs Array of signs. Sign is boolean|null. - * @return void */ public function __construct($terms = null, $signs = null) { @@ -139,6 +138,7 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc public function addTerm(Zend_Search_Lucene_Index_Term $term, $sign = null) { if ($sign !== true || $this->_signs !== null) { // Skip, if all terms are required if ($this->_signs === null) { // Check, If all previous terms are required + $this->_signs = array(); foreach ($this->_terms as $prevTerm) { $this->_signs[] = true; } @@ -316,22 +316,47 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc $this->_resVector = array(); } - foreach( $this->_terms as $termId=>$term ) { + $resVectors = array(); + $resVectorsSizes = array(); + $resVectorsIds = array(); // is used to prevent arrays comparison + foreach ($this->_terms as $termId => $term) { + $resVectors[] = array_flip($reader->termDocs($term)); + $resVectorsSizes[] = count(end($resVectors)); + $resVectorsIds[] = $termId; + + $this->_termsFreqs[$termId] = $reader->termFreqs($term); + } + // sort resvectors in order of subquery cardinality increasing + array_multisort($resVectorsSizes, SORT_ASC, SORT_NUMERIC, + $resVectorsIds, SORT_ASC, SORT_NUMERIC, + $resVectors); + + foreach ($resVectors as $nextResVector) { if($this->_resVector === null) { - $this->_resVector = array_flip($reader->termDocs($term)); + $this->_resVector = $nextResVector; } else { - $this->_resVector = array_intersect_key($this->_resVector, array_flip($reader->termDocs($term))); + //$this->_resVector = array_intersect_key($this->_resVector, $nextResVector); + + /** + * This code is used as workaround for array_intersect_key() slowness problem. + */ + $updatedVector = array(); + foreach ($this->_resVector as $id => $value) { + if (isset($nextResVector[$id])) { + $updatedVector[$id] = $value; + } + } + $this->_resVector = $updatedVector; } if (count($this->_resVector) == 0) { // Empty result set, we don't need to check other terms break; } - - $this->_termsFreqs[$termId] = $reader->termFreqs($term); } - ksort($this->_resVector, SORT_NUMERIC); + // ksort($this->_resVector, SORT_NUMERIC); + // Docs are returned ordered. Used algorithm doesn't change elements order. } @@ -343,21 +368,21 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc */ private function _calculateNonConjunctionResult(Zend_Search_Lucene_Interface $reader) { - $required = null; + $requiredVectors = array(); + $requiredVectorsSizes = array(); + $requiredVectorsIds = array(); // is used to prevent arrays comparison + $optional = array(); $prohibited = array(); - + foreach ($this->_terms as $termId => $term) { $termDocs = array_flip($reader->termDocs($term)); if ($this->_signs[$termId] === true) { // required - if ($required !== null) { - // array intersection - $required = array_intersect_key($required, $termDocs); - } else { - $required = $termDocs; - } + $requiredVectors[] = $termDocs; + $requiredVectorsSizes[] = count($termDocs); + $requiredVectorsIds[] = $termId; } elseif ($this->_signs[$termId] === false) { // prohibited // array union @@ -371,16 +396,65 @@ class Zend_Search_Lucene_Search_Query_MultiTerm extends Zend_Search_Lucene_Searc $this->_termsFreqs[$termId] = $reader->termFreqs($term); } + // sort resvectors in order of subquery cardinality increasing + array_multisort($requiredVectorsSizes, SORT_ASC, SORT_NUMERIC, + $requiredVectorsIds, SORT_ASC, SORT_NUMERIC, + $requiredVectors); + + $required = null; + foreach ($requiredVectors as $nextResVector) { + if($required === null) { + $required = $nextResVector; + } else { + //$required = array_intersect_key($required, $nextResVector); + + /** + * This code is used as workaround for array_intersect_key() slowness problem. + */ + $updatedVector = array(); + foreach ($required as $id => $value) { + if (isset($nextResVector[$id])) { + $updatedVector[$id] = $value; + } + } + $required = $updatedVector; + } + + if (count($required) == 0) { + // Empty result set, we don't need to check other terms + break; + } + } + if ($required !== null) { - $this->_resVector = (count($prohibited) > 0) ? - array_diff_key($required, $prohibited) : - $required; + $this->_resVector = $required; } else { - $this->_resVector = (count($prohibited) > 0) ? - array_diff_key($optional, $prohibited) : - $optional; + $this->_resVector = $optional; } + if (count($prohibited) != 0) { + // $this->_resVector = array_diff_key($this->_resVector, $prohibited); + + /** + * This code is used as workaround for array_diff_key() slowness problem. + */ + if (count($this->_resVector) < count($prohibited)) { + $updatedVector = $this->_resVector; + foreach ($this->_resVector as $id => $value) { + if (isset($prohibited[$id])) { + unset($updatedVector[$id]); + } + } + $this->_resVector = $updatedVector; + } else { + $updatedVector = $this->_resVector; + foreach ($prohibited as $id => $value) { + unset($updatedVector[$id]); + } + $this->_resVector = $updatedVector; + } + } + ksort($this->_resVector, SORT_NUMERIC); } diff --git a/search/Zend/Search/Lucene/Search/Query/Phrase.php b/search/Zend/Search/Lucene/Search/Query/Phrase.php index 3e7b782792f..d275fe37f1f 100644 --- a/search/Zend/Search/Lucene/Search/Query/Phrase.php +++ b/search/Zend/Search/Lucene/Search/Query/Phrase.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -23,12 +23,12 @@ /** * Zend_Search_Lucene_Search_Query */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; +require_once 'Zend/Search/Lucene/Search/Query.php'; /** * Zend_Search_Lucene_Search_Weight_MultiTerm */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Phrase.php'; +require_once 'Zend/Search/Lucene/Search/Weight/Phrase.php'; /** @@ -37,7 +37,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Phrase.php' * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Query @@ -416,23 +416,48 @@ class Zend_Search_Lucene_Search_Query_Phrase extends Zend_Search_Lucene_Search_Q $this->_resVector = array(); } - foreach( $this->_terms as $termId=>$term ) { + $resVectors = array(); + $resVectorsSizes = array(); + $resVectorsIds = array(); // is used to prevent arrays comparison + foreach ($this->_terms as $termId => $term) { + $resVectors[] = array_flip($reader->termDocs($term)); + $resVectorsSizes[] = count(end($resVectors)); + $resVectorsIds[] = $termId; + + $this->_termsPositions[$termId] = $reader->termPositions($term); + } + // sort resvectors in order of subquery cardinality increasing + array_multisort($resVectorsSizes, SORT_ASC, SORT_NUMERIC, + $resVectorsIds, SORT_ASC, SORT_NUMERIC, + $resVectors); + + foreach ($resVectors as $nextResVector) { if($this->_resVector === null) { - $this->_resVector = array_flip($reader->termDocs($term)); + $this->_resVector = $nextResVector; } else { - $this->_resVector = array_intersect_key($this->_resVector, array_flip($reader->termDocs($term))); + //$this->_resVector = array_intersect_key($this->_resVector, $nextResVector); + + /** + * This code is used as workaround for array_intersect_key() slowness problem. + */ + $updatedVector = array(); + foreach ($this->_resVector as $id => $value) { + if (isset($nextResVector[$id])) { + $updatedVector[$id] = $value; + } + } + $this->_resVector = $updatedVector; } if (count($this->_resVector) == 0) { // Empty result set, we don't need to check other terms break; } - - $this->_termsPositions[$termId] = $reader->termPositions($term); } - ksort($this->_resVector, SORT_NUMERIC); - + // ksort($this->_resVector, SORT_NUMERIC); + // Docs are returned ordered. Used algorithm doesn't change elements order. + // Initialize weight if it's not done yet $this->_initWeight($reader); } diff --git a/search/Zend/Search/Lucene/Search/Query/Term.php b/search/Zend/Search/Lucene/Search/Query/Term.php index 0240104e325..f9aa071b8ec 100644 --- a/search/Zend/Search/Lucene/Search/Query/Term.php +++ b/search/Zend/Search/Lucene/Search/Query/Term.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Query */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query.php"; /** Zend_Search_Lucene_Search_Weight_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight/Term.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Weight/Term.php"; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Query @@ -65,7 +65,7 @@ class Zend_Search_Lucene_Search_Query_Term extends Zend_Search_Lucene_Search_Que * @param Zend_Search_Lucene_Index_Term $term * @param boolean $sign */ - public function __construct($term) + public function __construct(Zend_Search_Lucene_Index_Term $term) { $this->_term = $term; } diff --git a/search/Zend/Search/Lucene/Search/QueryEntry.php b/search/Zend/Search/Lucene/Search/QueryEntry.php index 53777c8570f..b5ea48317e9 100644 --- a/search/Zend/Search/Lucene/Search/QueryEntry.php +++ b/search/Zend/Search/Lucene/Search/QueryEntry.php @@ -15,36 +15,36 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Index_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; +require_once 'Zend/Search/Lucene/Index/Term.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Search_QueryEntry_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Term.php'; +require_once 'Zend/Search/Lucene/Search/QueryEntry/Term.php'; /** Zend_Search_Lucene_Search_QueryEntry_Phrase */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php'; +require_once 'Zend/Search/Lucene/Search/QueryEntry/Phrase.php'; /** Zend_Search_Lucene_Search_QueryEntry_Subquery */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php'; +require_once 'Zend/Search/Lucene/Search/QueryEntry/Subquery.php'; /** Zend_Search_Lucene_Search_QueryParserException */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; +require_once 'Zend/Search/Lucene/Search/QueryParserException.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Search_QueryEntry diff --git a/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php b/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php index e90a58b4977..0f0090ab4f4 100644 --- a/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php +++ b/search/Zend/Search/Lucene/Search/QueryEntry/Phrase.php @@ -15,25 +15,25 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Index_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; +require_once 'Zend/Search/Lucene/Index/Term.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Search_QueryEntry */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; +require_once 'Zend/Search/Lucene/Search/QueryEntry.php'; /** Zend_Search_Lucene_Search_QueryParserException */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; +require_once 'Zend/Search/Lucene/Search/QueryParserException.php'; /** Zend_Search_Lucene_Analysis_Analyzer */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer.php'; @@ -41,7 +41,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryEntry_Phrase extends Zend_Search_Lucene_Search_QueryEntry @@ -118,7 +118,7 @@ class Zend_Search_Lucene_Search_QueryEntry_Phrase extends Zend_Search_Lucene_Sea $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_phrase, $encoding); if (count($tokens) == 0) { - return new Zend_Search_Lucene_Search_Query_Empty(); + return new Zend_Search_Lucene_Search_Query_Insignificant(); } if (count($tokens) == 1) { @@ -130,10 +130,12 @@ class Zend_Search_Lucene_Search_QueryEntry_Phrase extends Zend_Search_Lucene_Sea } //It's not empty or one term query + $position = -1; $query = new Zend_Search_Lucene_Search_Query_Phrase(); foreach ($tokens as $token) { + $position += $token->getPositionIncrement(); $term = new Zend_Search_Lucene_Index_Term($token->getTermText(), $this->_field); - $query->addTerm($term); + $query->addTerm($term, $position); } if ($this->_proximityQuery) { diff --git a/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php b/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php index 8d6e64fc63d..de15ba81df2 100644 --- a/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php +++ b/search/Zend/Search/Lucene/Search/QueryEntry/Subquery.php @@ -15,29 +15,29 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Index_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; +require_once 'Zend/Search/Lucene/Index/Term.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Search_QueryEntry */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; +require_once 'Zend/Search/Lucene/Search/QueryEntry.php'; /** Zend_Search_Lucene_Search_QueryParserException */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; +require_once 'Zend/Search/Lucene/Search/QueryParserException.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryEntry_Subquery extends Zend_Search_Lucene_Search_QueryEntry diff --git a/search/Zend/Search/Lucene/Search/QueryEntry/Term.php b/search/Zend/Search/Lucene/Search/QueryEntry/Term.php index f9b3d9746eb..9b81e84dcfd 100644 --- a/search/Zend/Search/Lucene/Search/QueryEntry/Term.php +++ b/search/Zend/Search/Lucene/Search/QueryEntry/Term.php @@ -15,25 +15,25 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Index_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; +require_once 'Zend/Search/Lucene/Index/Term.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Search_QueryEntry */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; +require_once 'Zend/Search/Lucene/Search/QueryEntry.php'; /** Zend_Search_Lucene_Search_QueryParserException */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; +require_once 'Zend/Search/Lucene/Search/QueryParserException.php'; /** Zend_Search_Lucene_Analysis_Analyzer */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; +require_once 'Zend/Search/Lucene/Analysis/Analyzer.php'; @@ -41,7 +41,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Analysis/Analyzer.php'; * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryEntry_Term extends Zend_Search_Lucene_Search_QueryEntry @@ -100,7 +100,7 @@ class Zend_Search_Lucene_Search_QueryEntry_Term extends Zend_Search_Lucene_Searc if ($parameter !== null) { $this->_similarity = $parameter; } else { - $this->_similarity = 0.5; + $this->_similarity = Zend_Search_Lucene_Search_Query_Fuzzy::DEFAULT_MIN_SIMILARITY; } } @@ -113,28 +113,77 @@ class Zend_Search_Lucene_Search_QueryEntry_Term extends Zend_Search_Lucene_Searc */ public function getQuery($encoding) { - if ($this->_fuzzyQuery) { - throw new Zend_Search_Lucene_Search_QueryParserException('Fuzzy search is not supported yet.'); - } - if (strpos($this->_term, '?') !== false || strpos($this->_term, '*') !== false) { - throw new Zend_Search_Lucene_Search_QueryParserException('Wildcard queries are not supported yet.'); + if ($this->_fuzzyQuery) { + throw new Zend_Search_Lucene_Search_QueryParserException('Fuzzy search is not supported for terms with wildcards.'); + } + + $pattern = ''; + + $subPatterns = explode('*', $this->_term); + + $astericFirstPass = true; + foreach ($subPatterns as $subPattern) { + if (!$astericFirstPass) { + $pattern .= '*'; + } else { + $astericFirstPass = false; + } + + $subPatternsL2 = explode('?', $subPattern); + + $qMarkFirstPass = true; + foreach ($subPatternsL2 as $subPatternL2) { + if (!$qMarkFirstPass) { + $pattern .= '?'; + } else { + $qMarkFirstPass = false; + } + + $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($subPatternL2, $encoding); + if (count($tokens) > 1) { + throw new Zend_Search_Lucene_Search_QueryParserException('Wildcard search is supported only for non-multiple word terms'); + } + + foreach ($tokens as $token) { + $pattern .= $token->getTermText(); + } + } + } + + $term = new Zend_Search_Lucene_Index_Term($pattern, $this->_field); + $query = new Zend_Search_Lucene_Search_Query_Wildcard($term); + $query->setBoost($this->_boost); + + return $query; } $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_term, $encoding); if (count($tokens) == 0) { - return new Zend_Search_Lucene_Search_Query_Empty(); + return new Zend_Search_Lucene_Search_Query_Insignificant(); } - if (count($tokens) == 1) { - $term = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field); + if (count($tokens) == 1 && !$this->_fuzzyQuery) { + $term = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field); $query = new Zend_Search_Lucene_Search_Query_Term($term); $query->setBoost($this->_boost); return $query; } + if (count($tokens) == 1 && $this->_fuzzyQuery) { + $term = new Zend_Search_Lucene_Index_Term($tokens[0]->getTermText(), $this->_field); + $query = new Zend_Search_Lucene_Search_Query_Fuzzy($term, $this->_similarity); + $query->setBoost($this->_boost); + + return $query; + } + + if ($this->_fuzzyQuery) { + throw new Zend_Search_Lucene_Search_QueryParserException('Fuzzy search is supported only for non-multiple word terms'); + } + //It's not empty or one term query $query = new Zend_Search_Lucene_Search_Query_MultiTerm(); diff --git a/search/Zend/Search/Lucene/Search/QueryHit.php b/search/Zend/Search/Lucene/Search/QueryHit.php index 12278ea4ad8..2a281979b73 100644 --- a/search/Zend/Search/Lucene/Search/QueryHit.php +++ b/search/Zend/Search/Lucene/Search/QueryHit.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,7 +24,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryHit diff --git a/search/Zend/Search/Lucene/Search/QueryLexer.php b/search/Zend/Search/Lucene/Search/QueryLexer.php index 6b72110bfd1..206e7a66b77 100644 --- a/search/Zend/Search/Lucene/Search/QueryLexer.php +++ b/search/Zend/Search/Lucene/Search/QueryLexer.php @@ -15,29 +15,29 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_FSM */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php'; +require_once 'Zend/Search/Lucene/FSM.php'; /** Zend_Search_Lucene_Search_QueryParser */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php'; +require_once 'Zend/Search/Lucene/Search/QueryToken.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Search_QueryParserException */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; +require_once 'Zend/Search/Lucene/Search/QueryParserException.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryLexer extends Zend_Search_Lucene_FSM diff --git a/search/Zend/Search/Lucene/Search/QueryParser.php b/search/Zend/Search/Lucene/Search/QueryParser.php index 1a3d5712de8..b1092a5af65 100644 --- a/search/Zend/Search/Lucene/Search/QueryParser.php +++ b/search/Zend/Search/Lucene/Search/QueryParser.php @@ -15,52 +15,64 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Index_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Index/Term.php"; /** Zend_Search_Lucene_Search_Query_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Term.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Term.php"; /** Zend_Search_Lucene_Search_Query_MultiTerm */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/MultiTerm.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/MultiTerm.php"; /** Zend_Search_Lucene_Search_Query_Boolean */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Boolean.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Boolean.php"; /** Zend_Search_Lucene_Search_Query_Phrase */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Phrase.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Phrase.php"; + +/** Zend_Search_Lucene_Search_Query_Wildcard */ +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Wildcard.php"; + +/** Zend_Search_Lucene_Search_Query_Range */ +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Range.php"; + +/** Zend_Search_Lucene_Search_Query_Fuzzy */ +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Fuzzy.php"; /** Zend_Search_Lucene_Search_Query_Empty */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Empty.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Empty.php"; + +/** Zend_Search_Lucene_Search_Query_Insignificant */ +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/Query/Insignificant.php"; /** Zend_Search_Lucene_Search_QueryLexer */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryLexer.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/QueryLexer.php"; /** Zend_Search_Lucene_Search_QueryParserContext */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserContext.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/QueryParserContext.php"; /** Zend_Search_Lucene_FSM */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/FSM.php"; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Exception.php"; /** Zend_Search_Lucene_Search_QueryParserException */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; +require_once "{$CFG->dirroot}/search/Zend/Search/Lucene/Search/QueryParserException.php"; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM @@ -139,6 +151,26 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ private $_defaultEncoding = ''; + /** + * Defines query parsing mode. + * + * If this option is turned on, then query parser suppress query parser exceptions + * and constructs multi-term query using all words from a query. + * + * That helps to avoid exceptions caused by queries, which don't conform to query language, + * but limits possibilities to check, that query entered by user has some inconsistencies. + * + * + * Default is true. + * + * Use {@link Zend_Search_Lucene::suppressQueryParsingExceptions()}, + * {@link Zend_Search_Lucene::dontSuppressQueryParsingExceptions()} and + * {@link Zend_Search_Lucene::checkQueryParsingExceptionsSuppressMode()} to operate + * with this setting. + * + * @var boolean + */ + private $_suppressQueryParsingExceptions = true; /** * Boolean operators constants @@ -256,6 +288,18 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM $this->_lexer = new Zend_Search_Lucene_Search_QueryLexer(); } + /** + * Get query parser instance + * + * @return Zend_Search_Lucene_Search_QueryParser + */ + private static function _getInstance() + { + if (self::$_instance === null) { + self::$_instance = new self(); + } + return self::$_instance; + } /** * Set query string default encoding @@ -264,11 +308,7 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ public static function setDefaultEncoding($encoding) { - if (self::$_instance === null) { - self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); - } - - self::$_instance->_defaultEncoding = $encoding; + self::_getInstance()->_defaultEncoding = $encoding; } /** @@ -278,11 +318,7 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ public static function getDefaultEncoding() { - if (self::$_instance === null) { - self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); - } - - return self::$_instance->_defaultEncoding; + return self::_getInstance()->_defaultEncoding; } /** @@ -292,11 +328,7 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ public static function setDefaultOperator($operator) { - if (self::$_instance === null) { - self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); - } - - self::$_instance->_defaultOperator = $operator; + self::_getInstance()->_defaultOperator = $operator; } /** @@ -306,13 +338,34 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ public static function getDefaultOperator() { - if (self::$_instance === null) { - self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); - } - - return self::$_instance->_defaultOperator; + return self::_getInstance()->_defaultOperator; } + /** + * Turn on 'suppress query parser exceptions' mode. + */ + public static function suppressQueryParsingExceptions() + { + self::_getInstance()->_suppressQueryParsingExceptions = true; + } + /** + * Turn off 'suppress query parser exceptions' mode. + */ + public static function dontSuppressQueryParsingExceptions() + { + self::_getInstance()->_suppressQueryParsingExceptions = false; + } + /** + * Check 'suppress query parser exceptions' mode. + * @return boolean + */ + public static function queryParsingExceptionsSuppressed() + { + return self::_getInstance()->_suppressQueryParsingExceptions; + } + + + /** * Parses a query string * @@ -323,42 +376,62 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ public static function parse($strQuery, $encoding = null) { - if (self::$_instance === null) { - self::$_instance = new Zend_Search_Lucene_Search_QueryParser(); - } - - self::$_instance->_encoding = ($encoding !== null) ? $encoding : self::$_instance->_defaultEncoding; - self::$_instance->_lastToken = null; - self::$_instance->_context = new Zend_Search_Lucene_Search_QueryParserContext(self::$_instance->_encoding); - self::$_instance->_contextStack = array(); - self::$_instance->_tokens = self::$_instance->_lexer->tokenize($strQuery, self::$_instance->_encoding); - - // Empty query - if (count(self::$_instance->_tokens) == 0) { - return new Zend_Search_Lucene_Search_Query_Empty(); - } - - - foreach (self::$_instance->_tokens as $token) { - try { - self::$_instance->_currentToken = $token; - self::$_instance->process($token->type); - - self::$_instance->_lastToken = $token; - } catch (Exception $e) { - if (strpos($e->getMessage(), 'There is no any rule for') !== false) { - throw new Zend_Search_Lucene_Search_QueryParserException( 'Syntax error at char position ' . $token->position . '.' ); + self::_getInstance(); + + // Reset FSM if previous parse operation didn't return it into a correct state + self::$_instance->reset(); + + try { + self::$_instance->_encoding = ($encoding !== null) ? $encoding : self::$_instance->_defaultEncoding; + self::$_instance->_lastToken = null; + self::$_instance->_context = new Zend_Search_Lucene_Search_QueryParserContext(self::$_instance->_encoding); + self::$_instance->_contextStack = array(); + self::$_instance->_tokens = self::$_instance->_lexer->tokenize($strQuery, self::$_instance->_encoding); + + // Empty query + if (count(self::$_instance->_tokens) == 0) { + return new Zend_Search_Lucene_Search_Query_Insignificant(); + } + + + foreach (self::$_instance->_tokens as $token) { + try { + self::$_instance->_currentToken = $token; + self::$_instance->process($token->type); + + self::$_instance->_lastToken = $token; + } catch (Exception $e) { + if (strpos($e->getMessage(), 'There is no any rule for') !== false) { + throw new Zend_Search_Lucene_Search_QueryParserException( 'Syntax error at char position ' . $token->position . '.' ); + } + + throw $e; } + } + + if (count(self::$_instance->_contextStack) != 0) { + throw new Zend_Search_Lucene_Search_QueryParserException('Syntax Error: mismatched parentheses, every opening must have closing.' ); + } + + return self::$_instance->_context->getQuery(); + } catch (Zend_Search_Lucene_Search_QueryParserException $e) { + if (self::$_instance->_suppressQueryParsingExceptions) { + $queryTokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($strQuery, self::$_instance->_encoding); + $query = new Zend_Search_Lucene_Search_Query_MultiTerm(); + $termsSign = (self::$_instance->_defaultOperator == self::B_AND) ? true /* required term */ : + null /* optional term */; + + foreach ($queryTokens as $token) { + $query->addTerm(new Zend_Search_Lucene_Index_Term($token->getTermText()), $termsSign); + } + + + return $query; + } else { throw $e; } } - - if (count(self::$_instance->_contextStack) != 0) { - throw new Zend_Search_Lucene_Search_QueryParserException('Syntax Error: mismatched parentheses, every opening must have closing.' ); - } - - return self::$_instance->_context->getQuery(); } @@ -433,7 +506,7 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM default: // It's not a user input exception - throw new Zend_Search_Lucene_Exception('Lexeme modifier parameter must follow lexeme modifier. Char position .' ); + throw new Zend_Search_Lucene_Exception('Lexeme modifier parameter must follow lexeme modifier. Char position 0.' ); } } @@ -485,13 +558,31 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ public function openedRQLastTerm() { - throw new Zend_Search_Lucene_Search_QueryParserException('Range queries are not supported yet.'); + $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_rqFirstTerm, $this->_encoding); + if (count($tokens) > 1) { + throw new Zend_Search_Lucene_Search_QueryParserException('Range query boundary terms must be non-multiple word terms'); + } else if (count($tokens) == 1) { + $from = new Zend_Search_Lucene_Index_Term(reset($tokens)->getTermText(), $this->_context->getField()); + } else { + $from = null; + } - // $firstTerm = new Zend_Search_Lucene_Index_Term($this->_rqFirstTerm, $this->_context->getField()); - // $lastTerm = new Zend_Search_Lucene_Index_Term($this->_currentToken->text, $this->_context->getField()); + $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_currentToken->text, $this->_encoding); + if (count($tokens) > 1) { + throw new Zend_Search_Lucene_Search_QueryParserException('Range query boundary terms must be non-multiple word terms'); + } else if (count($tokens) == 1) { + $to = new Zend_Search_Lucene_Index_Term(reset($tokens)->getTermText(), $this->_context->getField()); + } else { + $to = null; + } - // $query = new Zend_Search_Lucene_Search_Query_Range($firstTerm, $lastTerm, false); - // $this->_context->addentry($query); + if ($from === null && $to === null) { + throw new Zend_Search_Lucene_Search_QueryParserException('At least one range query boundary term must be non-empty term'); + } + + $rangeQuery = new Zend_Search_Lucene_Search_Query_Range($from, $to, false); + $entry = new Zend_Search_Lucene_Search_QueryEntry_Subquery($rangeQuery); + $this->_context->addEntry($entry); } /** @@ -509,13 +600,31 @@ class Zend_Search_Lucene_Search_QueryParser extends Zend_Search_Lucene_FSM */ public function closedRQLastTerm() { - throw new Zend_Search_Lucene_Search_QueryParserException('Range queries are not supported yet.'); + $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_rqFirstTerm, $this->_encoding); + if (count($tokens) > 1) { + throw new Zend_Search_Lucene_Search_QueryParserException('Range query boundary terms must be non-multiple word terms'); + } else if (count($tokens) == 1) { + $from = new Zend_Search_Lucene_Index_Term(reset($tokens)->getTermText(), $this->_context->getField()); + } else { + $from = null; + } - // $firstTerm = new Zend_Search_Lucene_Index_Term($this->_rqFirstTerm, $this->_context->getField()); - // $lastTerm = new Zend_Search_Lucene_Index_Term($this->_currentToken->text, $this->_context->getField()); + $tokens = Zend_Search_Lucene_Analysis_Analyzer::getDefault()->tokenize($this->_currentToken->text, $this->_encoding); + if (count($tokens) > 1) { + throw new Zend_Search_Lucene_Search_QueryParserException('Range query boundary terms must be non-multiple word terms'); + } else if (count($tokens) == 1) { + $to = new Zend_Search_Lucene_Index_Term(reset($tokens)->getTermText(), $this->_context->getField()); + } else { + $to = null; + } - // $query = new Zend_Search_Lucene_Search_Query_Range($firstTerm, $lastTerm, true); - // $this->_context->addentry($query); + if ($from === null && $to === null) { + throw new Zend_Search_Lucene_Search_QueryParserException('At least one range query boundary term must be non-empty term'); + } + + $rangeQuery = new Zend_Search_Lucene_Search_Query_Range($from, $to, true); + $entry = new Zend_Search_Lucene_Search_QueryEntry_Subquery($rangeQuery); + $this->_context->addEntry($entry); } } diff --git a/search/Zend/Search/Lucene/Search/QueryParserContext.php b/search/Zend/Search/Lucene/Search/QueryParserContext.php index 9d172c69fa5..f63914ee2e8 100644 --- a/search/Zend/Search/Lucene/Search/QueryParserContext.php +++ b/search/Zend/Search/Lucene/Search/QueryParserContext.php @@ -15,50 +15,50 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_FSM */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/FSM.php'; +require_once 'Zend/Search/Lucene/FSM.php'; /** Zend_Search_Lucene_Index_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Index/Term.php'; +require_once 'Zend/Search/Lucene/Index/Term.php'; /** Zend_Search_Lucene_Search_QueryToken */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryToken.php'; +require_once 'Zend/Search/Lucene/Search/QueryToken.php'; /** Zend_Search_Lucene_Search_Query_Term */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Term.php'; +require_once 'Zend/Search/Lucene/Search/Query/Term.php'; /** Zend_Search_Lucene_Search_Query_MultiTerm */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/MultiTerm.php'; +require_once 'Zend/Search/Lucene/Search/Query/MultiTerm.php'; /** Zend_Search_Lucene_Search_Query_Boolean */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Boolean.php'; +require_once 'Zend/Search/Lucene/Search/Query/Boolean.php'; /** Zend_Search_Lucene_Search_Query_Phrase */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Query/Phrase.php'; +require_once 'Zend/Search/Lucene/Search/Query/Phrase.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** Zend_Search_Lucene_Search_QueryParserException */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryParserException.php'; +require_once 'Zend/Search/Lucene/Search/QueryParserException.php'; /** Zend_Search_Lucene_Search_BooleanExpressionRecognizer */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php'; +require_once 'Zend/Search/Lucene/Search/BooleanExpressionRecognizer.php'; /** Zend_Search_Lucene_Search_QueryEntry */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/QueryEntry.php'; +require_once 'Zend/Search/Lucene/Search/QueryEntry.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryParserContext @@ -360,7 +360,7 @@ class Zend_Search_Lucene_Search_QueryParserContext } if (!$nonNegativeEntryFound) { - unset($conjuctions[$conjuctionId]); + unset($conjuctions[$conjuctionId]); } } @@ -382,7 +382,7 @@ class Zend_Search_Lucene_Search_QueryParserContext } if (count($subqueries) == 0) { - return new Zend_Search_Lucene_Search_Query_Empty(); + return new Zend_Search_Lucene_Search_Query_Insignificant(); } if (count($subqueries) == 1) { diff --git a/search/Zend/Search/Lucene/Search/QueryParserException.php b/search/Zend/Search/Lucene/Search/QueryParserException.php index 8ca791ffb7e..52f6fb7bf58 100644 --- a/search/Zend/Search/Lucene/Search/QueryParserException.php +++ b/search/Zend/Search/Lucene/Search/QueryParserException.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -23,14 +23,14 @@ /** * Zend_Search_Lucene base exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License * * Special exception type, which may be used to intercept wrong user input diff --git a/search/Zend/Search/Lucene/Search/QueryToken.php b/search/Zend/Search/Lucene/Search/QueryToken.php index cf153096d2a..f8cbdcfbe01 100644 --- a/search/Zend/Search/Lucene/Search/QueryToken.php +++ b/search/Zend/Search/Lucene/Search/QueryToken.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_QueryToken diff --git a/search/Zend/Search/Lucene/Search/Similarity.php b/search/Zend/Search/Lucene/Search/Similarity.php index 016d232a244..95b05656560 100644 --- a/search/Zend/Search/Lucene/Search/Similarity.php +++ b/search/Zend/Search/Lucene/Search/Similarity.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Similarity_Default */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity/Default.php'; +require_once 'Zend/Search/Lucene/Search/Similarity/Default.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Search_Similarity diff --git a/search/Zend/Search/Lucene/Search/Similarity/Default.php b/search/Zend/Search/Lucene/Search/Similarity/Default.php index 8263f2a72f8..7b8d6ef7c75 100644 --- a/search/Zend/Search/Lucene/Search/Similarity/Default.php +++ b/search/Zend/Search/Lucene/Search/Similarity/Default.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Similarity */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Similarity.php'; +require_once 'Zend/Search/Lucene/Search/Similarity.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Similarity_Default extends Zend_Search_Lucene_Search_Similarity @@ -38,7 +38,7 @@ class Zend_Search_Lucene_Search_Similarity_Default extends Zend_Search_Lucene_Se * Implemented as '1/sqrt(numTerms)'. * * @param string $fieldName - * @param integer numTerms + * @param integer $numTerms * @return float */ public function lengthNorm($fieldName, $numTerms) diff --git a/search/Zend/Search/Lucene/Search/Weight.php b/search/Zend/Search/Lucene/Search/Weight.php index 2faba2861bf..285fed8e758 100644 --- a/search/Zend/Search/Lucene/Search/Weight.php +++ b/search/Zend/Search/Lucene/Search/Weight.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -32,7 +32,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Search_Weight diff --git a/search/Zend/Search/Lucene/Search/Weight/Boolean.php b/search/Zend/Search/Lucene/Search/Weight/Boolean.php index 7a42ed28269..08385b79713 100644 --- a/search/Zend/Search/Lucene/Search/Weight/Boolean.php +++ b/search/Zend/Search/Lucene/Search/Weight/Boolean.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Weight */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; +require_once 'Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_Boolean extends Zend_Search_Lucene_Search_Weight diff --git a/search/Zend/Search/Lucene/Search/Weight/Empty.php b/search/Zend/Search/Lucene/Search/Weight/Empty.php index 00c1bce4517..0b1702ae096 100644 --- a/search/Zend/Search/Lucene/Search/Weight/Empty.php +++ b/search/Zend/Search/Lucene/Search/Weight/Empty.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Weight */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; +require_once 'Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_Empty extends Zend_Search_Lucene_Search_Weight diff --git a/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php b/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php index 122cfa4035a..f284a28dacd 100644 --- a/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php +++ b/search/Zend/Search/Lucene/Search/Weight/MultiTerm.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Weight */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; +require_once 'Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_MultiTerm extends Zend_Search_Lucene_Search_Weight diff --git a/search/Zend/Search/Lucene/Search/Weight/Phrase.php b/search/Zend/Search/Lucene/Search/Weight/Phrase.php index 7faaa7ad130..873e44c3039 100644 --- a/search/Zend/Search/Lucene/Search/Weight/Phrase.php +++ b/search/Zend/Search/Lucene/Search/Weight/Phrase.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -23,14 +23,14 @@ /** * Zend_Search_Lucene_Search_Weight */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; +require_once 'Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_Phrase extends Zend_Search_Lucene_Search_Weight diff --git a/search/Zend/Search/Lucene/Search/Weight/Term.php b/search/Zend/Search/Lucene/Search/Weight/Term.php index 478cb2a8058..1d880b2db00 100644 --- a/search/Zend/Search/Lucene/Search/Weight/Term.php +++ b/search/Zend/Search/Lucene/Search/Weight/Term.php @@ -15,20 +15,20 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Search_Weight */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Search/Weight.php'; +require_once 'Zend/Search/Lucene/Search/Weight.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Search - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Search_Weight_Term extends Zend_Search_Lucene_Search_Weight diff --git a/search/Zend/Search/Lucene/SoftLockManager.php b/search/Zend/Search/Lucene/SoftLockManager.php new file mode 100644 index 00000000000..c08e818f508 --- /dev/null +++ b/search/Zend/Search/Lucene/SoftLockManager.php @@ -0,0 +1,225 @@ +fileExists(self::WRITE_LOCK_FILE)){ + throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock : somebody else is writing'); + } + try { + $lock = $lockDirectory->createFile(self::WRITE_LOCK_FILE); + return $lock; + } + catch(Zend_Search_Lucene_Exception $e) { + throw new Zend_Search_Lucene_Exception('Can\'t obtain exclusive index lock'); + } + return null; + } + + /** + * Release exclusive write lock + * + * @param Zend_Search_Lucene_Storage_Directory $lockDirectory + */ + public static function releaseWriteLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) + { + $lockDirectory->deleteFile(self::WRITE_LOCK_FILE); + } + + /** + * Obtain shared read lock on the index + * + * It doesn't block other read or update processes, but prevent index from the premature cleaning-up + * + * @param Zend_Search_Lucene_Storage_Directory $defaultLockDirectory + * @return Zend_Search_Lucene_Storage_File + * @throws Zend_Search_Lucene_Exception + */ + public static function obtainReadLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) + { + // is being written. Can't read + if ($lockDirectory->fileExists(self::WRITE_LOCK_FILE)){ + throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock : write in progress'); + } + + if ($lockDirectory->fileExists(self::EXCLUSIVE_READ_LOCK_FILE)){ + throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock : read lock operation in progress'); + } else { + $lock = $lockDirectory->createFile(self::EXCLUSIVE_READ_LOCK_FILE); + } + + try { + if (!$lockDirectory->fileExists(self::READ_LOCK_FILE)){ + $accesscount = 0; + } else { + // get the existing lock file + $lock = $lockDirectory->getFileObject(self::READ_LOCK_FILE, false); + $accesscount = $lock->readBytes($lock->size()); + $lock->close(); + $lockDirectory->deleteFile(self::READ_LOCK_FILE); + } + + // raise the access counter + $accesscount++; + + // store the access counter + $lock = $lockDirectory->createFile(self::READ_LOCK_FILE); + $lock->writeBytes($accesscount); + $lock->flush(); + $lock->close(); + } + catch (Zend_Search_Lucene_Exception $e) { + $lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE); + print_object($e); + throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock'); + } + + $lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE); + return $lock; + } + + /** + * Release shared read lock + * + * @param Zend_Search_Lucene_Storage_Directory $lockDirectory + */ + public static function releaseReadLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) + { + + // get exclusive operation lock on read lock file. + if ($lockDirectory->fileExists(self::EXCLUSIVE_READ_LOCK_FILE)){ + throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock : read lock operation in progress'); + } else { + $lock = $lockDirectory->createFile(self::EXCLUSIVE_READ_LOCK_FILE); + } + + try { + $lock = $lockDirectory->getFileObject(self::READ_LOCK_FILE, false); + $accesscounter = $lock->readBytes($lock->size()); + $lock->close(); + + $accesscounter--; + + $lockDirectory->deleteFile(self::READ_LOCK_FILE); + + if ($accesscounter > 1){ // we have'nt released all locks. Create the file again. + $lock = $lockDirectory->createFile(self::READ_LOCK_FILE); + $lock->writeBytes($accesscount); + $lock->flush(); + $lock->close(); + } + } + catch(Zend_Search_Lucene_Exception $e){ + $lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE); + // accept failover here + // throw new Zend_Search_Lucene_Exception('Can\'t obtain shared reading index lock'); + } + + $lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE); + } + + /** + * Escalate Read lock to exclusive level + * + * @param Zend_Search_Lucene_Storage_Directory $lockDirectory + * @return boolean + */ + public static function escalateReadLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) + { + // get exclusive operation lock on read lock file. + if ($lockDirectory->fileExists(self::EXCLUSIVE_READ_LOCK_FILE)){ + return false; + } else { + $lock = $lockDirectory->createFile(self::EXCLUSIVE_READ_LOCK_FILE); + return true; + } + } + + /** + * De-escalate Read lock to shared level + * + * @param Zend_Search_Lucene_Storage_Directory $lockDirectory + */ + public static function deEscalateReadLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) + { + $lockDirectory->deleteFile(self::EXCLUSIVE_READ_LOCK_FILE); + } + + /** + * Obtain exclusive optimization lock on the index + * + * Returns lock object on success and false otherwise (doesn't block execution) + * + * @param Zend_Search_Lucene_Storage_Directory $lockDirectory + * @return mixed + */ + public static function obtainOptimizationLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) + { + if ($lockDirectory->fileExists(self::OPTIMIZATION_LOCK_FILE)){ + return false; + } + $lock = $lockDirectory->createFile(self::OPTIMIZATION_LOCK_FILE); + return $lock; + } + + /** + * Release exclusive optimization lock + * + * @param Zend_Search_Lucene_Storage_Directory $lockDirectory + */ + public static function releaseOptimizationLock(Zend_Search_Lucene_Storage_Directory $lockDirectory) + { + $lockDirectory->deleteFile(self::OPTIMIZATION_LOCK_FILE); + } +} diff --git a/search/Zend/Search/Lucene/Storage/Directory.php b/search/Zend/Search/Lucene/Storage/Directory.php index 67e2b295f52..011734efa5e 100644 --- a/search/Zend/Search/Lucene/Storage/Directory.php +++ b/search/Zend/Search/Lucene/Storage/Directory.php @@ -15,7 +15,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ @@ -24,7 +24,7 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Storage_Directory @@ -61,7 +61,16 @@ abstract class Zend_Search_Lucene_Storage_Directory */ abstract public function deleteFile($filename); - + /** + * Purge file if it's cached by directory object + * + * Method is used to prevent 'too many open files' error + * + * @param string $filename + * @return void + */ + abstract public function purgeFile($filename); + /** * Returns true if a file with the given $filename exists. * diff --git a/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php b/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php index 7ac76810ab6..48195cf1336 100644 --- a/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php +++ b/search/Zend/Search/Lucene/Storage/Directory/Filesystem.php @@ -15,16 +15,16 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Storage_Directory */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/Directory.php'; +require_once 'Zend/Search/Lucene/Storage/Directory.php'; /** Zend_Search_Lucene_Storage_File_Filesystem */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File/Filesystem.php'; +require_once 'Zend/Search/Lucene/Storage/File/Filesystem.php'; /** @@ -33,7 +33,7 @@ require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File/Filesystem.p * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene_Storage_Directory @@ -43,7 +43,7 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene * * @var string */ - private $_dirPath = null; + protected $_dirPath = null; /** * Cache for Zend_Search_Lucene_Storage_File_Filesystem objects @@ -52,7 +52,35 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene * @var array * @throws Zend_Search_Lucene_Exception */ - private $_fileHandlers; + protected $_fileHandlers; + + /** + * Default file permissions + * + * @var integer + */ + protected static $_defaultFilePermissions = 0666; + + + /** + * Get default file permissions + * + * @return integer + */ + public static function getDefaultFilePermissions() + { + return self::$_defaultFilePermissions; + } + + /** + * Set default file permissions + * + * @param integer $mode + */ + public static function setDefaultFilePermissions($mode) + { + self::$_defaultFilePermissions = $mode; + } /** @@ -144,6 +172,7 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene * * @param string $filename * @return Zend_Search_Lucene_Storage_File + * @throws Zend_Search_Lucene_Exception */ public function createFile($filename) { @@ -152,6 +181,16 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene } unset($this->_fileHandlers[$filename]); $this->_fileHandlers[$filename] = new Zend_Search_Lucene_Storage_File_Filesystem($this->_dirPath . '/' . $filename, 'w+b'); + + global $php_errormsg; + $trackErrors = ini_get('track_errors'); + ini_set('track_errors', '1'); + if (!@chmod($this->_dirPath . '/' . $filename, self::$_defaultFilePermissions)) { + ini_set('track_errors', $trackErrors); + throw new Zend_Search_Lucene_Exception($php_errormsg); + } + ini_set('track_errors', $trackErrors); + return $this->_fileHandlers[$filename]; } @@ -161,22 +200,42 @@ class Zend_Search_Lucene_Storage_Directory_Filesystem extends Zend_Search_Lucene * * @param string $filename * @return void + * @throws Zend_Search_Lucene_Exception */ public function deleteFile($filename) { - /** - * @todo add support of "deletable" file - * "deletable" is used on Windows systems if file can't be deleted - * (while it is still open). - */ - if (isset($this->_fileHandlers[$filename])) { $this->_fileHandlers[$filename]->close(); } unset($this->_fileHandlers[$filename]); - unlink($this->_dirPath . '/' . $filename); + + global $php_errormsg; + $trackErrors = ini_get('track_errors'); ini_set('track_errors', '1'); + if (!@unlink($this->_dirPath . '/' . $filename)) { + ini_set('track_errors', $trackErrors); + + // keep loose on failed deletion. + // throw new Zend_Search_Lucene_Exception('Can\'t delete file: ' . $php_errormsg); + } + ini_set('track_errors', $trackErrors); } + /** + * Purge file if it's cached by directory object + * + * Method is used to prevent 'too many open files' error + * + * @param string $filename + * @return void + */ + public function purgeFile($filename) + { + if (isset($this->_fileHandlers[$filename])) { + $this->_fileHandlers[$filename]->close(); + } + unset($this->_fileHandlers[$filename]); + } + /** * Returns true if a file with the given $filename exists. diff --git a/search/Zend/Search/Lucene/Storage/File.php b/search/Zend/Search/Lucene/Storage/File.php index 8370c719db7..8a4fab5a188 100644 --- a/search/Zend/Search/Lucene/Storage/File.php +++ b/search/Zend/Search/Lucene/Storage/File.php @@ -15,21 +15,21 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ abstract class Zend_Search_Lucene_Storage_File diff --git a/search/Zend/Search/Lucene/Storage/File/Filesystem.php b/search/Zend/Search/Lucene/Storage/File/Filesystem.php index 1f2097eb4f6..496551d6fa6 100644 --- a/search/Zend/Search/Lucene/Storage/File/Filesystem.php +++ b/search/Zend/Search/Lucene/Storage/File/Filesystem.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Storage_File */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File.php'; +require_once 'Zend/Search/Lucene/Storage/File.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Storage_File @@ -41,7 +41,8 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor * * @var resource */ - private $_fileHandle; + protected $_fileHandle; + /** * Class constructor. Open the file. @@ -53,6 +54,11 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor { global $php_errormsg; + if (strpos($mode, 'w') === false && !is_readable($filename)) { + // opening for reading non-readable file + throw new Zend_Search_Lucene_Exception('File \'' . $filename . '\' is not readable.'); + } + $trackErrors = ini_get('track_errors'); ini_set('track_errors', '1'); @@ -185,12 +191,12 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor * Lock type may be a LOCK_SH (shared lock) or a LOCK_EX (exclusive lock) * * @param integer $lockType - * @param boolean $nonBlockinLock + * @param boolean $nonBlockingLock * @return boolean */ - public function lock($lockType, $nonBlockinLock = false) + public function lock($lockType, $nonBlockingLock = false) { - if ($nonBlockinLock) { + if ($nonBlockingLock) { return flock($this->_fileHandle, $lockType | LOCK_NB); } else { return flock($this->_fileHandle, $lockType); @@ -214,3 +220,4 @@ class Zend_Search_Lucene_Storage_File_Filesystem extends Zend_Search_Lucene_Stor } } +?> \ No newline at end of file diff --git a/search/Zend/Search/Lucene/Storage/File/Memory.php b/search/Zend/Search/Lucene/Storage/File/Memory.php index e3567c66dab..e830bcea703 100644 --- a/search/Zend/Search/Lucene/Storage/File/Memory.php +++ b/search/Zend/Search/Lucene/Storage/File/Memory.php @@ -15,23 +15,23 @@ * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ /** Zend_Search_Lucene_Storage_File */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Storage/File.php'; +require_once 'Zend/Search/Lucene/Storage/File.php'; /** Zend_Search_Lucene_Exception */ -require_once $CFG->dirroot.'/search/Zend/Search/Lucene/Exception.php'; +require_once 'Zend/Search/Lucene/Exception.php'; /** * @category Zend * @package Zend_Search_Lucene * @subpackage Storage - * @copyright Copyright (c) 2005-2007 Zend Technologies USA Inc. (http://www.zend.com) + * @copyright Copyright (c) 2005-2008 Zend Technologies USA Inc. (http://www.zend.com) * @license http://framework.zend.com/license/new-bsd New BSD License */ class Zend_Search_Lucene_Storage_File_Memory extends Zend_Search_Lucene_Storage_File diff --git a/search/documents/assignment_document.php b/search/documents/assignment_document.php new file mode 100644 index 00000000000..7633111809b --- /dev/null +++ b/search/documents/assignment_document.php @@ -0,0 +1,362 @@ + 1.8 +* @date 2008/03/31 +* @license http://www.gnu.org/copyleft/gpl.html GNU Public License +* +* document handling for assignment activity module +* +*/ + +/** +* includes and requires +*/ +require_once("$CFG->dirroot/search/documents/document.php"); +require_once("$CFG->dirroot/mod/assignment/lib.php"); + +/** +* a class for representing searchable information +* +*/ +class AssignmentSearchDocument extends SearchDocument { + + /** + * constructor + */ + public function __construct(&$assignmentitem, $assignment_module_id, $itemtype, $course_id, $owner_id, $context_id) { + // generic information; required + $doc->docid = $assignmentitem['id']; + $doc->documenttype = SEARCH_TYPE_ASSIGNMENT; + $doc->itemtype = $itemtype; + $doc->contextid = $context_id; + + // we cannot call userdate with relevant locale at indexing time. + $doc->title = "{$itemtype}: {$assignmentitem['name']}"; + $doc->date = $assignmentitem['date']; + + //remove '(ip.ip.ip.ip)' from chat author list + $doc->author = $assignmentitem['authors']; + $doc->contents = $assignmentitem['description']; + $doc->url = assignment_make_link($assignment_module_id, $itemtype, $owner_id); + + // module specific information; optional + $data->assignment = $assignment_module_id; + $data->assignmenttype = $assignmentitem['assignmenttype']; + + // construct the parent class + parent::__construct($doc, $data, $course_id, 0, 0, 'mod/'.SEARCH_TYPE_ASSIGNMENT); + } +} + + +/** +* constructs a valid link to a chat content +* @param cm_id the chat course module +* @param start the start time of the session +* @param end th end time of the session +* @uses CFG +* @return a well formed link to session display +*/ +function assignment_make_link($cm_id, $itemtype, $owner) { + global $CFG; + + if ($itemtype == 'description') + return $CFG->wwwroot.'/mod/assignment/view.php?id='.$cm_id; +} + +/** +* part of search engine API +* +*/ +function assignment_iterator() { + if ($assignments = get_records('assignment')) + return $assignments; + else + return array(); +} + +/** +* part of search engine API +* +*/ +function assignment_get_content_for_index(&$assignment) { + global $CFG; + + $documents = array(); + $course = get_record('course', 'id', $assignment->course); + $coursemodule = get_field('modules', 'id', 'name', 'assignment'); + $cm = get_record('course_modules', 'course', $assignment->course, 'module', $coursemodule, 'instance', $assignment->id); + if ($cm){ + $context = get_context_instance(CONTEXT_MODULE, $cm->id); + + $assignment->authors = ''; + $assignment->date = $assignment->timemodified; + $documents[] = new AssignmentSearchDocument(get_object_vars($assignment), $cm->id, 'description', $assignment->course, null, $context->id); + + $submissions = assignment_get_all_submissions($assignment); + if ($submissions){ + foreach($submissions as $submission){ + $owner = get_record('user', 'id', $submission->userid); + $submission->authors = fullname($owner); + $submission->assignmenttype = $assignment->assignmenttype; + $submission->date = $submission->timemodified; + $submission->name = "submission:"; + if (file_exists("{$CFG->dirroot}/mod/assignment/type/{$assignment->assignmenttype}/searchlib.php")){ + include_once("{$CFG->dirroot}/mod/assignment/type/{$assignment->assignmenttype}/searchlib.php"); + if (function_exists('assignment_get_submission_location')){ + $submitted = assignment_get_submission_location($assignment, $submission); + } + } + if (empty($submitted)){ + // this is for moodle legacy types that would need not to be patched for searchlib.php + switch($assignment->assignmenttype){ + case 'online' : { + $submitted->source = 'text'; + $submitted->data = $submission->data1; + } + break; + case 'uploadsingle' : + case 'upload' : { + $submitted->source = 'files'; + $submitted->data = "{$assignment->course}/moddata/assignment/{$assignment->id}/{$submission->userid}"; + } + break; + case 'offline' : continue; // cannot index, no content in Moodle !! + } + } + if (empty($submitted)) continue; // ignoring + + if ($submitted->source = 'text'){ + $submission->description = $submitted->data; + $submission->description = preg_replace("/<[^>]*>/", '', $submission->description); // stip all tags + $documents[] = new AssignmentSearchDocument(get_object_vars($submission), $cm->id, 'submission', $assignment->course, $submission->userid, $context->id); + mtrace("finished online submission for {$submission->authors} in assignement {$assignment->name}"); + } elseif ($submitted->source = 'files'){ + $SUBMITTED = opendir($submitted->path); + while($entry = readdir($SUBMITTED)){ + if (ereg("^\.", $entry)) continue; // exclude hidden and dirs . and .. + $path = "{$submitted->path}/{$entry}"; + $documents[] = assignment_get_physical_file($submission, $assignment, $cm, $path, $context_id, $documents); + mtrace("finished attachement $path for {$submission->authors} in assignement {$assignment->name}"); + } + closedir($submission->path); + } + } + } + mtrace("finished assignment {$assignment->name}"); + return $documents; + } + return array(); +} + +/** +* get text from a physical file in an assignment submission +* @param object $submission a submission for which to fetch some representative text +* @param object $assignment the relevant assignment as a context +* @param object $cm the corresponding coursemodule +* @param string $path a file from which to fetch some representative text +* @param int $contextid the moodle context if needed +* @param documents the array of documents, by ref, where to add the new document. +* @return a search document when unique or false. +*/ +function assignment_get_physical_file(&$submission, &$assignment, &$cm, $path, $context_id, &$documents = null){ + global $CFG; + + $fileparts = pathinfo($path); + // cannot index unknown or masked types + if (empty($fileparts['extension'])) { + mtrace("Cannot index without explicit extension."); + return false; + } + + $ext = strtolower($fileparts['extension']); + + // cannot index unallowed or unhandled types + if (!preg_match("/\b$ext\b/i", $CFG->block_search_filetypes)) { + mtrace($fileparts['extension'] . ' is not an allowed extension for indexing'); + return false; + } + if (file_exists($CFG->dirroot.'/search/documents/physical_'.$ext.'.php')){ + include_once($CFG->dirroot.'/search/documents/physical_'.$ext.'.php'); + $function_name = 'get_text_for_indexing_'.$ext; + $submission->description = $function_name(null, $path); + + // get authors + $user = get_record('user', 'id', $submission->userid); + $submission->authors = fullname($user); + + // we need a real id on file + $submission->id = "{$submission->id}/{$path}"; + + if (!empty($submission->description)){ + if ($getsingle){ + $single = new AssignmentSearchDocument(get_object_vars($submission), $cm->id, 'submitted', $assignment->course, $submission->userid, $context_id); + mtrace("finished submission file from {$submission->authors}"); + return $single; + } else { + $documents[] = new AssignmentSearchDocument(get_object_vars($submission), $cm->id, 'submitted', $assignment->course, $submission->userid, $context_id); + } + mtrace("finished submission file from {$submission->authors}"); + } + } else { + mtrace("fulltext handler not found for $ext type"); + } + return false; +} + +/** +* returns a single data search document based on an assignment +* @param string $id the id of the searchable item +* @param string $itemtype the type of information +*/ +function assignment_single_document($id, $itemtype) { + if ($itemtype == 'requirement'){ + if (!$assignment = get_record('assignment', 'id', $id)){ + return null; + } + } elseif ($itemtype == 'submission') { + if ($submission = get_record('assignment_submissions', 'id', $id)){ + $assignment = get_record('assignment', 'id', $submission->assignment); + } else { + return null; + } + } + $course = get_record('course', 'id', $assignment->course); + $coursemodule = get_field('modules', 'id', 'name', 'assignment'); + $cm = get_record('course_modules', 'course', $course->id, 'module', $coursemodule, 'instance', $assignment->id); + if ($cm){ + $context = get_context_instance(CONTEXT_MODULE, $cm->id); + + // should be only one + if ($itemtype == 'description'){ + $document = new AssignmentSearchDocument(get_object_vars($assignment), $cm->id, 'description', $assignment->course, null, $context->id); + return $document; + } + if ($itemtype == 'submittted'){ + $document = new AssignmentSearchDocument(get_object_vars($submission), $cm->id, 'submitted', $assignment->course, null, $context->id); + return $document; + } + } + return null; +} + +/** +* dummy delete function that packs id with itemtype. +* this was here for a reason, but I can't remember it at the moment. +* +*/ +function assignment_delete($info, $itemtype) { + $object->id = $info; + $object->itemtype = $itemtype; + return $object; +} + +/** +* returns the var names needed to build a sql query for addition/deletions +* // TODO chat indexable records are virtual. Should proceed in a special way +*/ +function assignment_db_names() { + //[primary id], [table name], [time created field name], [time modified field name] + return array( + array('id', 'assignment', 'timemodified', 'timemodified', 'description'), + array('id', 'assignment_submissions', 'timecreated', 'timemodified', 'submitted') + ); +} + +/** +* this function handles the access policy to contents indexed as searchable documents. If this +* function does not exist, the search engine assumes access is allowed. +* When this point is reached, we already know that : +* - user is legitimate in the surrounding context +* - user may be guest and guest access is allowed to the module +* - the function may perform local checks within the module information logic +* @param path the access path to the module script code +* @param itemtype the information subclassing (usefull for complex modules, defaults to 'standard') +* @param this_id the item id within the information class denoted by entry_type. In chats, this id +* points out a session history which is a close sequence of messages. +* @param user the user record denoting the user who searches +* @param group_id the current group used by the user when searching +* @uses CFG +* @return true if access is allowed, false elsewhere +*/ +function assignment_check_text_access($path, $itemtype, $this_id, $user, $group_id, $context_id){ + global $CFG, $USER; + + include_once("{$CFG->dirroot}/{$path}/lib.php"); + + // get the chat session and all related stuff + if ($itemtype == 'description'){ + $assignment = get_record('assignment', 'id', $this_id); + } elseif ($itemtype == 'submitted'){ + $submission = get_record('assignment_submissions', 'id', $this_id); + $assignment = get_record('assignment', 'id', $submission->assignment); + } + $context = get_record('context', 'id', $context_id); + $cm = get_record('course_modules', 'id', $context->instanceid); + + if (!$cm->visible and !has_capability('moodle/course:viewhiddenactivities', $context)){ + if (!empty($CFG->search_access_debug)) echo "search reject : hidden assignment "; + return false; + } + + /* + group consistency check : checks the following situations about groups + // trap if user is not same group and groups are separated + $current_group = get_current_group($course->id); + $course = get_record('course', 'id', $assignment->course); + if ((groupmode($course, $cm) == SEPARATEGROUPS) && !ismember($group_id) && !has_capability('moodle/site:accessallgroups', $context)){ + if (!empty($CFG->search_access_debug)) echo "search reject : assignment element is in separated group "; + return false; + } + */ + + //user ownership check : + // trap if user is not owner of the resource and the ressource is a submission/attachement + if ($itemtype == 'submitted' && $USER->id != $submission->userid && !has_capability('mod/assignment:view', $context)){ + if (!empty($CFG->search_access_debug)) echo "search reject : i'm not owner of this assignment "; + return false; + } + + //date check : no submission may be viewed before timedue + if ($itemtype == 'submitted' && $assignment->timedue < time()){ + if (!empty($CFG->search_access_debug)) echo "search reject : cannot read submissions before end of assignment "; + return false; + } + + //ownership check : checks the following situations about user + // trap if user is not owner and cannot see other's entries + // TODO : typically may be stored into indexing cache + if (!has_capability('mod/assignment:view', $context)){ + if (!empty($CFG->search_access_debug)) echo "search reject : cannot read past sessions "; + return false; + } + + return true; +} + +/** +* this call back is called when displaying the link for some last post processing +* +*/ +function assignment_link_post_processing($title){ + global $CFG; + + if (!function_exists('search_assignment_getstring')){ + function search_assignment_getstring($matches){ + return get_string($matches[1], 'assignment'); + } + } + + $title = preg_replace_callback('/^(description|submitted)/', 'search_assignment_getstring', $title); + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); +} +?> \ No newline at end of file diff --git a/search/documents/chat_document.php b/search/documents/chat_document.php index acc4ffb479b..363b4389c54 100644 --- a/search/documents/chat_document.php +++ b/search/documents/chat_document.php @@ -32,9 +32,9 @@ class ChatTrackSearchDocument extends SearchDocument { /** * constructor */ - public function __construct(&$chatsession, $chat_module_id, $course_id, $group_id, $context_id) { + public function __construct(&$chatsession, $chat_id, $chat_module_id, $course_id, $group_id, $context_id) { // generic information; required - $doc->docid = $chat_module_id.'-'.$chatsession['sessionstart'].'-'.$chatsession['sessionend']; + $doc->docid = $chat_id.'-'.$chatsession['sessionstart'].'-'.$chatsession['sessionend']; $doc->documenttype = SEARCH_TYPE_CHAT; $doc->itemtype = 'session'; $doc->contextid = $context_id; @@ -50,10 +50,10 @@ class ChatTrackSearchDocument extends SearchDocument { $doc->url = chat_make_link($chat_module_id, $chatsession['sessionstart'], $chatsession['sessionend']); // module specific information; optional - $data->chat = $chat_module_id; + $data->chat = $chat_id; // construct the parent class - parent::__construct($doc, $data, $course_id, $group_id, 0, PATH_FOR_SEARCH_TYPE_CHAT); + parent::__construct($doc, $data, $course_id, $group_id, 0, 'mod/'.SEARCH_TYPE_CHAT); } } @@ -176,7 +176,7 @@ function chat_get_content_for_index(&$chat) { foreach($aTrack->sessionusers as $aUserId){ $user = get_record('user', 'id', $aUserId); $aTrack->authors = ($user) ? fullname($user) : '' ; - $documents[] = new ChatTrackSearchDocument(get_object_vars($aTrack), $cm->id, $chat->course, $aTrack->groupid, $context->id); + $documents[] = new ChatTrackSearchDocument(get_object_vars($aTrack), $chat->id, $cm->id, $chat->course, $aTrack->groupid, $context->id); } } } @@ -207,7 +207,7 @@ function chat_single_document($id, $itemtype) { $tracks = chat_get_session_tracks($chat->id, $sessionstart, $sessionstart); if ($tracks){ $aTrack = $tracks[0]; - $document = new ChatTrackSearchDocument(get_object_vars($aTrack), $cm->id, $chat->course, $aTrack->groupid, $context->id); + $document = new ChatTrackSearchDocument(get_object_vars($aTrack), $chat_id, $cm->id, $chat->course, $aTrack->groupid, $context->id); return $document; } } @@ -255,8 +255,7 @@ function chat_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c include_once("{$CFG->dirroot}/{$path}/lib.php"); - list($chat_id, $sessionstart, $sessionend) = split('-', $id); - + list($chat_id, $sessionstart, $sessionend) = split('-', $this_id); // get the chat session and all related stuff $chat = get_record('chat', 'id', $chat_id); $context = get_record('context', 'id', $context_id); @@ -271,7 +270,6 @@ function chat_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c //group consistency check : checks the following situations about groups // trap if user is not same group and groups are separated - $current_group = get_current_group($course->id); $course = get_record('course', 'id', $chat->course); if ((groupmode($course, $cm) == SEPARATEGROUPS) && !ismember($group_id) && !has_capability('moodle/site:accessallgroups', $context)){ if (!empty($CFG->search_access_debug)) echo "search reject : chat element is in separated group "; @@ -294,8 +292,13 @@ function chat_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c * */ function chat_link_post_processing($title){ - setLocale(LC_TIME, substr(current_language(), 0, 2)); - $title = preg_replace('/TT_(.*)_TT/e', "userdate(\\1)", $title); - return mb_convert_encoding($title, 'UTF-8', 'auto'); + global $CFG; + setLocale(LC_TIME, substr(current_language(), 0, 2)); + $title = preg_replace('/TT_(.*)_TT/e', "userdate(\\1)", $title); + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); } ?> \ No newline at end of file diff --git a/search/documents/data_document.php b/search/documents/data_document.php index 4b301ffaa82..21b0682a41a 100644 --- a/search/documents/data_document.php +++ b/search/documents/data_document.php @@ -54,7 +54,7 @@ class DataSearchDocument extends SearchDocument { $data->database = $record['dataid']; // construct the parent class - parent::__construct($doc, $data, $course_id, $record['groupid'], $record['userid'], PATH_FOR_SEARCH_TYPE_DATA); + parent::__construct($doc, $data, $course_id, $record['groupid'], $record['userid'], 'mod/'.SEARCH_TYPE_DATA); } } @@ -203,6 +203,7 @@ function data_get_content_for_index(&$database) { unset($records_content[$aRecordId]['_first']); // concatenates all other texts + $content = ''; foreach($records_content[$aRecordId] as $aField){ $content = @$content.' '.$aField; } @@ -348,7 +349,6 @@ function data_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c //group consistency check : checks the following situations about groups // trap if user is not same group and groups are separated - $current_group = get_current_group($course->id); $course = get_record('course', 'id', $data->course); if ((groupmode($course, $cm) == SEPARATEGROUPS) && !ismember($group_id) && !has_capability('moodle/site:accessallgroups', $context)){ if (!empty($CFG->search_access_debug)) echo "search reject : separated group owned resource "; @@ -368,7 +368,7 @@ function data_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c // trap if unapproved and has not approval capabilities // TODO : report a potential capability lack of : mod/data:approve $approval = get_field('data_records', 'approved', 'id', $record->id); - if (!$approval && !isteacher($data->course) && !has_capability('mod/data:manageentries', $context)){ + if (!$approval && !has_capability('mod/data:manageentries', $context)){ if (!empty($CFG->search_access_debug)) echo "search reject : unapproved resource "; return false; } @@ -377,7 +377,7 @@ function data_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c // trap if too few records // TODO : report a potential capability lack of : mod/data:viewhiddenentries $recordsAmount = count_records('data_records', 'dataid', $data->id); - if ($data->requiredentriestoview > $recordsAmount && !isteacher($data->course) && !has_capability('mod/data:manageentries', $context)) { + if ($data->requiredentriestoview > $recordsAmount && !has_capability('mod/data:manageentries', $context)) { if (!empty($CFG->search_access_debug)) echo "search reject : not enough records to view "; return false; } @@ -387,12 +387,12 @@ function data_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c // TODO : report a potential capability lack of : mod/data:viewhiddenentries $now = usertime(time()); if ($data->timeviewfrom > 0) - if ($now < $data->timeviewfrom && !isteacher($data->course) && !has_capability('mod/data:manageentries', $context)) { + if ($now < $data->timeviewfrom && !has_capability('mod/data:manageentries', $context)) { if (!empty($CFG->search_access_debug)) echo "search reject : still not open activity "; return false; } if ($data->timeviewto > 0) - if ($now > $data->timeviewto && !isteacher($data->course) && !has_capability('mod/data:manageentries', $context)) { + if ($now > $data->timeviewto && !has_capability('mod/data:manageentries', $context)) { if (!empty($CFG->search_access_debug)) echo "search reject : closed activity "; return false; } @@ -405,7 +405,12 @@ function data_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c * @param string $title */ function data_link_post_processing($title){ - return mb_convert_encoding($title, 'UTF-8', 'auto'); + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); } ?> \ No newline at end of file diff --git a/search/documents/document.php b/search/documents/document.php index 266dead49a9..ae3f46c7a4f 100644 --- a/search/documents/document.php +++ b/search/documents/document.php @@ -17,7 +17,7 @@ * */ abstract class SearchDocument extends Zend_Search_Lucene_Document { - public function __construct(&$doc, &$data, $course_id, $group_id, $user_id, $path) { + public function __construct(&$doc, &$data, $course_id, $group_id, $user_id, $path, $additional_keyset = null) { //document identification and indexing $this->addField(Zend_Search_Lucene_Field::Keyword('docid', $doc->docid)); //document type : the name of the Moodle element that manages it @@ -59,6 +59,16 @@ abstract class SearchDocument extends Zend_Search_Lucene_Document { // of multiple capabilities in their code. This possibility should be left open here. $this->addField(Zend_Search_Lucene_Field::UnIndexed('capabilities', $caps)); */ + + /* + // Additional key set allows a module to ask for extensible criteria based search + // depending on the module internal needs. + */ + if (!empty($additional_keyset)){ + foreach($additional_keyset as $keyname => $keyvalue){ + $this->addField(Zend_Search_Lucene_Field::Keyword($keyname, $keyvalue)); + } + } } } diff --git a/search/documents/forum_document.php b/search/documents/forum_document.php index 3cbe2e108a3..012eefc9532 100644 --- a/search/documents/forum_document.php +++ b/search/documents/forum_document.php @@ -51,7 +51,7 @@ class ForumSearchDocument extends SearchDocument { $data->forum = $forum_id; $data->discussion = $post['discussion']; - parent::__construct($doc, $data, $course_id, $post['groupid'], $post['userid'], PATH_FOR_SEARCH_TYPE_FORUM); + parent::__construct($doc, $data, $course_id, $post['groupid'], $post['userid'], 'mod/'.SEARCH_TYPE_FORUM); } } @@ -87,6 +87,7 @@ function forum_get_content_for_index(&$forum) { if (!$forum) return $documents; $posts = forum_get_discussions_fast($forum->id); + mtrace("Found ".count($posts)." discussions to analyse in forum ".$forum->name); if (!$posts) return $documents; $coursemodule = get_field('modules', 'id', 'name', 'forum'); @@ -96,11 +97,13 @@ function forum_get_content_for_index(&$forum) { foreach($posts as $aPost) { $aPost->itemtype = 'head'; if ($aPost) { - if (strlen($aPost->message) > 0) { + if (!empty($aPost->message)) { + echo "*"; $documents[] = new ForumSearchDocument(get_object_vars($aPost), $forum->id, $forum->course, 'head', $context->id); } if ($children = forum_get_child_posts_fast($aPost->id, $forum->id)) { foreach($children as $aChild) { + echo "."; $aChild->itemtype = 'post'; if (strlen($aChild->message) > 0) { $documents[] = new ForumSearchDocument(get_object_vars($aChild), $forum->id, $forum->course, 'post', $context->id); @@ -109,6 +112,7 @@ function forum_get_content_for_index(&$forum) { } } } + mtrace("Finished discussion"); return $documents; } @@ -166,7 +170,8 @@ function forum_get_discussions_fast($forum_id) { $timelimit=''; if (!empty($CFG->forum_enabletimedposts)) { - if (!((isadmin() and !empty($CFG->admineditalways)) || isteacher(get_field('forum', 'course', 'id', $forum_id)))) { + if (!((has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM)) + and !empty($CFG->admineditalways)) || isteacher(get_field('forum', 'course', 'id', $forum_id)))) { $now = time(); $timelimit = " AND ((d.timestart = 0 OR d.timestart <= '$now') AND (d.timeend = 0 OR d.timeend > '$now')"; if (!empty($USER->id)) { @@ -302,7 +307,12 @@ function forum_check_text_access($path, $itemtype, $this_id, $user, $group_id, $ * @param string $title */ function forum_link_post_processing($title){ - return mb_convert_encoding($title, 'UTF-8', 'auto'); + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); } ?> \ No newline at end of file diff --git a/search/documents/glossary_document.php b/search/documents/glossary_document.php index 2ec59ef8fd6..c1b89eaec27 100644 --- a/search/documents/glossary_document.php +++ b/search/documents/glossary_document.php @@ -52,7 +52,7 @@ class GlossarySearchDocument extends SearchDocument { $data->glossary = $entry['glossaryid']; // construct the parent class - parent::__construct($doc, $data, $course_id, -1, $entry['userid'], PATH_FOR_SEARCH_TYPE_GLOSSARY); + parent::__construct($doc, $data, $course_id, -1, $entry['userid'], 'mod/'.SEARCH_TYPE_GLOSSARY); } } @@ -122,6 +122,7 @@ function glossary_iterator() { * @return an array of searchable documents */ function glossary_get_content_for_index(&$glossary) { + // global $DB; // get context $coursemodule = get_field('modules', 'id', 'name', 'glossary'); @@ -144,8 +145,7 @@ function glossary_get_content_for_index(&$glossary) { // index comments if (count($entryIds)){ - $entryIdList = implode(',', $entryIds); - $comments = get_records_list('glossary_comments', 'entryid', $entryIdList); + $comments = get_records_list('glossary_comments', 'entryid', implode(',',$entryIds)); if ($comments){ foreach($comments as $comment) { if (strlen($comment->entrycomment) > 0) { @@ -227,7 +227,7 @@ function glossary_check_text_access($path, $itemtype, $this_id, $user, $group_id global $CFG; // get the glossary object and all related stuff - $entry = get_record('glossary_entries', 'id', $id); + $entry = get_record('glossary_entries', 'id', $this_id); $glossary = get_record('glossary', 'id', $entry->glossaryid); $context = get_record('context', 'id', $context_id); $cm = get_record('course_modules', 'id', $context->instanceid); @@ -251,7 +251,12 @@ function glossary_check_text_access($path, $itemtype, $this_id, $user, $group_id * @param string $title */ function glossary_link_post_processing($title){ - return mb_convert_encoding($title, 'UTF-8', 'auto'); + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); } ?> \ No newline at end of file diff --git a/search/documents/label_document.php b/search/documents/label_document.php new file mode 100644 index 00000000000..db6af70756f --- /dev/null +++ b/search/documents/label_document.php @@ -0,0 +1,184 @@ + 1.9 +* @date 2008/03/31 +* @license http://www.gnu.org/copyleft/gpl.html GNU Public License +* +* document handling for all resources +* This file contains the mapping between a resource and it's indexable counterpart, +* +* Functions for iterating and retrieving the necessary records are now also included +* in this file, rather than mod/resource/lib.php +*/ + +/** +* requires and includes +*/ +require_once("$CFG->dirroot/search/documents/document.php"); +require_once("$CFG->dirroot/mod/resource/lib.php"); + +/* * +* a class for representing searchable information +* +*/ +class LabelSearchDocument extends SearchDocument { + public function __construct(&$label, $context_id) { + // generic information; required + $doc->docid = $label['course']; + $doc->documenttype = SEARCH_TYPE_LABEL; + $doc->itemtype = 'label'; + $doc->contextid = $context_id; + + $doc->title = strip_tags($label['name']); + $doc->date = $label['timemodified']; + $doc->author = ''; + $doc->contents = strip_tags($label['name']); + $doc->url = label_make_link($label['course']); + + // module specific information; optional + $data = array(); + + // construct the parent class + parent::__construct($doc, $data, $label['course'], 0, 0, 'mod/'.SEARCH_TYPE_LABEL); + } //constructor +} + +/** +* constructs valid access links to information +* @param resourceId the of the resource +* @return a full featured link element as a string +*/ +function label_make_link($course_id) { + global $CFG; + + return $CFG->wwwroot.'/course/view.php?id='.$course_id; +} + +/** +* part of standard API +* +*/ +function label_iterator() { + //trick to leave search indexer functionality intact, but allow + //this document to only use the below function to return info + //to be searched + $labels = get_records('label'); + return $labels; +} + +/** +* part of standard API +* this function does not need a content iterator, returns all the info +* itself; +* @param notneeded to comply API, remember to fake the iterator array though +* @uses CFG +* @return an array of searchable documents +*/ +function label_get_content_for_index(&$label) { + global $CFG; + + // starting with Moodle native resources + $documents = array(); + + $coursemodule = get_field('modules', 'id', 'name', 'label'); + $cm = get_record('course_modules', 'course', $label->course, 'module', $coursemodule, 'instance', $label->id); + $context = get_context_instance(CONTEXT_MODULE, $cm->id); + + $documents[] = new LabelSearchDocument(get_object_vars($label), $context->id); + + mtrace("finished label {$label->id}"); + return $documents; +} + +/** +* part of standard API. +* returns a single resource search document based on a label id +* @param id the id of the accessible document +* @return a searchable object or null if failure +*/ +function label_single_document($id, $itemtype) { + global $CFG; + + $label = get_record('label', 'id', $id); + + if ($label){ + $coursemodule = get_field('modules', 'id', 'name', 'label'); + $cm = get_record('course_modules', 'id', $label->id); + $context = get_context_instance(CONTEXT_MODULE, $cm->id); + return new LabelSearchDocument(get_object_vars($label), $context->id); + } + return null; +} + +/** +* dummy delete function that aggregates id with itemtype. +* this was here for a reason, but I can't remember it at the moment. +* +*/ +function label_delete($info, $itemtype) { + $object->id = $info; + $object->itemtype = $itemtype; + return $object; +} //resource_delete + +/** +* returns the var names needed to build a sql query for addition/deletions +* +*/ +function label_db_names() { + //[primary id], [table name], [time created field name], [time modified field name], [docsubtype], [additional where conditions for sql] + return array(array('id', 'label', 'timemodified', 'timemodified', 'label', '')); +} + +/** +* this function handles the access policy to contents indexed as searchable documents. If this +* function does not exist, the search engine assumes access is allowed. +* @param path the access path to the module script code +* @param itemtype the information subclassing (usefull for complex modules, defaults to 'standard') +* @param this_id the item id within the information class denoted by itemtype. In resources, this id +* points to the resource record and not to the module that shows it. +* @param user the user record denoting the user who searches +* @param group_id the current group used by the user when searching +* @return true if access is allowed, false elsewhere +*/ +function label_check_text_access($path, $itemtype, $this_id, $user, $group_id, $context_id){ + global $CFG; + + // include_once("{$CFG->dirroot}/{$path}/lib.php"); + + $r = get_record('label', 'id', $this_id); + $module_context = get_record('context', 'id', $context_id); + $cm = get_record('course_modules', 'id', $module_context->instanceid); + $course_context = get_context_instance(COURSE_CONTEXT, $r->course); + + //check if englobing course is visible + if (!has_capability('moodle/course:view', $course_context)){ + return false; + } + + //check if found course module is visible + if (!$cm->visible and !has_capability('moodle/course:viewhiddenactivities', $module_context)){ + return false; + } + + return true; +} + +/** +* post processes the url for cleaner output. +* @param string $title +*/ +function label_link_post_processing($title){ + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding("(".shorten_text(clean_text($title), 60)."...) ", 'UTF-8', 'auto'); + } + return mb_convert_encoding("(".shorten_text(clean_text($title), 60)."...) ", 'auto', 'UTF-8'); +} +?> \ No newline at end of file diff --git a/search/documents/lesson_document.php b/search/documents/lesson_document.php index 674dc0b63ca..a18d569259a 100644 --- a/search/documents/lesson_document.php +++ b/search/documents/lesson_document.php @@ -49,7 +49,7 @@ class LessonPageSearchDocument extends SearchDocument { // module specific information $data->lesson = $page['lessonid']; - parent::__construct($doc, $data, $course_id, 0, 0, PATH_FOR_SEARCH_TYPE_LESSON); + parent::__construct($doc, $data, $course_id, 0, 0, 'mod/'.SEARCH_TYPE_LESSON); } } @@ -73,8 +73,11 @@ function lesson_make_link($lessonmoduleid, $itemid, $itemtype) { * */ function lesson_iterator() { - $lessons = get_records('lesson'); - return $lessons; + if ($lessons = get_records('lesson')){ + return $lessons; + } else { + return array(); + } } /** @@ -210,7 +213,12 @@ function lesson_check_text_access($path, $itemtype, $this_id, $user, $group_id, * */ function lesson_link_post_processing($title){ - return mb_convert_encoding($title, 'UTF-8', 'auto'); + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); } ?> \ No newline at end of file diff --git a/search/documents/physical_doc.php b/search/documents/physical_doc.php index ca053bde51d..2cd48cd4fae 100644 --- a/search/documents/physical_doc.php +++ b/search/documents/physical_doc.php @@ -16,13 +16,13 @@ /** * MS Word extractor * @param object $resource -* @uses CFG, USER +* @uses $CFG */ -function get_text_for_indexing_doc(&$resource){ - global $CFG, $USER; +function get_text_for_indexing_doc(&$resource, $directfile = ''){ + global $CFG; // SECURITY : do not allow non admin execute anything on system !! - if (!isadmin($USER->id)) return; + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; $moodleroot = (@$CFG->block_search_usemoodleroot) ? "{$CFG->dirroot}/" : '' ; @@ -30,26 +30,27 @@ function get_text_for_indexing_doc(&$resource){ if (!empty($CFG->block_search_word_to_text_cmd)){ if (!file_exists("{$moodleroot}{$CFG->block_search_word_to_text_cmd}")){ mtrace('Error with MSWord to text converter command : exectuable not found.'); - } - else{ - $file = escapeshellarg($CFG->dataroot.'/'.$resource->course.'/'.$resource->reference); + } else { + if ($directfile == ''){ + $file = escapeshellarg("{$CFG->dataroot}/{$resource->course}/{$resource->reference}"); + } else { + $file = escapeshellarg("{$CFG->dataroot}/{$directfile}"); + } $command = trim($CFG->block_search_word_to_text_cmd); - $text_converter_cmd = "{$moodleroot}{$command} $file"; + $text_converter_cmd = "{$moodleroot}{$command} -m UTF-8.txt $file"; if ($CFG->block_search_word_to_text_env){ putenv($CFG->block_search_word_to_text_env); } mtrace("Executing : $text_converter_cmd"); $result = shell_exec($text_converter_cmd); if ($result){ - return mb_convert_encoding($result, 'UTF8', 'auto'); - } - else{ + return mb_convert_encoding($result, 'UTF-8', 'auto'); + } else { mtrace('Error with MSWord to text converter command : execution failed. '); return ''; } } - } - else { + } else { mtrace('Error with MSWord to text converter command : command not set up. Execute once search block configuration.'); return ''; } diff --git a/search/documents/physical_htm.php b/search/documents/physical_htm.php index 76d6073dd69..6cfe5967f52 100644 --- a/search/documents/physical_htm.php +++ b/search/documents/physical_htm.php @@ -15,16 +15,20 @@ /** * @param object $resource -* @uses CFG, USER +* @uses $CFG */ -function get_text_for_indexing_htm(&$resource){ - global $CFG, $USER; +function get_text_for_indexing_htm(&$resource, $directfile = ''){ + global $CFG; // SECURITY : do not allow non admin execute anything on system !! - if (!isadmin($USER->id)) return; + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; // just get text - $text = implode('', file("{$CFG->dataroot}/{$resource->course}/{$resource->reference}")); + if ($directfile == ''){ + $text = implode('', file("{$CFG->dataroot}/{$resource->course}/{$resource->reference}")); + } else { + $text = implode('', file("{$CFG->dataroot}/{$directfile}")); + } // extract keywords and other interesting meta information and put it back as real content for indexing if (preg_match('/(.*)]*)>(.*)/is', $text, $matches)){ @@ -40,7 +44,7 @@ function get_text_for_indexing_htm(&$resource){ $text = preg_replace("/<[^>]*>/", '', $text); $text = preg_replace("//", '', $text); $text = html_entity_decode($text, ENT_COMPAT, 'UTF-8'); - $text = mb_convert_encoding($text, 'UTF-8', 'AUTO'); + $text = mb_convert_encoding($text, 'UTF-8', 'auto'); /* * debug code for tracing input diff --git a/search/documents/physical_html.php b/search/documents/physical_html.php index e03d44566f2..a4929931306 100644 --- a/search/documents/physical_html.php +++ b/search/documents/physical_html.php @@ -16,10 +16,10 @@ /** * @param object $resource */ -function get_text_for_indexing_html(&$resource){ +function get_text_for_indexing_html(&$resource, $directfile = ''){ // wraps to htm handler include_once 'physical_htm.php'; - return get_text_for_indexing_htm($resource); + return get_text_for_indexing_htm($resource, $directfile); } ?> \ No newline at end of file diff --git a/search/documents/physical_odt.php b/search/documents/physical_odt.php new file mode 100644 index 00000000000..2ffa13fe595 --- /dev/null +++ b/search/documents/physical_odt.php @@ -0,0 +1,55 @@ + 1.8 +* @date 2008/03/31 +* @license http://www.gnu.org/copyleft/gpl.html GNU Public License +* +* this is a format handler for getting text out of the opensource ODT binary format +* so it can be indexed by Lucene search engine +*/ + +/** +* OpenOffice Odt extractor +* @param object $resource +* @uses $CFG +*/ +function get_text_for_indexing_odt(&$resource, $directfile = ''){ + global $CFG; + + // SECURITY : do not allow non admin execute anything on system !! + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; + + $moodleroot = (@$CFG->block_search_usemoodleroot) ? "{$CFG->dirroot}/" : '' ; + + // just call pdftotext over stdout and capture the output + if (!empty($CFG->block_search_odt_to_text_cmd)){ + if (!file_exists("{$moodleroot}{$CFG->block_search_odt_to_text_cmd}")){ + mtrace('Error with OpenOffice ODT to text converter command : exectuable not found.'); + } else { + if ($directfile == ''){ + $file = escapeshellarg("{$CFG->dataroot}/{$resource->course}/{$resource->reference}"); + } else { + $file = escapeshellarg("{$CFG->dataroot}/{$directfile}"); + } + $command = trim($CFG->block_search_odt_to_text_cmd); + $text_converter_cmd = "{$moodleroot}{$command} --encoding=UTF-8 $file"; + mtrace("Executing : $text_converter_cmd"); + $result = shell_exec($text_converter_cmd); + if ($result){ + return $result; + } else { + mtrace('Error with OpenOffice ODT to text converter command : execution failed. '); + return ''; + } + } + } else { + mtrace('Error with OpenOffice ODT to text converter command : command not set up. Execute once search block configuration.'); + return ''; + } +} +?> \ No newline at end of file diff --git a/search/documents/physical_pdf.php b/search/documents/physical_pdf.php index ffbd4521682..f1b73f7eaf0 100644 --- a/search/documents/physical_pdf.php +++ b/search/documents/physical_pdf.php @@ -15,13 +15,13 @@ /** * @param object $resource -* @uses CFG, USER +* @uses $CFG */ -function get_text_for_indexing_pdf(&$resource){ - global $CFG, $USER; +function get_text_for_indexing_pdf(&$resource, $directfile = ''){ + global $CFG; // SECURITY : do not allow non admin execute anything on system !! - if (!isadmin($USER->id)) return; + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; // adds moodle root switch if none was defined if (!isset($CFG->block_search_usemoodleroot)){ @@ -35,22 +35,23 @@ function get_text_for_indexing_pdf(&$resource){ preg_match("/^\S+/", $CFG->block_search_pdf_to_text_cmd, $matches); if (!file_exists("{$moodleroot}{$matches[0]}")){ mtrace('Error with pdf to text converter command : executable not found at '.$moodleroot.$matches[0]); - } - else{ - $file = escapeshellarg($CFG->dataroot.'/'.$resource->course.'/'.$resource->reference); + } else { + if ($directfile == ''){ + $file = escapeshellarg("{$CFG->dataroot}/{$resource->course}/{$resource->reference}"); + } else { + $file = escapeshellarg("{$CFG->dataroot}/{$directfile}"); + } $command = trim($CFG->block_search_pdf_to_text_cmd); $text_converter_cmd = "{$moodleroot}{$command} $file -"; $result = shell_exec($text_converter_cmd); if ($result){ return $result; - } - else{ + } else { mtrace('Error with pdf to text converter command : execution failed for '.$text_converter_cmd.'. Check for execution permission on pdf converter executable.'); return ''; } } - } - else { + } else { mtrace('Error with pdf to text converter command : command not set up. Execute once search block configuration.'); return ''; } diff --git a/search/documents/physical_ppt.php b/search/documents/physical_ppt.php index 2de768ffaf8..76b97ae7bd1 100644 --- a/search/documents/physical_ppt.php +++ b/search/documents/physical_ppt.php @@ -29,17 +29,21 @@ /** * @param object $resource -* @uses CFG, USER +* @uses $CFG */ -function get_text_for_indexing_ppt(&$resource){ - global $CFG, $USER; +function get_text_for_indexing_ppt(&$resource, $directfile = ''){ + global $CFG; $indextext = null; // SECURITY : do not allow non admin execute anything on system !! - if (!isadmin($USER->id)) return; + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; - $text = implode('', file("{$CFG->dataroot}/{$resource->course}/{$resource->reference}")); + if ($directfile == ''){ + $text = implode('', file("{$CFG->dataroot}/{$resource->course}/{$resource->reference}")); + } else { + $text = implode('', file("{$CFG->dataroot}/{$directfile}")); + } $remains = $text; $fragments = array(); diff --git a/search/documents/physical_swf.php b/search/documents/physical_swf.php new file mode 100644 index 00000000000..2d8a2628c8b --- /dev/null +++ b/search/documents/physical_swf.php @@ -0,0 +1,69 @@ + 1.8 +* @date 2008/03/31 +* @license http://www.gnu.org/copyleft/gpl.html GNU Public License +* +* @note : The Adobe SWF Converters library is not GPL, although it can be of free use in some +* situations. This file is provided for convenience, but should use having a glance at +* {@link http://www.adobe.com/licensing/developer/} +* +* this is a format handler for getting text out of a proprietary binary format +* so it can be indexed by Lucene search engine +*/ + +/** +* @param object $resource +* @uses $CFG +*/ +function get_text_for_indexing_swf(&$resource, $directfile = ''){ + global $CFG; + + // SECURITY : do not allow non admin execute anything on system !! + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; + + // adds moodle root switch if none was defined + if (!isset($CFG->block_search_usemoodleroot)){ + set_config('block_search_usemoodleroot', 1); + } + + $moodleroot = ($CFG->block_search_usemoodleroot) ? "{$CFG->dirroot}/" : '' ; + + // just call pdftotext over stdout and capture the output + if (!empty($CFG->block_search_pdf_to_text_cmd)){ + $command = trim($CFG->block_search_swf_to_text_cmd); + if (!file_exists("{$moodleroot}{$command}")){ + mtrace('Error with swf to text converter command : executable not found as '.$moodleroot.$command); + } else { + if ($directfile == ''){ + $file = escapeshellarg("{$CFG->dataroot}/{$resource->course}/{$resource->reference}"); + } else { + $file = escapeshellarg("{$CFG->dataroot}/{$directfile}"); + } + $text_converter_cmd = "{$moodleroot}{$command} -t $file"; + $result = shell_exec($text_converter_cmd); + + // result is in html. We must strip it off + $result = preg_replace("/<[^>]*>/", '', $result); + $result = preg_replace("//", '', $result); + $result = html_entity_decode($result, ENT_COMPAT, 'UTF-8'); + $result = mb_convert_encoding($result, 'UTF-8', 'auto'); + + if ($result){ + return $result; + } else { + mtrace('Error with swf to text converter command : execution failed for '.$text_converter_cmd.'. Check for execution permission on swf converter executable.'); + return ''; + } + } + } else { + mtrace('Error with swf to text converter command : command not set up. Execute once search block configuration.'); + return ''; + } +} +?> \ No newline at end of file diff --git a/search/documents/physical_txt.php b/search/documents/physical_txt.php index b20be4152f0..9db35f1bad1 100644 --- a/search/documents/physical_txt.php +++ b/search/documents/physical_txt.php @@ -15,16 +15,21 @@ /** * @param object $resource -* @uses CFG, USER +* @uses $CFG */ -function get_text_for_indexing_txt(&$resource){ - global $CFG, $USER; +function get_text_for_indexing_txt(&$resource, $directfile = ''){ + global $CFG; // SECURITY : do not allow non admin execute anything on system !! - if (!isadmin($USER->id)) return; + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; // just try to get text empirically from ppt binary flow - $text = implode('', file("{$CFG->dataroot}/{$resource->course}/{$resource->reference}")); + if ($directfile == ''){ + $text = implode('', file("{$CFG->dataroot}/{$resource->course}/{$resource->reference}")); + } else { + $text = implode('', file("{$CFG->dataroot}/{$directfile}")); + } + if (!empty($CFG->block_search_limit_index_body)){ $text = shorten($text, $CFG->block_search_limit_index_body); } diff --git a/search/documents/physical_xml.php b/search/documents/physical_xml.php index 85f9ba8f721..9654b54e4f9 100644 --- a/search/documents/physical_xml.php +++ b/search/documents/physical_xml.php @@ -15,13 +15,13 @@ /** * @param object $resource -* @uses CFG, USER +* @uses $CFG */ function get_text_for_indexing_xml(&$resource, $directfile = ''){ - global $CFG, $USER; + global $CFG; // SECURITY : do not allow non admin execute anything on system !! - if (!isadmin($USER->id)) return; + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) return; // just get text if ($directfile == ''){ diff --git a/search/documents/resource_document.php b/search/documents/resource_document.php index 844da54bc7a..3b22f119da2 100644 --- a/search/documents/resource_document.php +++ b/search/documents/resource_document.php @@ -44,7 +44,7 @@ class ResourceSearchDocument extends SearchDocument { $data = array(); // construct the parent class - parent::__construct($doc, $data, $resource['course'], 0, 0, PATH_FOR_SEARCH_TYPE_RESOURCE); + parent::__construct($doc, $data, $resource['course'], 0, 0, 'mod/'.SEARCH_TYPE_RESOURCE); } //constructor } //ResourceSearchDocument @@ -95,15 +95,15 @@ function resource_get_content_for_index(&$notneeded) { alltext != ' ' AND type != 'file' "; - $resources = get_records_sql($query); - - foreach($resources as $aResource){ - $coursemodule = get_field('modules', 'id', 'name', 'resource'); - $cm = get_record('course_modules', 'course', $aResource->course, 'module', $coursemodule, 'instance', $aResource->id); - $context = get_context_instance(CONTEXT_MODULE, $cm->id); - $aResource->id = $cm->id; - $documents[] = new ResourceSearchDocument(get_object_vars($aResource), $context->id); - mtrace("finished $aResource->name"); + if ($resources = get_records_sql($query)){ + foreach($resources as $aResource){ + $coursemodule = get_field('modules', 'id', 'name', 'resource'); + $cm = get_record('course_modules', 'course', $aResource->course, 'module', $coursemodule, 'instance', $aResource->id); + $context = get_context_instance(CONTEXT_MODULE, $cm->id); + $aResource->id = $cm->id; + $documents[] = new ResourceSearchDocument(get_object_vars($aResource), $context->id); + mtrace("finished $aResource->name"); + } } // special physical files handling @@ -135,10 +135,8 @@ function resource_get_content_for_index(&$notneeded) { cm.module = m.id AND m.name = 'resource' "; - $resources = get_records_sql($query); - + if ($resources = get_records_sql($query)){ // invokes external content extractor if exists. - if ($resources){ foreach($resources as $aResource){ // fetches a physical indexable document and adds it to documents passed by ref $coursemodule = get_field('modules', 'id', 'name', 'resource'); @@ -323,6 +321,11 @@ function resource_check_text_access($path, $itemtype, $this_id, $user, $group_id * @param string $title */ function resource_link_post_processing($title){ - return mb_convert_encoding($title, 'UTF-8', 'auto'); + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); } ?> \ No newline at end of file diff --git a/search/documents/techproject_document.php b/search/documents/techproject_document.php deleted file mode 100644 index 53f5a4a81ae..00000000000 --- a/search/documents/techproject_document.php +++ /dev/null @@ -1,301 +0,0 @@ - 1.8 -* @date 2008/03/31 -* @license http://www.gnu.org/copyleft/gpl.html GNU Public License -* -* document handling for techproject activity module -*/ - -/** -* requires and includes -*/ -require_once("$CFG->dirroot/search/documents/document.php"); -require_once("$CFG->dirroot/mod/techproject/lib.php"); - -/** -* a class for representing searchable information -* -*/ -class TechprojectEntrySearchDocument extends SearchDocument { - - /** - * constructor - * - */ - public function __construct(&$entry, $course_id, $context_id) { - // generic information - $doc->docid = $entry['id']; - $doc->documenttype = SEARCH_TYPE_TECHPROJECT; - $doc->itemtype = $entry['entry_type']; - $doc->contextid = $context_id; - - - $doc->title = $entry['abstract']; - $doc->author = ($entry['userid']) ? $entry['author'] : ''; - $doc->contents = strip_tags($entry['description']); - $doc->date = ''; - - $doc->url = techproject_make_link($entry['projectid'], $entry['id'], $entry['entry_type'], $entry['groupid']); - - // module specific information - $data->techproject = $entry['projectid']; - - parent::__construct($doc, $data, $course_id, $entry['groupid'], $entry['userid'], PATH_FOR_SEARCH_TYPE_TECHPROJECT); - } //constructor -} //TechprojectEntrySearchDocument - -/** -* constructs a valid link to a description detail -* -*/ -function techproject_make_link($techproject_id, $entry_id, $entry_type, $group_id) { - global $CFG; - return $CFG->wwwroot.'/mod/techproject/view.php?view=view_detail&id='.$techproject_id.'&objectId='.$entry_id.'&objectClass='.$entry_type.'&group='.$group_id; -} //techproject_make_link - -/** -* search standard API -* -*/ -function techproject_iterator() { - $techprojects = get_records('techproject'); - return $techprojects; -} //techproject_iterator - -/** -* search standard API -* @param techproject a techproject instance -* @return an array of collected searchable documents -*/ -function techproject_get_content_for_index(&$techproject) { - $documents = array(); - if (!$techproject) return $documents; - - $requirements = techproject_get_entries($techproject->id, 'requirement'); - $specifications = techproject_get_entries($techproject->id, 'specification'); - $tasks = techproject_get_tasks($techproject->id); - $milestones = techproject_get_entries($techproject->id, 'milestone'); - $deliverables = techproject_get_entries($techproject->id, 'deliverable'); - $coursemodule = get_field('modules', 'id', 'name', 'techproject'); - $cm = get_record('course_modules', 'course', $techproject->course, 'module', $coursemodule, 'instance', $techproject->id); - $context = get_context_instance(CONTEXT_MODULE, $cm->id); - - $entries = @array_merge($requirements, $specifications, $milestones, $deliverables); - if ($entries){ - foreach($entries as $anEntry) { - if ($anEntry) { - if (strlen($anEntry->description) > 0) { - $anEntry->author = ''; - $documents[] = new TechprojectEntrySearchDocument(get_object_vars($anEntry), $techproject->course, $context->id); - } - } - } - } - if ($tasks){ - foreach($tasks as $aTask) { - if ($aTask) { - if (strlen($aTask->description) > 0) { - if ($aTask->assignee){ - $user = get_record('user', 'id', $aTask->assignee); - $aTask->author = $user->firstname.' '.$user->lastname; - } - $documents[] = new TechprojectEntrySearchDocument(get_object_vars($aTask), $techproject->course, $context->id); - } - } - } - } - return $documents; -} //techproject_get_content_for_index - -/** -* returns a single techproject search document based on a techproject_entry id and itemtype -* -*/ -function techproject_single_document($id, $itemtype) { - switch ($itemtype){ - case 'requirement':{ - $entry = get_record('techproject_requirement', 'id', $id); - $entry->author = ''; - break; - } - case 'specification':{ - $entry = get_record('techproject_specification', 'id', $id); - $entry->author = ''; - break; - } - case 'milestone':{ - $entry = get_record('techproject_milestone', 'id', $id); - $entry->author = ''; - break; - } - case 'deliverable':{ - $entry = get_record('techproject_deliverable', 'id', $id); - $entry->author = ''; - break; - } - case 'task':{ - $entry = get_record('techproject_task', 'id', $id); - if ($entry->assignee){ - $user = get_record('user', 'id', $entry->assignee); - $entry->author = $user->firstname.' '.$user->lastname; - } - break; - } - } - $techproject_course = get_field('techproject', 'course', 'id', $entry->projectid); - $coursemodule = get_field('modules', 'id', 'name', 'techproject'); - $cm = get_record('course_modules', 'course', $techproject_course, 'module', $coursemodule, 'instance', $entry->projectid); - $context = get_context_instance(CONTEXT_MODULE, $cm->id); - $entry->type = $itemtype; - $techproject = get_record('techproject', 'id', $requirement->projectid); - return new TechprojectEntrySearchDocument(get_object_vars($anEntry), $techproject->course, $context->id); -} //techproject_single_document - -/** -* dummy delete function that packs id with itemtype. -* this was here for a reason, but I can't remember it at the moment. -* -*/ -function techproject_delete($info, $itemtype) { - $object->id = $info; - $object->itemtype = $itemtype; - return $object; -} //techproject_delete - -/** -* returns the var names needed to build a sql query for addition/deletions -* -*/ -// TODO : what should we do there ? -function techproject_db_names() { - //[primary id], [table name], [time created field name], [time modified field name] - return array( - array('id', 'techproject_requirement', 'created', 'modified', 'requirement'), - array('id', 'techproject_specification', 'created', 'modified', 'specification'), - array('id', 'techproject_task', 'created', 'modified', 'task'), - array('id', 'techproject_milestone', 'created', 'modified', 'milestone'), - array('id', 'techproject_deliverable', 'created', 'modified', 'deliverable') - ); -} //techproject_db_names - -/** -* get a complete list of entries of one particular type -* @param techprojectId the project instance -* @param type the entity type -* @return an array of records -*/ -function techproject_get_entries($techproject_id, $type) { - global $CFG; - - $query = " - SELECT - e.id, - e.abstract, - e.description, - e.projectid, - e.groupid, - e.userid, - '$type' AS entry_type - FROM - {$CFG->prefix}techproject_{$type} AS e - WHERE - e.projectid = '{$techproject_id}' - "; - return get_records_sql($query); -} //techproject_get_entries - -/** -* get the task list for a project instance -* @param techprojectId the project -* @return an array of records that represent tasks -*/ -function techproject_get_tasks($techproject_id) { - global $CFG; - - $query = " - SELECT - t.id, - t.abstract, - t.description, - t.projectid, - t.groupid, - t.owner as userid, - u.firstname, - u.lastname, - 'task' as entry_type - FROM - {$CFG->prefix}techproject_task AS t - LEFT JOIN - {$CFG->prefix}user AS u - ON - t.owner = u.id - WHERE - t.projectid = '{$techproject_id}' - ORDER BY - t.taskstart ASC - "; - return get_records_sql($query); -} //techproject_get_tasks - -/** -* this function handles the access policy to contents indexed as searchable documents. If this -* function does not exist, the search engine assumes access is allowed. -* When this point is reached, we already know that : -* - user is legitimate in the surrounding context -* - user may be guest and guest access is allowed to the module -* - the function may perform local checks within the module information logic -* @param path the access path to the module script code -* @param entry_type the information subclassing (usefull for complex modules, defaults to 'standard') -* @param this_id the item id within the information class denoted by entry_type. In techprojects, this id -* points to the techproject instance in which all resources are indexed. -* @param user the user record denoting the user who searches -* @param group_id the current group used by the user when searching -* @return true if access is allowed, false elsewhere -*/ -function techproject_check_text_access($path, $entry_type, $this_id, $user, $group_id, $context_id){ - global $CFG; - - include_once("{$CFG->dirroot}/{$path}/lib.php"); - - // get the techproject object and all related stuff - $techproject = get_record('techproject', 'id', $this_id); - $course = get_record('course', 'id', $techproject->course); - $module_context = get_record('context', 'id', $context_id); - $cm = get_record('course_modules', 'id', $module_context->instanceid); - if (!$cm->visible and !has_capability('moodle/course:viewhiddenactivities', $module_context)) return false; - - //group consistency check : checks the following situations about groups - // if user is guest check access capabilities for guests : - // guests can see default project, and other records if groups are liberal - // TODO : change guestsallowed in a capability - if (isguest() && $techproject->guestsallowed){ - if ($group_id && groupmode($course, $cm) == SEPARATEGROUPS) - return false; - return true; - } - - // trap if user is not same group and groups are separated - $current_group = get_current_group($course->id); - if ((groupmode($course) == SEPARATEGROUPS) && $group_id != $current_group && $group_id) return false; - - //trap if ungroupedsees is off in strict access mode and user is not teacher - if ((groupmode($course) == SEPARATEGROUPS) && !$techproject->ungroupedsees && !$group_id && isteacher($user->id)) return false; - - return true; -} //techproject_check_text_access - -/** -* this call back is called when displaying the link for some last post processing -* -*/ -function techproject_link_post_processing($title){ - return mb_convert_encoding($title, 'UTF-8', 'auto'); -} - -?> \ No newline at end of file diff --git a/search/documents/user_document.php b/search/documents/user_document.php new file mode 100644 index 00000000000..83eac3abab5 --- /dev/null +++ b/search/documents/user_document.php @@ -0,0 +1,366 @@ + 1.8 +* @date 2008/03/31 +* @license http://www.gnu.org/copyleft/gpl.html GNU Public License +* +* special (EXTRA) document handling for user related data +* +*/ + +/** +* includes and requires +*/ +require_once("$CFG->dirroot/search/documents/document.php"); +require_once("$CFG->dirroot/blog/lib.php"); + +/** +* a class for representing searchable information in user metadata +* +*/ +class UserSearchDocument extends SearchDocument { + + /** + * constructor + */ + public function __construct(&$userhash, $user_id, $context_id) { + + // generic information; required + $doc->docid = $userhash['id']; + $doc->documenttype = SEARCH_TYPE_USER; + $doc->itemtype = 'user'; + $doc->contextid = $context_id; + + $user = get_record('user', 'id', $user_id); + $doc->title = get_string('user').': '.fullname($user); + $doc->date = ($userhash['lastaccess']) ? $userhash['lastaccess'] : time() ; + + //remove '(ip.ip.ip.ip)' from chat author list + $doc->author = $user->id; + $doc->contents = $userhash['description']; + $doc->url = user_make_link($user_id, 'user'); + + // module specific information; optional + + // construct the parent class + parent::__construct($doc, $data, 0, 0, 0, PATH_FOR_SEARCH_TYPE_USER); + } +} + +/** +* a class for representing searchable information in user metadata +* +*/ +class UserPostSearchDocument extends SearchDocument { + + /** + * constructor + */ + public function __construct(&$post, $user_id, $context_id) { + // generic information; required + $doc->docid = $post['id']; + $doc->documenttype = SEARCH_TYPE_USER; + $doc->itemtype = 'post'; + $doc->contextid = $context_id; + + $user = get_record('user', 'id', $user_id); + + // we cannot call userdate with relevant locale at indexing time. + $doc->title = get_string('post').': '.fullname($user); + $doc->date = $post['created']; + + //remove '(ip.ip.ip.ip)' from chat author list + $doc->author = fullname($user); + $doc->contents = $post['description']; + $doc->url = user_make_link($user_id, 'post'); + + // module specific information; optional + + // construct the parent class + parent::__construct($doc, $data, 0, 0, 0, PATH_FOR_SEARCH_TYPE_USER); + } +} + +/** +* a class for representing searchable information in user metadata +* +*/ +class UserBlogAttachmentSearchDocument extends SearchDocument { + + /** + * constructor + */ + public function __construct(&$post, $context_id) { + // generic information; required + $doc->docid = $post['id']; + $doc->documenttype = SEARCH_TYPE_USER; + $doc->itemtype = 'attachment'; + $doc->contextid = $context_id; + + $user = get_record('user', 'id', $post['userid']); + + // we cannot call userdate with relevant locale at indexing time. + $doc->title = get_string('file').' : '.$post['subject']; + $doc->date = $post['created']; + + //remove '(ip.ip.ip.ip)' from chat author list + $doc->author = fullname($user); + $doc->contents = $post['alltext']; + $doc->url = user_make_link($post['id'], 'attachment'); + + // module specific information; optional + + // construct the parent class + parent::__construct($doc, $data, 0, 0, 0, PATH_FOR_SEARCH_TYPE_USER); + } +} + + +/** +* constructs a valid link to a user record +* @param userid the user +* @param itemtype +* @uses CFG +* @return a well formed link to user information +*/ +function user_make_link($itemid, $itemtype) { + global $CFG; + + if ($itemtype == 'user'){ + return $CFG->wwwroot.'/user/view.php?id='.$itemid; + } elseif ($itemtype == 'post') { + return $CFG->wwwroot.'/blog/index.php?userid='.$itemid; + } elseif ($itemtype == 'attachment') { + $post = get_record('post', 'id', $itemid); + if (!$CFG->slasharguments){ + return $CFG->wwwroot."/file.php?file=/blog/attachments/{$post->id}/{$post->attachment}"; + } else { + return $CFG->wwwroot."/file.php/blog/attachments/{$post->id}/{$post->attachment}"; + } + } else { + return null; + } +} + +/** +* part of search engine API +* +*/ +function user_iterator() { + $users = get_records('user'); + return $users; +} + +/** +* part of search engine API +* @uses CFG +* @return an array of documents generated from data +*/ +function user_get_content_for_index(&$user) { + global $CFG; + + $documents = array(); + + $userhash = get_object_vars($user); + $documents[] = new UserSearchDocument($userhash, $user->id, null); + + if ($posts = get_records('post', 'userid', $user->id, 'created')){ + foreach($posts as $post){ + $texts = array(); + $texts[] = $post->subject; + $texts[] = $post->summary; + $texts[] = $post->content; + $post->description = implode(" ", $texts); + + // record the attachment if any and physical files can be indexed + if (@$CFG->block_search_enable_file_indexing){ + if ($post->attachment){ + user_get_physical_file($post, null, false, $documents); + } + } + + $posthash = get_object_vars($post); + $documents[] = new UserPostSearchDocument($posthash, $user->id, null); + } + } + return $documents; +} + +/** +* get text from a physical file +* @param object $post a post to whech the file is attached to +* @param boolean $context_id if in future we need recording a context along with the search document, pass it here +* @param boolean $getsingle if true, returns a single search document, elsewhere return the array +* given as documents increased by one +* @param array $documents the array of documents, by ref, where to add the new document. +* @return a search document when unique or false. +*/ +function user_get_physical_file(&$post, $context_id, $getsingle, &$documents = null){ + global $CFG; + + // cannot index empty references + if (empty($post->attachment)){ + mtrace("Cannot index, empty reference."); + return false; + } + + $fileparts = pathinfo($post->attachment); + // cannot index unknown or masked types + if (empty($fileparts['extension'])) { + mtrace("Cannot index without explicit extension."); + return false; + } + + // cannot index non existent file + $file = "{$CFG->dataroot}/blog/attachments/{$post->id}/{$post->attachment}"; + if (!file_exists($file)){ + mtrace("Missing attachment file $file : will not be indexed."); + return false; + } + + $ext = strtolower($fileparts['extension']); + + // cannot index unallowed or unhandled types + if (!preg_match("/\b$ext\b/i", $CFG->block_search_filetypes)) { + mtrace($fileparts['extension'] . ' is not an allowed extension for indexing'); + return false; + } + if (file_exists($CFG->dirroot.'/search/documents/physical_'.$ext.'.php')){ + include_once($CFG->dirroot.'/search/documents/physical_'.$ext.'.php'); + $function_name = 'get_text_for_indexing_'.$ext; + $directfile = "blog/attachments/{$post->id}/{$post->attachment}"; + $post->alltext = $function_name($post, $directfile); + if (!empty($post->alltext)){ + if ($getsingle){ + $posthash = get_object_vars($post); + $single = new UserBlogAttachmentSearchDocument($posthash, $context_id); + mtrace("finished attachment {$post->attachment} in {$post->title}"); + return $single; + } else { + $posthash = get_object_vars($post); + $documents[] = new UserBlogAttachmentSearchDocument($posthash, $context_id); + } + mtrace("finished attachment {$post->attachment} in {$post->subject}"); + } + } else { + mtrace("fulltext handler not found for $ext type"); + } + return false; +} + +/** +* returns a single user search document +* @param composite $id a unique document id made with +* @param itemtype the type of information (session is the only type) +*/ +function user_single_document($id, $itemtype) { + if ($itemtype == 'user'){ + if ($user = get_record('user', 'id', $id)){ + $userhash = get_object_vars($user); + return new UserSearchDocument($userhash, $user->id, 'user', null); + } + } elseif ($itemtype == 'post') { + if ($post = get_records('post', 'id', $id)){ + $texts = array(); + $texts[] = $post->subject; + $texts[] = $post->summary; + $texts[] = $post->content; + $post->description = implode(" ", $texts); + $posthash = get_object_vars($post); + return new UserPostSearchDocument($posthash, $user->id, 'post', null); + } + } elseif ($itemtype == 'attachment' && @$CFG->block_search_enable_file_indexing) { + if ($post = get_records('post', 'id', $id)){ + if ($post->attachment){ + return user_get_physical_file($post, null, true); + } + } + } + return null; +} + +/** +* dummy delete function that packs id with itemtype. +* this was here for a reason, but I can't remember it at the moment. +* +*/ +function user_delete($info, $itemtype) { + $object->id = $info; + $object->itemtype = $itemtype; + return $object; +} + +/** +* returns the var names needed to build a sql query for addition/deletions +* attachments are indirect records, linked to its post +*/ +function user_db_names() { + //[primary id], [table name], [time created field name], [time modified field name] [itemtype] [select restriction clause] + return array( + array('id', 'user', 'firstaccess', 'timemodified', 'user'), + array('id', 'post', 'created', 'lastmodified', 'post'), + array('id', 'post', 'created', 'lastmodified', 'attachment') + ); +} + +/** +* this function handles the access policy to contents indexed as searchable documents. If this +* function does not exist, the search engine assumes access is allowed. +* When this point is reached, we already know that : +* - user is legitimate in the surrounding context +* - user may be guest and guest access is allowed to the module +* - the function may perform local checks within the module information logic +* @param path the access path to the module script code +* @param itemtype the information subclassing (usefull for complex modules, defaults to 'standard') +* @param this_id the item id within the information class denoted by entry_type. In chats, this id +* points out a session history which is a close sequence of messages. +* @param user the user record denoting the user who searches +* @param group_id the current group used by the user when searching +* @uses CFG +* @return true if access is allowed, false elsewhere +*/ +function user_check_text_access($path, $itemtype, $this_id, $user, $group_id, $context_id){ + global $CFG; + + include_once("{$CFG->dirroot}/{$path}/lib.php"); + + if ($itemtype == 'user'){ + // get the user + $userrecord = get_record('user', 'id', $this_id); + + // we cannot see nothing from unconfirmed users + if (!$userrecord->confirmed and !has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))){ + if (!empty($CFG->search_access_debug)) echo "search reject : unconfirmed user "; + return false; + } + } elseif ($itemtype == 'post' || $itemtype == 'attachment'){ + // get the post + $post = get_record('post', 'id', $this_id); + $userrecord = get_record('user', 'id', $post->userid); + + // we can try using blog visibility check + return blog_user_can_view_user_post($user->id, $post); + } + $context = get_record('context', 'id', $context_id); + + return true; +} + +/** +* this call back is called when displaying the link for some last post processing +* +*/ +function user_link_post_processing($title){ + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); +} +?> \ No newline at end of file diff --git a/search/documents/wiki_document.php b/search/documents/wiki_document.php index d23ca9e8a4b..113dc39118b 100644 --- a/search/documents/wiki_document.php +++ b/search/documents/wiki_document.php @@ -50,7 +50,7 @@ class WikiSearchDocument extends SearchDocument { $data->wiki = $wiki_id; // construct the parent class - parent::__construct($doc, $data, $course_id, $group_id, $user_id, PATH_FOR_SEARCH_TYPE_WIKI); + parent::__construct($doc, $data, $course_id, $group_id, $user_id, 'mod/'.SEARCH_TYPE_WIKI); } } @@ -268,7 +268,12 @@ function wiki_check_text_access($path, $itemtype, $this_id, $user, $group_id, $c * */ function wiki_link_post_processing($title){ - return mb_convert_encoding($title, 'UTF-8', 'auto'); + global $CFG; + + if ($CFG->block_search_utf8dir){ + return mb_convert_encoding($title, 'UTF-8', 'auto'); + } + return mb_convert_encoding($title, 'auto', 'UTF-8'); } ?> \ No newline at end of file diff --git a/search/tests/index.php b/search/tests/index.php index 2b0290292b2..430d3b6b5c8 100644 --- a/search/tests/index.php +++ b/search/tests/index.php @@ -1,106 +1,148 @@ dirroot/search/lib.php"); + require_once('../../config.php'); + require_once("$CFG->dirroot/search/lib.php"); -require_login(); - -$strsearch = get_string('search', 'search'); -$strquery = get_string('stats'); +/// makes inclusions of the Zend Engine more reliable + $separator = (array_key_exists('WINDIR', $_SERVER)) ? ';' : ':' ; + ini_set('include_path', $CFG->dirroot.'\search'.$separator.ini_get('include_path')); + + require_login(); -$navlinks[] = array('name' => $strsearch, 'link' => "../index.php", 'type' => 'misc'); -$navlinks[] = array('name' => $strquery, 'link' => "../stats.php", 'type' => 'misc'); -$navlinks[] = array('name' => get_string('runindexertest','search'), 'link' => null, 'type' => 'misc'); -$navigation = build_navigation($navlinks); -$site = get_site(); -print_header("$strsearch", "$site->fullname" , $navigation, "", "", true, " ", navmenu($site)); - -if (empty($CFG->enableglobalsearch)) { - error('Global searching is not enabled.'); -} + if (empty($CFG->enableglobalsearch)) { + error('Global searching is not enabled.'); + } + + if (!has_capability('moodle/site:doanything', get_context_instance(CONTEXT_SYSTEM))) { + error("You need to be an admin user to use this page.", "$CFG->wwwroot/login/index.php"); + } //if + + mtrace('
Server Time: '.date('r',time()));
+    mtrace("Testing global search capabilities:\n");
+    
+    $phpversion = phpversion();
+    
+    //fix paths for testing
+    set_include_path(get_include_path().":../");
+    require_once("$CFG->dirroot/search/Zend/Search/Lucene.php");
+    
+    mtrace("Checking activity modules:\n");
+    
+    //the presence of the required search functions -
+    // * mod_iterator
+    // * mod_get_content_for_index
+    //are the sole basis for including a module in the index at the moment.
+    
+/// get all installed modules
+    if ($mods = get_records('modules', '', '', 'name', 'id,name')){
 
-if (!isadmin()) {
-    error("You need to be an admin user to use this page.", "$CFG->wwwroot/login/index.php");
-} //if
+        $searchabletypes = array_values(search_get_document_types());
 
-mtrace('
Server Time: '.date('r',time()));
-mtrace("Testing global search capabilities:\n");
+        foreach($mods as $mod){
+            if (in_array($mod->name, $searchabletypes)){
+                $mod->location = 'internal';
+                $searchables[] = $mod;
+            } else {
+                $documentfile = $CFG->dirroot."/mod/{$mod->name}/search_document.php";
+                $mod->location = 'mod';
+                if (file_exists($documentfile)){
+                    $searchables[] = $mod;
+                }
+            }        
+        }    
+        mtrace(count($searchables).' modules to search in / '.count($mods).' modules found.');
+    }
+      
+/// collects blocks as indexable information may be found in blocks either
+    if ($blocks = get_records('block', '', '', 'name', 'id,name')) {
+        $blocks_searchables = array();
+        // prepend the "block_" prefix to discriminate document type plugins
+        foreach($blocks as $block){
+            $block->dirname = $block->name;
+            $block->name = 'block_'.$block->name;
+            if (in_array('SEARCH_TYPE_'.strtoupper($block->name), $searchabletypes)){
+                $mod->location = 'internal';
+                $blocks_searchables[] = $block;
+            } else {
+                $documentfile = $CFG->dirroot."/blocks/{$block->dirname}/search_document.php";
+                if (file_exists($documentfile)){
+                    $mod->location = 'blocks';
+                    $blocks_searchables[] = $block;
+                }
+            }        
+        }    
+        mtrace(count($blocks_searchables).' blocks to search in / '.count($blocks).' blocks found.');
+        $searchables = array_merge($searchables, $blocks_searchables);
+    }
+      
+/// add virtual modules onto the back of the array
 
-$phpversion = phpversion();
+    $additional = search_get_additional_modules();
+    mtrace(count($additional).' additional to search in.');
+    $searchables = array_merge($searchables, $additional);
+    
+    foreach ($searchables as $mod) {
 
-if (!search_check_php5()) {
-    mtrace("ERROR: PHP 5.0.0 or later required (currently using version $phpversion).");
-    exit(0);
-} else {
-    mtrace("Success: PHP 5.0.0 or later is installed ($phpversion).\n");
-} //else
-
-//fix paths for testing
-set_include_path(get_include_path().":../");
-require_once("$CFG->dirroot/search/Zend/Search/Lucene.php");
-
-mtrace("Checking activity modules:\n");
-
-//the presence of the required search functions -
-// * mod_iterator
-// * mod_get_content_for_index
-//are the sole basis for including a module in the index at the moment.
-
-if ($mods = get_records_select('modules')) {
-    $mods = array_merge($mods, search_get_additional_modules());
-
-    foreach ($mods as $mod) {
-        $class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php';
+        $key = 'search_in_'.$mod->name;
+        if (isset($CFG->$key) && !$CFG->$key) {
+            mtrace("module $key has been administratively disabled. Skipping...\n");
+            continue;
+        }
 
+        if ($mod->location == 'internal'){
+            $class_file = $CFG->dirroot.'/search/documents/'.$mod->name.'_document.php';
+        } else {
+            $class_file = $CFG->dirroot.'/'.$mod->location.'/'.$mod->name.'/search_document.php';
+        }
+    
         if (file_exists($class_file)) {
             include_once($class_file);
-
-            if (!defined('SEARCH_TYPE_'.strtoupper($mod->name))) {
-                mtrace("ERROR: Constant 'SEARCH_TYPE_".strtoupper($mod->name)."' is not defined in /search/lib.php");
+    
+            if ($mod->location != 'internal' && !defined('X_SEARCH_TYPE_'.strtoupper($mod->name))) {
+                mtrace("ERROR: Constant 'X_SEARCH_TYPE_".strtoupper($mod->name)."' is not defined in search/searchtypes.php or in module");
                 continue;
-            } //if
-
+            }
+            
             $iter_function = $mod->name.'_iterator';
             $index_function = $mod->name.'_get_content_for_index';
-
+            
             if (function_exists($index_function) && function_exists($iter_function)) {
-                if (is_array($iter_function())) {
-                    $documents = $index_function(array_pop($iter_function()));
-
+                $entries = $iter_function();
+                if (!empty($entries)) {
+                    $documents = $index_function(array_pop($entries));
+            
                     if (is_array($documents)) {
                         mtrace("Success: '$mod->name' module seems to be ready for indexing.");
                     } else {
                         mtrace("ERROR: $index_function() doesn't seem to be returning an array.");
-                    } //else
+                    } 
                 } else {
-                    mtrace("ERROR: $iter_function() doesn't seem to be returning an object array.");
-                } //else
+                    mtrace("Success : '$mod->name' has nothing to index.");
+                } 
             } else {
                 mtrace("ERROR: $iter_function() and/or $index_function() does not exist in $class_file");
-            } //else
+            } 
         } else {
             mtrace("Notice: $class_file does not exist, this module will not be indexed.");
-        } //else
-    } //foreach
-} //if
-
-//finished modules
-mtrace("\nFinished checking activity modules.");
-
-//now blocks...
-//
-
-mtrace("
Back to query page or Start indexing.");
-mtrace('
');
-print_footer();
+        } 
+    } 
+    
+    //finished modules
+    mtrace("\nFinished checking activity modules.");
+    
+    //now blocks...
+    //
+    
+    mtrace("
Back to query page or Start indexing.");
+    mtrace('
'); ?> \ No newline at end of file